mirror of
https://github.com/HKUDS/nanobot.git
synced 2026-08-04 16:38:49 +00:00
feat(dream): single-phase consolidation (cherry-pick from feat/single-phase-dream)
This commit is contained in:
parent
418cb23da2
commit
8dd35c373a
@ -8,6 +8,7 @@ import os
|
||||
import time
|
||||
from contextlib import AsyncExitStack, nullcontext, suppress
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from enum import Enum, auto
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Awaitable, Callable
|
||||
@ -19,7 +20,13 @@ from nanobot.agent import model_presets as preset_helpers
|
||||
from nanobot.agent.autocompact import AutoCompact
|
||||
from nanobot.agent.context import ContextBuilder
|
||||
from nanobot.agent.hook import AgentHook, CompositeHook
|
||||
from nanobot.agent.memory import Consolidator, Dream
|
||||
from nanobot.agent.memory import (
|
||||
_STALE_THRESHOLD_DAYS,
|
||||
Consolidator,
|
||||
Dream,
|
||||
_estimate_tokens,
|
||||
_strip_skip_lines,
|
||||
)
|
||||
from nanobot.agent.progress_hook import AgentProgressHook
|
||||
from nanobot.agent.runner import _MAX_INJECTIONS_PER_TURN, AgentRunner, AgentRunSpec
|
||||
from nanobot.agent.subagent import SubagentManager
|
||||
@ -34,6 +41,8 @@ from nanobot.config.schema import AgentDefaults, ModelPresetConfig
|
||||
from nanobot.providers.base import LLMProvider
|
||||
from nanobot.providers.factory import ProviderSnapshot
|
||||
from nanobot.session.goal_state import (
|
||||
GOAL_STATE_KEY,
|
||||
goal_state_runtime_lines,
|
||||
runner_wall_llm_timeout_s,
|
||||
)
|
||||
from nanobot.session.manager import Session, SessionManager
|
||||
@ -47,7 +56,11 @@ from nanobot.utils.helpers import image_placeholder_text
|
||||
from nanobot.utils.helpers import truncate_text as truncate_text_fn
|
||||
from nanobot.utils.image_generation_intent import image_generation_prompt
|
||||
from nanobot.utils.llm_runtime import LLMRuntime
|
||||
from nanobot.utils.runtime import EMPTY_FINAL_RESPONSE_MESSAGE
|
||||
from nanobot.utils.prompt_templates import _TEMPLATES_ROOT, render_template
|
||||
from nanobot.utils.runtime import (
|
||||
EMPTY_FINAL_RESPONSE_MESSAGE,
|
||||
SUSTAINED_GOAL_CONTINUE_PROMPT,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from nanobot.config.schema import (
|
||||
@ -191,6 +204,7 @@ class AgentLoop:
|
||||
model_preset: str | None = None,
|
||||
preset_snapshot_loader: preset_helpers.PresetSnapshotLoader | None = None,
|
||||
runtime_model_publisher: Callable[[str, str | None], None] | None = None,
|
||||
dream_model_override: str | None = None,
|
||||
):
|
||||
from nanobot.config.schema import ToolsConfig
|
||||
|
||||
@ -203,6 +217,7 @@ class AgentLoop:
|
||||
self._preset_snapshot_loader = preset_snapshot_loader
|
||||
self._runtime_model_publisher = runtime_model_publisher
|
||||
self._provider_signature = provider_signature
|
||||
self._dream_model_override = dream_model_override
|
||||
self._default_selection_signature = preset_helpers.default_selection_signature(provider_signature)
|
||||
self.workspace = workspace
|
||||
self.model = model or provider.get_default_model()
|
||||
@ -310,6 +325,7 @@ class AgentLoop:
|
||||
self._active_preset: str | None = None
|
||||
if model_preset:
|
||||
self.set_model_preset(model_preset, publish_update=False)
|
||||
self._configure_dream()
|
||||
self._register_default_tools()
|
||||
self._runtime_vars: dict[str, Any] = {}
|
||||
self._current_iteration: int = 0
|
||||
@ -369,6 +385,7 @@ class AgentLoop:
|
||||
model_preset=defaults.model_preset,
|
||||
provider_snapshot_loader=provider_snapshot_loader,
|
||||
preset_snapshot_loader=preset_snapshot_loader,
|
||||
dream_model_override=config.agents.defaults.dream.model_override,
|
||||
**extra,
|
||||
)
|
||||
|
||||
@ -394,7 +411,7 @@ class AgentLoop:
|
||||
self.runner.provider = provider
|
||||
self.subagents.set_provider(provider, model)
|
||||
self.consolidator.set_provider(provider, model, context_window_tokens)
|
||||
self.dream.set_provider(provider, model)
|
||||
self._configure_dream()
|
||||
self._provider_signature = snapshot.signature
|
||||
if publish_update and self._runtime_model_publisher is not None:
|
||||
self._runtime_model_publisher(
|
||||
@ -403,6 +420,20 @@ class AgentLoop:
|
||||
)
|
||||
logger.info("Runtime model switched for next turn: {} -> {}", old_model, model)
|
||||
|
||||
def _configure_dream(self) -> None:
|
||||
"""Apply dream.model_override, resolving preset names if needed."""
|
||||
if not self._dream_model_override:
|
||||
self.dream.set_provider(self.provider, self.model)
|
||||
return
|
||||
|
||||
if self._dream_model_override in self.model_presets:
|
||||
snapshot = self._build_model_preset_snapshot(self._dream_model_override)
|
||||
self.dream.set_provider(snapshot.provider, snapshot.model)
|
||||
return
|
||||
|
||||
# Raw model name fallback — same provider, different model
|
||||
self.dream.set_provider(self.provider, self._dream_model_override)
|
||||
|
||||
def _refresh_provider_snapshot(self) -> None:
|
||||
if self._provider_snapshot_loader is None:
|
||||
return
|
||||
@ -1006,6 +1037,28 @@ class AgentLoop:
|
||||
msg.chat_id.split(":", 1) if ":" in msg.chat_id else ("cli", msg.chat_id)
|
||||
)
|
||||
logger.info("Processing system message from {}", msg.sender_id)
|
||||
if msg.sender_id == "dream":
|
||||
session_key = "system:dream"
|
||||
session = self.sessions.get_or_create(session_key)
|
||||
session.metadata["is_dream"] = True
|
||||
# Capture trigger source on first batch so _dream_finalize_commit
|
||||
# can notify the user who ran /dream (cron-triggered runs have no trigger).
|
||||
if "_dream_trigger_channel" not in session.metadata:
|
||||
trigger_ch = msg.metadata.get("trigger_channel")
|
||||
trigger_ci = msg.metadata.get("trigger_chat_id")
|
||||
if trigger_ch and trigger_ci:
|
||||
session.metadata["_dream_trigger_channel"] = trigger_ch
|
||||
session.metadata["_dream_trigger_chat_id"] = trigger_ci
|
||||
if not sustained_goal_active(session.metadata):
|
||||
session.metadata[GOAL_STATE_KEY] = {
|
||||
"status": "active",
|
||||
"objective": "Dream: consolidate unprocessed memory backlog into MEMORY.md, SOUL.md, USER.md",
|
||||
"started_at": datetime.now().isoformat(),
|
||||
}
|
||||
self.sessions.save(session)
|
||||
await self._process_dream_batch(session, msg)
|
||||
await self._dream_finalize_commit(session)
|
||||
return None
|
||||
key = msg.session_key_override or f"{channel}:{chat_id}"
|
||||
session = self.sessions.get_or_create(key)
|
||||
if self._restore_runtime_checkpoint(session):
|
||||
@ -1082,6 +1135,204 @@ class AgentLoop:
|
||||
metadata=outbound_metadata,
|
||||
)
|
||||
|
||||
async def _process_dream_batch(self, session: Session, msg: InboundMessage) -> None:
|
||||
"""Process the full Dream backlog in batches within a single invocation."""
|
||||
from nanobot.agent.skills import BUILTIN_SKILLS_DIR
|
||||
|
||||
# System prompt caching with mtime invalidation
|
||||
template_path = _TEMPLATES_ROOT / "agent" / "dream.md"
|
||||
cached_prompt = session.metadata.get("_dream_system_prompt")
|
||||
cached_mtime = session.metadata.get("_dream_system_prompt_mtime")
|
||||
current_mtime = template_path.stat().st_mtime if template_path.exists() else None
|
||||
|
||||
if cached_prompt is None or cached_mtime != current_mtime:
|
||||
skill_creator_path = BUILTIN_SKILLS_DIR / "skill-creator" / "SKILL.md"
|
||||
workspace = self.dream.store.workspace
|
||||
cached_prompt = render_template(
|
||||
"agent/dream.md",
|
||||
strip=True,
|
||||
skill_creator_path=str(skill_creator_path),
|
||||
soul_path=str(workspace / "SOUL.md"),
|
||||
user_path=str(workspace / "USER.md"),
|
||||
memory_path=str(workspace / "memory" / "MEMORY.md"),
|
||||
stale_threshold_days=_STALE_THRESHOLD_DAYS,
|
||||
)
|
||||
session.metadata["_dream_system_prompt"] = cached_prompt
|
||||
session.metadata["_dream_system_prompt_mtime"] = current_mtime
|
||||
|
||||
while True:
|
||||
last_cursor = self.dream.store.get_last_dream_cursor()
|
||||
entries = self.dream.store.read_unprocessed_history(since_cursor=last_cursor)
|
||||
if not entries:
|
||||
return
|
||||
|
||||
batch = entries[: self.dream.max_batch_size]
|
||||
logger.info(
|
||||
"Dream: processing {}/{} entries (cursor {}→{})",
|
||||
len(batch), len(entries), last_cursor, batch[-1]["cursor"],
|
||||
)
|
||||
|
||||
# Build history text — cap each entry and strip [skip] lines
|
||||
history_text = "\n".join(
|
||||
f"[{e['timestamp']}] "
|
||||
f"{truncate_text_fn(_strip_skip_lines(e['content']), self.dream._HISTORY_ENTRY_PREVIEW_MAX_CHARS)}"
|
||||
for e in batch
|
||||
)
|
||||
|
||||
# Current file contents + per-line age annotations
|
||||
current_date = datetime.now().strftime("%Y-%m-%d")
|
||||
annotate = self.dream.annotate_line_ages
|
||||
raw_memory = self.dream.store.read_memory() or "(empty)"
|
||||
raw_soul = self.dream.store.read_soul() or "(empty)"
|
||||
raw_user = self.dream.store.read_user() or "(empty)"
|
||||
annotated_memory = (
|
||||
self.dream._annotate_with_ages(raw_memory, "memory/MEMORY.md")
|
||||
if annotate else raw_memory
|
||||
)
|
||||
annotated_soul = (
|
||||
self.dream._annotate_with_ages(raw_soul, "SOUL.md")
|
||||
if annotate else raw_soul
|
||||
)
|
||||
annotated_user = (
|
||||
self.dream._annotate_with_ages(raw_user, "USER.md")
|
||||
if annotate else raw_user
|
||||
)
|
||||
current_memory = truncate_text_fn(annotated_memory, self.dream._MEMORY_FILE_MAX_CHARS)
|
||||
current_soul = truncate_text_fn(annotated_soul, self.dream._SOUL_FILE_MAX_CHARS)
|
||||
current_user = truncate_text_fn(annotated_user, self.dream._USER_FILE_MAX_CHARS)
|
||||
|
||||
file_context = (
|
||||
f"## Current Date\n{current_date}\n\n"
|
||||
f"## Current MEMORY.md ({len(current_memory)} chars)\n{current_memory}\n\n"
|
||||
f"## Current SOUL.md ({len(current_soul)} chars)\n{current_soul}\n\n"
|
||||
f"## Current USER.md ({len(current_user)} chars)\n{current_user}"
|
||||
)
|
||||
|
||||
existing_skills = self.dream._list_existing_skills()
|
||||
skills_section = ""
|
||||
if existing_skills:
|
||||
skills_section = (
|
||||
"\n\n## Existing Skills\n"
|
||||
+ "\n".join(f"- {s}" for s in existing_skills)
|
||||
)
|
||||
|
||||
user_prompt = f"## Conversation History\n{history_text}\n\n{file_context}{skills_section}"
|
||||
logger.info("Dream prompt: {} chars, ~{} tokens", len(user_prompt), _estimate_tokens(user_prompt))
|
||||
|
||||
messages: list[dict[str, Any]] = [
|
||||
{"role": "system", "content": cached_prompt},
|
||||
{"role": "user", "content": user_prompt},
|
||||
]
|
||||
|
||||
t_start = time.perf_counter()
|
||||
try:
|
||||
result = await self.dream._runner.run(AgentRunSpec(
|
||||
initial_messages=messages,
|
||||
tools=self.dream._tools,
|
||||
model=self.dream.model,
|
||||
max_iterations=self.dream.max_iterations,
|
||||
max_tool_result_chars=self.dream.max_tool_result_chars,
|
||||
context_window_tokens=self.context_window_tokens,
|
||||
fail_on_tool_error=False,
|
||||
))
|
||||
elapsed = time.perf_counter() - t_start
|
||||
logger.info(
|
||||
"Dream run complete in {:.1f}s: stop_reason={}, tool_events={}",
|
||||
elapsed, result.stop_reason, len(result.tool_events),
|
||||
)
|
||||
except Exception:
|
||||
elapsed = time.perf_counter() - t_start
|
||||
logger.exception("Dream run failed after {:.1f}s", elapsed)
|
||||
result = None
|
||||
|
||||
# Build changelog from tool events
|
||||
changelog: list[str] = []
|
||||
if result and result.tool_events:
|
||||
for event in result.tool_events:
|
||||
if event.get("status") == "ok":
|
||||
changelog.append(f"{event['name']}: {event['detail']}")
|
||||
|
||||
success = result is not None and result.stop_reason == "completed"
|
||||
if success:
|
||||
new_cursor = batch[-1]["cursor"]
|
||||
self.dream.store.set_last_dream_cursor(new_cursor)
|
||||
session.metadata.setdefault("_dream_changelog", []).extend(changelog)
|
||||
self.sessions.save(session)
|
||||
logger.info(
|
||||
"Dream done: {} change(s), cursor advanced to {}",
|
||||
len(changelog), new_cursor,
|
||||
)
|
||||
else:
|
||||
reason = result.stop_reason if result else "exception"
|
||||
logger.warning(
|
||||
"Dream incomplete ({}): cursor NOT advanced, stopping",
|
||||
reason,
|
||||
)
|
||||
return
|
||||
|
||||
self.dream.store.compact_history()
|
||||
|
||||
# Persist session record for debugging / visualization
|
||||
record = {
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"batch": {
|
||||
"from_cursor": last_cursor,
|
||||
"to_cursor": batch[-1]["cursor"],
|
||||
"count": len(batch),
|
||||
},
|
||||
"prompt_chars": len(user_prompt),
|
||||
"elapsed_seconds": elapsed,
|
||||
"stop_reason": result.stop_reason,
|
||||
"usage": result.usage,
|
||||
"tool_events": result.tool_events,
|
||||
"changelog": changelog,
|
||||
"commit_sha": None,
|
||||
"messages": result.messages,
|
||||
}
|
||||
self.dream.store.write_dream_session(record)
|
||||
session.metadata["_dream_last_record"] = record
|
||||
|
||||
|
||||
async def _dream_finalize_commit(self, session: Session) -> None:
|
||||
"""Collapse accumulated changelog into a single git commit, clear caches, and complete the goal."""
|
||||
changelog = session.metadata.pop("_dream_changelog", [])
|
||||
sha = None
|
||||
if changelog and self.dream.store.git.is_initialized():
|
||||
ts = datetime.now().strftime("%Y-%m-%d %H:%M")
|
||||
summary = f"dream: {ts}, {len(changelog)} change(s)"
|
||||
commit_msg = f"{summary}\n\n" + "\n".join(changelog)
|
||||
sha = self.dream.store.git.auto_commit(commit_msg)
|
||||
if sha:
|
||||
logger.info("Dream commit: {}", sha)
|
||||
record = session.metadata.pop("_dream_last_record", None)
|
||||
if record and sha:
|
||||
record["commit_sha"] = sha
|
||||
self.dream.store.write_dream_session(record)
|
||||
session.metadata.pop("_dream_system_prompt", None)
|
||||
session.metadata.pop("_dream_system_prompt_mtime", None)
|
||||
trigger_channel = session.metadata.pop("_dream_trigger_channel", None)
|
||||
trigger_chat_id = session.metadata.pop("_dream_trigger_chat_id", None)
|
||||
goal = session.metadata.get(GOAL_STATE_KEY)
|
||||
if isinstance(goal, dict) and goal.get("status") == "active":
|
||||
session.metadata[GOAL_STATE_KEY] = {
|
||||
**goal,
|
||||
"status": "completed",
|
||||
"completed_at": datetime.now().isoformat(),
|
||||
"recap": f"Memory backlog consolidated ({len(changelog)} change(s)).",
|
||||
}
|
||||
self.sessions.save(session)
|
||||
session.metadata["_dream_finalized"] = True
|
||||
# Notify the user who triggered /dream
|
||||
if trigger_channel and trigger_chat_id:
|
||||
content = f"Dream completed: {len(changelog)} change(s) committed."
|
||||
if not changelog:
|
||||
content = "Dream: nothing to process."
|
||||
await self.bus.publish_outbound(OutboundMessage(
|
||||
channel=trigger_channel,
|
||||
chat_id=trigger_chat_id,
|
||||
content=content,
|
||||
))
|
||||
|
||||
async def _process_message(
|
||||
self,
|
||||
msg: InboundMessage,
|
||||
|
||||
@ -6,6 +6,7 @@ import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import weakref
|
||||
from contextlib import suppress
|
||||
from datetime import datetime
|
||||
@ -15,7 +16,7 @@ from typing import TYPE_CHECKING, Any, Callable, Iterator
|
||||
import tiktoken
|
||||
from loguru import logger
|
||||
|
||||
from nanobot.agent.runner import AgentRunner, AgentRunSpec
|
||||
from nanobot.agent.runner import AgentRunner
|
||||
from nanobot.agent.tools.registry import ToolRegistry
|
||||
from nanobot.session.manager import Session
|
||||
from nanobot.utils.gitstore import GitStore
|
||||
@ -33,6 +34,20 @@ if TYPE_CHECKING:
|
||||
from nanobot.providers.base import LLMProvider
|
||||
from nanobot.session.manager import SessionManager
|
||||
|
||||
# Cache the tiktoken encoding to avoid repeated instantiation on every
|
||||
# truncate/encode call. Encoding objects are thread-safe and reusable.
|
||||
try:
|
||||
_TIKTOKEN_ENC = tiktoken.get_encoding("cl100k_base")
|
||||
except Exception: # pragma: no cover
|
||||
_TIKTOKEN_ENC = None
|
||||
|
||||
|
||||
def _estimate_tokens(text: str) -> int:
|
||||
"""Approximate token count for a text string."""
|
||||
if _TIKTOKEN_ENC is not None:
|
||||
return len(_TIKTOKEN_ENC.encode(text))
|
||||
return len(text) // 4
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MemoryStore — pure file I/O layer
|
||||
@ -400,6 +415,26 @@ class MemoryStore:
|
||||
def set_last_dream_cursor(self, cursor: int) -> None:
|
||||
self._dream_cursor_file.write_text(str(cursor), encoding="utf-8")
|
||||
|
||||
def write_dream_session(self, data: dict[str, Any]) -> None:
|
||||
"""Atomic overwrite of the latest Dream run record."""
|
||||
path = self.memory_dir / ".dream_session.json"
|
||||
tmp_path = path.with_suffix(".tmp")
|
||||
try:
|
||||
with open(tmp_path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp_path, path)
|
||||
with suppress(PermissionError):
|
||||
fd = os.open(str(path.parent), os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
except BaseException:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
# -- message formatting utility ------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
@ -618,19 +653,21 @@ class Consolidator:
|
||||
"""Available input token budget for consolidation LLM."""
|
||||
return self.context_window_tokens - self.max_completion_tokens - self._SAFETY_BUFFER
|
||||
|
||||
def _truncate_to_token_budget(self, text: str) -> str:
|
||||
"""Truncate text so it fits within the consolidation LLM's token budget."""
|
||||
budget = self._input_token_budget
|
||||
def _truncate_to_token_budget(self, text: str, reserve_tokens: int = 0) -> str:
|
||||
"""Truncate text so it fits within the consolidation LLM's token budget.
|
||||
|
||||
reserve_tokens: additional tokens to reserve for dedup context or other
|
||||
overhead that will be appended after truncation.
|
||||
"""
|
||||
budget = self._input_token_budget - reserve_tokens
|
||||
if budget <= 0:
|
||||
return truncate_text(text, _RAW_ARCHIVE_MAX_CHARS)
|
||||
try:
|
||||
enc = tiktoken.get_encoding("cl100k_base")
|
||||
tokens = enc.encode(text)
|
||||
if _TIKTOKEN_ENC is not None:
|
||||
tokens = _TIKTOKEN_ENC.encode(text)
|
||||
if len(tokens) <= budget:
|
||||
return text
|
||||
return enc.decode(tokens[:budget]) + "\n... (truncated)"
|
||||
except Exception:
|
||||
return truncate_text(text, budget * 4)
|
||||
return _TIKTOKEN_ENC.decode(tokens[:budget]) + "\n... (truncated)"
|
||||
return truncate_text(text, budget * 4)
|
||||
|
||||
async def archive(self, messages: list[dict]) -> str | None:
|
||||
"""Summarize messages via LLM and append to history.jsonl.
|
||||
@ -639,9 +676,53 @@ class Consolidator:
|
||||
"""
|
||||
if not messages:
|
||||
return None
|
||||
t_start = time.perf_counter()
|
||||
try:
|
||||
formatted = MemoryStore._format_messages(messages)
|
||||
formatted = self._truncate_to_token_budget(formatted)
|
||||
logger.debug(
|
||||
"Consolidator: {} messages, formatted={} chars",
|
||||
len(messages), len(formatted),
|
||||
)
|
||||
|
||||
# Inject current memory context for dedup-aware summarization.
|
||||
memory_preview = self.store.read_memory()[:4000]
|
||||
user_preview = self.store.read_user()[:2000]
|
||||
dedup_context = ""
|
||||
if memory_preview:
|
||||
dedup_context += f"\n\n## Current MEMORY.md (for dedup)\n{memory_preview}"
|
||||
if user_preview:
|
||||
dedup_context += f"\n\n## Current USER.md (for dedup)\n{user_preview}"
|
||||
|
||||
reserve_tokens = 0
|
||||
if dedup_context:
|
||||
if _TIKTOKEN_ENC is not None:
|
||||
reserve_tokens = len(_TIKTOKEN_ENC.encode(dedup_context)) + 100
|
||||
else:
|
||||
reserve_tokens = len(dedup_context) // 4 + 100
|
||||
|
||||
if self._input_token_budget <= reserve_tokens:
|
||||
logger.warning(
|
||||
"Consolidator: dedup_context ({} tokens) exceeds budget ({}), dropping it",
|
||||
reserve_tokens, self._input_token_budget,
|
||||
)
|
||||
dedup_context = ""
|
||||
reserve_tokens = 0
|
||||
else:
|
||||
logger.debug(
|
||||
"Consolidator: dedup_context={} chars, reserve_tokens={}",
|
||||
len(dedup_context), reserve_tokens,
|
||||
)
|
||||
|
||||
formatted_before = len(formatted)
|
||||
formatted = self._truncate_to_token_budget(
|
||||
formatted, reserve_tokens=reserve_tokens
|
||||
)
|
||||
if len(formatted) < formatted_before:
|
||||
logger.warning(
|
||||
"Consolidator: truncated formatted messages from {} to {} chars",
|
||||
formatted_before, len(formatted),
|
||||
)
|
||||
|
||||
response = await self.provider.chat_with_retry(
|
||||
model=self.model,
|
||||
messages=[
|
||||
@ -652,18 +733,31 @@ class Consolidator:
|
||||
strip=True,
|
||||
),
|
||||
},
|
||||
{"role": "user", "content": formatted},
|
||||
{"role": "user", "content": formatted + dedup_context},
|
||||
],
|
||||
tools=None,
|
||||
tool_choice=None,
|
||||
)
|
||||
elapsed = time.perf_counter() - t_start
|
||||
if response.finish_reason == "error":
|
||||
logger.warning(
|
||||
"Consolidator LLM error after {:.1f}s: {}",
|
||||
elapsed, response.content,
|
||||
)
|
||||
raise RuntimeError(f"LLM returned error: {response.content}")
|
||||
summary = response.content or "[no summary]"
|
||||
logger.info(
|
||||
"Consolidator: {} entries -> {} chars summary in {:.1f}s",
|
||||
len(messages), len(summary), elapsed,
|
||||
)
|
||||
self.store.append_history(summary, max_chars=_ARCHIVE_SUMMARY_MAX_CHARS)
|
||||
return summary
|
||||
except Exception:
|
||||
logger.warning("Consolidation LLM call failed, raw-dumping to history")
|
||||
elapsed = time.perf_counter() - t_start
|
||||
logger.warning(
|
||||
"Consolidation LLM call failed after {:.1f}s, raw-dumping to history",
|
||||
elapsed,
|
||||
)
|
||||
self.store.raw_archive(messages)
|
||||
return None
|
||||
|
||||
@ -851,35 +945,44 @@ class Consolidator:
|
||||
|
||||
|
||||
# Single source of truth for the staleness threshold used in _annotate_with_ages
|
||||
# *and* in the Phase 1 prompt template (passed as `stale_threshold_days`).
|
||||
# *and* in the system prompt template (passed as `stale_threshold_days`).
|
||||
# Keep code and prompt aligned — if you bump this, the LLM's instruction string
|
||||
# updates automatically.
|
||||
_STALE_THRESHOLD_DAYS = 14
|
||||
|
||||
_SKIP_LINE_RE = re.compile(r"^\s*-\s*\[skip\]\s*.*$", re.MULTILINE | re.IGNORECASE)
|
||||
|
||||
|
||||
def _strip_skip_lines(text: str) -> str:
|
||||
"""Remove lines marked [skip] from history content."""
|
||||
lines = text.splitlines()
|
||||
kept = [line for line in lines if not _SKIP_LINE_RE.match(line)]
|
||||
return "\n".join(kept)
|
||||
|
||||
|
||||
class Dream:
|
||||
"""Two-phase memory processor: analyze history.jsonl, then edit files via AgentRunner.
|
||||
"""Single-phase memory processor: analyze history.jsonl and edit files via AgentRunner.
|
||||
|
||||
Phase 1 produces an analysis summary (plain LLM call).
|
||||
Phase 2 delegates to AgentRunner with read_file / edit_file tools so the
|
||||
LLM can make targeted, incremental edits instead of replacing entire files.
|
||||
Delegates to AgentRunner with read_file / edit_file tools so the LLM can
|
||||
analyze conversation history, extract facts, deduplicate, and make targeted
|
||||
incremental edits — all in a single agent run.
|
||||
"""
|
||||
|
||||
# Caps on prompt-bound inputs so Dream's LLM calls never exceed the model's
|
||||
# context window just because a file (or a legacy large history entry) grew
|
||||
# unexpectedly. Each file still appears in full via read_file when the agent
|
||||
# needs it in Phase 2 — these caps only bound the Phase 1/2 prompt preview.
|
||||
_MEMORY_FILE_MAX_CHARS = 32_000
|
||||
_SOUL_FILE_MAX_CHARS = 16_000
|
||||
_USER_FILE_MAX_CHARS = 16_000
|
||||
_HISTORY_ENTRY_PREVIEW_MAX_CHARS = 4_000
|
||||
# needs it — these caps only bound the prompt preview.
|
||||
_MEMORY_FILE_MAX_CHARS = 16_000
|
||||
_SOUL_FILE_MAX_CHARS = 4_000
|
||||
_USER_FILE_MAX_CHARS = 4_000
|
||||
_HISTORY_ENTRY_PREVIEW_MAX_CHARS = 2_000
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store: MemoryStore,
|
||||
provider: LLMProvider,
|
||||
model: str,
|
||||
max_batch_size: int = 20,
|
||||
max_batch_size: int = 5,
|
||||
max_iterations: int = 10,
|
||||
max_tool_result_chars: int = 16_000,
|
||||
annotate_line_ages: bool = True,
|
||||
@ -890,9 +993,9 @@ class Dream:
|
||||
self.max_batch_size = max_batch_size
|
||||
self.max_iterations = max_iterations
|
||||
self.max_tool_result_chars = max_tool_result_chars
|
||||
# Kill switch for the git-blame-based per-line age annotation in Phase 1.
|
||||
# Default True keeps the #3212 behavior; set False to feed MEMORY.md raw
|
||||
# (e.g. if a specific LLM reacts poorly to the `← Nd` suffix).
|
||||
# Kill switch for the git-blame-based per-line age annotation in the prompt.
|
||||
# Default True keeps the #3212 behavior; set False to feed all memory
|
||||
# files raw (e.g. if a specific LLM reacts poorly to the `← Nd` suffix).
|
||||
self.annotate_line_ages = annotate_line_ages
|
||||
self._runner = AgentRunner(provider)
|
||||
self._tools = self._build_tools()
|
||||
@ -907,6 +1010,7 @@ class Dream:
|
||||
def _build_tools(self) -> ToolRegistry:
|
||||
"""Build a minimal tool registry for the Dream agent."""
|
||||
from nanobot.agent.skills import BUILTIN_SKILLS_DIR
|
||||
from nanobot.agent.tools.apply_patch import ApplyPatchTool
|
||||
from nanobot.agent.tools.file_state import FileStates
|
||||
from nanobot.agent.tools.filesystem import EditFileTool, ReadFileTool, WriteFileTool
|
||||
|
||||
@ -924,6 +1028,7 @@ class Dream:
|
||||
file_states=file_states,
|
||||
))
|
||||
tools.register(EditFileTool(workspace=workspace, allowed_dir=workspace, file_states=file_states))
|
||||
tools.register(ApplyPatchTool(workspace=workspace, allowed_dir=workspace, file_states=file_states))
|
||||
# write_file resolves relative paths from workspace root, but can only
|
||||
# write under skills/ so the prompt can safely use skills/<name>/SKILL.md.
|
||||
skills_dir = workspace / "skills"
|
||||
@ -961,8 +1066,8 @@ class Dream:
|
||||
|
||||
# -- main entry ----------------------------------------------------------
|
||||
|
||||
def _annotate_with_ages(self, content: str) -> str:
|
||||
"""Append per-line age suffixes to MEMORY.md content.
|
||||
def _annotate_with_ages(self, content: str, file_path: str = "memory/MEMORY.md") -> str:
|
||||
"""Append per-line age suffixes to file content.
|
||||
|
||||
Each non-blank line whose age exceeds ``_STALE_THRESHOLD_DAYS`` gets a
|
||||
suffix like ``← 30d`` indicating days since last modification.
|
||||
@ -970,9 +1075,7 @@ class Dream:
|
||||
annotate fails, or the line count doesn't match the age count
|
||||
(which can happen with an uncommitted working-tree edit — better to
|
||||
skip annotation than to tag the wrong line).
|
||||
SOUL.md and USER.md are never annotated.
|
||||
"""
|
||||
file_path = "memory/MEMORY.md"
|
||||
try:
|
||||
ages = self.store.git.line_ages(file_path)
|
||||
except Exception:
|
||||
@ -1007,156 +1110,3 @@ class Dream:
|
||||
result += "\n"
|
||||
return result
|
||||
|
||||
async def run(self) -> bool:
|
||||
"""Process unprocessed history entries. Returns True if work was done."""
|
||||
from nanobot.agent.skills import BUILTIN_SKILLS_DIR
|
||||
|
||||
last_cursor = self.store.get_last_dream_cursor()
|
||||
entries = self.store.read_unprocessed_history(since_cursor=last_cursor)
|
||||
if not entries:
|
||||
return False
|
||||
|
||||
batch = entries[: self.max_batch_size]
|
||||
logger.info(
|
||||
"Dream: processing {} entries (cursor {}→{}), batch={}",
|
||||
len(entries), last_cursor, batch[-1]["cursor"], len(batch),
|
||||
)
|
||||
|
||||
# Build history text for LLM — cap each entry so a legacy oversized
|
||||
# record (e.g. pre-#3412 raw_archive dump) can't blow up the prompt.
|
||||
history_text = "\n".join(
|
||||
f"[{e['timestamp']}] "
|
||||
f"{truncate_text(e['content'], self._HISTORY_ENTRY_PREVIEW_MAX_CHARS)}"
|
||||
for e in batch
|
||||
)
|
||||
|
||||
# Current file contents + per-line age annotations (MEMORY.md only).
|
||||
# Each file is capped in the *prompt preview* only; Phase 2 still sees
|
||||
# the full file via the read_file tool.
|
||||
current_date = datetime.now().strftime("%Y-%m-%d")
|
||||
raw_memory = self.store.read_memory() or "(empty)"
|
||||
annotated_memory = (
|
||||
self._annotate_with_ages(raw_memory)
|
||||
if self.annotate_line_ages
|
||||
else raw_memory
|
||||
)
|
||||
current_memory = truncate_text(annotated_memory, self._MEMORY_FILE_MAX_CHARS)
|
||||
current_soul = truncate_text(
|
||||
self.store.read_soul() or "(empty)", self._SOUL_FILE_MAX_CHARS,
|
||||
)
|
||||
current_user = truncate_text(
|
||||
self.store.read_user() or "(empty)", self._USER_FILE_MAX_CHARS,
|
||||
)
|
||||
|
||||
file_context = (
|
||||
f"## Current Date\n{current_date}\n\n"
|
||||
f"## Current MEMORY.md ({len(current_memory)} chars)\n{current_memory}\n\n"
|
||||
f"## Current SOUL.md ({len(current_soul)} chars)\n{current_soul}\n\n"
|
||||
f"## Current USER.md ({len(current_user)} chars)\n{current_user}"
|
||||
)
|
||||
|
||||
# Phase 1: Analyze (no skills list — dedup is Phase 2's job)
|
||||
phase1_prompt = (
|
||||
f"## Conversation History\n{history_text}\n\n{file_context}"
|
||||
)
|
||||
|
||||
try:
|
||||
phase1_response = await self.provider.chat_with_retry(
|
||||
model=self.model,
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": render_template(
|
||||
"agent/dream_phase1.md",
|
||||
strip=True,
|
||||
stale_threshold_days=_STALE_THRESHOLD_DAYS,
|
||||
),
|
||||
},
|
||||
{"role": "user", "content": phase1_prompt},
|
||||
],
|
||||
tools=None,
|
||||
tool_choice=None,
|
||||
)
|
||||
analysis = phase1_response.content or ""
|
||||
logger.debug("Dream Phase 1 analysis ({} chars): {}", len(analysis), analysis[:500])
|
||||
except Exception:
|
||||
logger.exception("Dream Phase 1 failed")
|
||||
return False
|
||||
|
||||
# Phase 2: Delegate to AgentRunner with read_file / edit_file
|
||||
existing_skills = self._list_existing_skills()
|
||||
skills_section = ""
|
||||
if existing_skills:
|
||||
skills_section = (
|
||||
"\n\n## Existing Skills\n"
|
||||
+ "\n".join(f"- {s}" for s in existing_skills)
|
||||
)
|
||||
phase2_prompt = f"## Analysis Result\n{analysis}\n\n{file_context}{skills_section}"
|
||||
|
||||
tools = self._tools
|
||||
skill_creator_path = BUILTIN_SKILLS_DIR / "skill-creator" / "SKILL.md"
|
||||
messages: list[dict[str, Any]] = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": render_template(
|
||||
"agent/dream_phase2.md",
|
||||
strip=True,
|
||||
skill_creator_path=str(skill_creator_path),
|
||||
),
|
||||
},
|
||||
{"role": "user", "content": phase2_prompt},
|
||||
]
|
||||
|
||||
try:
|
||||
result = await self._runner.run(AgentRunSpec(
|
||||
initial_messages=messages,
|
||||
tools=tools,
|
||||
model=self.model,
|
||||
max_iterations=self.max_iterations,
|
||||
max_tool_result_chars=self.max_tool_result_chars,
|
||||
fail_on_tool_error=False,
|
||||
))
|
||||
logger.debug(
|
||||
"Dream Phase 2 complete: stop_reason={}, tool_events={}",
|
||||
result.stop_reason, len(result.tool_events),
|
||||
)
|
||||
for ev in (result.tool_events or []):
|
||||
logger.info("Dream tool_event: name={}, status={}, detail={}", ev.get("name"), ev.get("status"), ev.get("detail", "")[:200])
|
||||
except Exception:
|
||||
logger.exception("Dream Phase 2 failed")
|
||||
result = None
|
||||
|
||||
# Build changelog from tool events
|
||||
changelog: list[str] = []
|
||||
if result and result.tool_events:
|
||||
for event in result.tool_events:
|
||||
if event["status"] == "ok":
|
||||
changelog.append(f"{event['name']}: {event['detail']}")
|
||||
|
||||
# Only advance cursor on successful completion to prevent silent loss
|
||||
if result and result.stop_reason == "completed":
|
||||
new_cursor = batch[-1]["cursor"]
|
||||
self.store.set_last_dream_cursor(new_cursor)
|
||||
logger.info(
|
||||
"Dream done: {} change(s), cursor advanced to {}",
|
||||
len(changelog), new_cursor,
|
||||
)
|
||||
else:
|
||||
reason = result.stop_reason if result else "exception"
|
||||
logger.warning(
|
||||
"Dream incomplete ({}): cursor NOT advanced, will retry next cron cycle",
|
||||
reason,
|
||||
)
|
||||
|
||||
self.store.compact_history()
|
||||
|
||||
# Git auto-commit (only when there are actual changes)
|
||||
if changelog and self.store.git.is_initialized():
|
||||
ts = batch[-1]["timestamp"]
|
||||
summary = f"dream: {ts}, {len(changelog)} change(s)"
|
||||
commit_msg = f"{summary}\n\n{analysis.strip()}"
|
||||
sha = self.store.git.auto_commit(commit_msg)
|
||||
if sha:
|
||||
logger.info("Dream commit: {}", sha)
|
||||
|
||||
return True
|
||||
|
||||
@ -762,7 +762,7 @@ def _run_gateway(
|
||||
)
|
||||
|
||||
from nanobot.agent.loop import UNIFIED_SESSION_KEY
|
||||
from nanobot.bus.events import OutboundMessage
|
||||
from nanobot.bus.events import InboundMessage, OutboundMessage
|
||||
|
||||
def _channel_session_key(channel: str, chat_id: str) -> str:
|
||||
return (
|
||||
@ -810,13 +810,13 @@ def _run_gateway(
|
||||
# Set cron callback (needs agent)
|
||||
async def on_cron_job(job: CronJob) -> str | None:
|
||||
"""Execute a cron job through the agent."""
|
||||
# Dream is an internal job — run directly, not through the agent loop.
|
||||
if job.name == "dream":
|
||||
try:
|
||||
await agent.dream.run()
|
||||
logger.info("Dream cron job completed")
|
||||
except Exception:
|
||||
logger.exception("Dream cron job failed")
|
||||
await bus.publish_inbound(InboundMessage(
|
||||
channel="system",
|
||||
sender_id="dream",
|
||||
chat_id="dream",
|
||||
content="",
|
||||
))
|
||||
return None
|
||||
|
||||
from nanobot.utils.evaluator import evaluate_response
|
||||
@ -1027,8 +1027,6 @@ def _run_gateway(
|
||||
await server.serve_forever()
|
||||
# Register Dream system job (always-on, idempotent on restart)
|
||||
dream_cfg = config.agents.defaults.dream
|
||||
if dream_cfg.model_override:
|
||||
agent.dream.model = dream_cfg.model_override
|
||||
agent.dream.max_batch_size = dream_cfg.max_batch_size
|
||||
agent.dream.max_iterations = dream_cfg.max_iterations
|
||||
agent.dream.annotate_line_ages = dream_cfg.annotate_line_ages
|
||||
|
||||
@ -299,30 +299,22 @@ async def cmd_model(ctx: CommandContext) -> OutboundMessage:
|
||||
|
||||
async def cmd_dream(ctx: CommandContext) -> OutboundMessage:
|
||||
"""Manually trigger a Dream consolidation run."""
|
||||
import time
|
||||
from nanobot.bus.events import InboundMessage
|
||||
|
||||
loop = ctx.loop
|
||||
msg = ctx.msg
|
||||
|
||||
async def _run_dream():
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
did_work = await loop.dream.run()
|
||||
elapsed = time.monotonic() - t0
|
||||
if did_work:
|
||||
content = f"Dream completed in {elapsed:.1f}s."
|
||||
else:
|
||||
content = "Dream: nothing to process."
|
||||
except Exception as e:
|
||||
elapsed = time.monotonic() - t0
|
||||
content = f"Dream failed after {elapsed:.1f}s: {e}"
|
||||
await loop.bus.publish_outbound(OutboundMessage(
|
||||
channel=msg.channel, chat_id=msg.chat_id, content=content,
|
||||
))
|
||||
|
||||
asyncio.create_task(_run_dream())
|
||||
await ctx.loop.bus.publish_inbound(InboundMessage(
|
||||
channel="system",
|
||||
sender_id="dream",
|
||||
chat_id="dream",
|
||||
content="",
|
||||
metadata={
|
||||
"trigger_channel": ctx.msg.channel,
|
||||
"trigger_chat_id": ctx.msg.chat_id,
|
||||
},
|
||||
))
|
||||
return OutboundMessage(
|
||||
channel=msg.channel, chat_id=msg.chat_id, content="Dreaming...",
|
||||
channel=ctx.msg.channel,
|
||||
chat_id=ctx.msg.chat_id,
|
||||
content="Dream started. It will process memory backlog and report when done.",
|
||||
)
|
||||
|
||||
|
||||
@ -355,6 +347,18 @@ def _format_changed_files(diff: str) -> str:
|
||||
|
||||
def _format_dream_log_content(commit, diff: str, *, requested_sha: str | None = None) -> str:
|
||||
files_line = _format_changed_files(diff)
|
||||
msg_lines = commit.message.splitlines() if commit.message else []
|
||||
msg_summary = msg_lines[0] if msg_lines else ""
|
||||
msg_body = []
|
||||
in_body = False
|
||||
for line in msg_lines[1:]:
|
||||
if not in_body:
|
||||
if not line:
|
||||
in_body = True
|
||||
continue
|
||||
msg_body.append(line)
|
||||
body_text = "\n".join(msg_body).strip()
|
||||
|
||||
lines = [
|
||||
"## Dream Update",
|
||||
"",
|
||||
@ -362,8 +366,12 @@ def _format_dream_log_content(commit, diff: str, *, requested_sha: str | None =
|
||||
"",
|
||||
f"- Commit: `{commit.sha}`",
|
||||
f"- Time: {commit.timestamp}",
|
||||
f"- Changed files: {files_line}",
|
||||
]
|
||||
if msg_summary:
|
||||
lines.append(f"- Summary: {msg_summary}")
|
||||
lines.append(f"- Changed files: {files_line}")
|
||||
if body_text:
|
||||
lines.extend(["", "### Analysis", "", body_text])
|
||||
if diff:
|
||||
lines.extend([
|
||||
"",
|
||||
@ -389,7 +397,8 @@ def _format_dream_restore_list(commits: list) -> str:
|
||||
"",
|
||||
]
|
||||
for c in commits:
|
||||
lines.append(f"- `{c.sha}` {c.timestamp} - {c.message.splitlines()[0]}")
|
||||
summary = c.message.splitlines()[0] if c.message else "(no message)"
|
||||
lines.append(f"- `{c.sha}` {c.timestamp} - {summary}")
|
||||
lines.extend([
|
||||
"",
|
||||
"Preview a version with `/dream-log <sha>` before restoring it.",
|
||||
|
||||
@ -52,13 +52,12 @@ class DreamConfig(Base):
|
||||
model_override: str | None = Field(
|
||||
default=None,
|
||||
validation_alias=AliasChoices("modelOverride", "model", "model_override"),
|
||||
) # Optional Dream-specific model override
|
||||
max_batch_size: int = Field(default=20, ge=1) # Max history entries per run
|
||||
# Bumped from 10 to 15 in #3212 (exp002: +30% dedup, no accuracy loss; >15 plateaus).
|
||||
max_iterations: int = Field(default=15, ge=1) # Max tool calls per Phase 2
|
||||
# Per-line git-blame age annotation in Phase 1 prompt (see #3212). Default
|
||||
# on — set to False to feed MEMORY.md raw if a specific LLM reacts poorly
|
||||
# to the `← Nd` suffix or you want deterministic, git-independent prompts.
|
||||
) # Optional Dream-specific model override. Supports preset names (resolved against model_presets) or raw model identifiers.
|
||||
max_batch_size: int = Field(default=5, ge=1) # Max history entries per run
|
||||
max_iterations: int = Field(default=15, ge=1) # Max tool calls per Dream run
|
||||
# Per-line git-blame age annotation in the Dream prompt (see #3212). Default
|
||||
# on — set to False to feed all memory files raw if a specific LLM reacts
|
||||
# poorly to the `← Nd` suffix or you want deterministic, git-independent prompts.
|
||||
annotate_line_ages: bool = True
|
||||
|
||||
def build_schedule(self, timezone: str) -> CronSchedule:
|
||||
|
||||
@ -34,3 +34,5 @@ Examples (replace `keyword`):
|
||||
- **Do NOT edit SOUL.md, USER.md, or MEMORY.md.** They are automatically managed by Dream.
|
||||
- If you notice outdated information, it will be corrected when Dream runs next.
|
||||
- Users can view Dream's activity with the `/dream-log` command.
|
||||
- Dream runs as a `system` session inside the AgentLoop, triggered by the `/dream` command or cron. Each turn processes one batch; if backlog remains, Dream automatically chains additional turns until complete. All changes are committed in a single git commit.
|
||||
- Dream can use a different model than the main agent via `agents.defaults.dream.modelOverride`. Supports preset names or raw model identifiers.
|
||||
|
||||
@ -1,13 +1,27 @@
|
||||
Extract key facts from this conversation. Only output items matching these categories, skip everything else:
|
||||
- User facts: personal info, preferences, stated opinions, habits
|
||||
- Decisions: choices made, conclusions reached
|
||||
- Solutions: working approaches discovered through trial and error, especially non-obvious methods that succeeded after failed attempts
|
||||
- Events: plans, deadlines, notable occurrences
|
||||
- Preferences: communication style, tool preferences
|
||||
Extract key facts from this conversation. For each fact, annotate its memory attributes.
|
||||
|
||||
Priority: user corrections and preferences > solutions > decisions > events > environment facts. The most valuable memory prevents the user from having to repeat themselves.
|
||||
Only SNIP facts deserve a non-[skip] mark:
|
||||
- Signal: would the user need to repeat this if forgotten?
|
||||
- Novel: not already in MEMORY.md or USER.md (check context below)
|
||||
- Important: prevents rework or captures preferences / rules
|
||||
- Persistent: still relevant after 2 weeks
|
||||
|
||||
Skip: code patterns derivable from source, git history, or anything already captured in existing memory.
|
||||
Output one fact per line in this format:
|
||||
- [mark] fact content
|
||||
|
||||
Marks (choose the best match):
|
||||
- [permanent] Core preferences, personal traits, habits — never becomes stale
|
||||
- [durable] Technical discoveries, project knowledge, config details — valid for months
|
||||
- [ephemeral] Active task state, temporary decisions — may change in weeks
|
||||
- [correction] Correction to a previous memory — must state what it replaces
|
||||
- [skip] Does not meet SNIP criteria — still written to history.jsonl for audit, but Dream will ignore it
|
||||
|
||||
Categories to capture: people/roles, decisions/rationale, solutions, events/dates, preferences.
|
||||
Decisions must include their motivation.
|
||||
Write densely. Prefer 'X=A, Y=B' over separate bullets for tightly coupled facts.
|
||||
Priority: user corrections > decisions with rationale > solutions > specific events > general context.
|
||||
Output in the same language as the input conversation.
|
||||
CRITICAL: Never drop person names, team names, or project names.
|
||||
Skip: code patterns derivable from source, git history, or anything already in existing memory.
|
||||
|
||||
Output as concise bullet points, one fact per line. No preamble, no commentary.
|
||||
If nothing noteworthy happened, output: (nothing)
|
||||
|
||||
58
nanobot/templates/agent/dream.md
Normal file
58
nanobot/templates/agent/dream.md
Normal file
@ -0,0 +1,58 @@
|
||||
Update memory files by analyzing conversation history and editing files directly.
|
||||
Prune before adding — removing stale content is as important as adding new facts.
|
||||
|
||||
## File routing
|
||||
Do NOT guess paths. Route each fact to its canonical file:
|
||||
|
||||
| File | Full path | Content |
|
||||
|------|------|---------|
|
||||
| SOUL.md | `{{ soul_path }}` | Agent behavior, guardrails, tone, interaction patterns |
|
||||
| USER.md | `{{ user_path }}` | Personal info, preferences, habits, work context, communication style |
|
||||
| MEMORY.md | `{{ memory_path }}` | Technical knowledge, project context, infrastructure, accounts |
|
||||
| SKILL.md | `skills/<name>/SKILL.md` | Reusable workflow templates ([SKILL] entries only) |
|
||||
|
||||
Cross-boundary rule: no technical configs in USER.md, no user facts in SOUL.md, no preferences in MEMORY.md. If a fact fits multiple files, keep the most specific copy and remove the rest.
|
||||
|
||||
## Delete-or-keep
|
||||
|
||||
**Always delete:**
|
||||
- Same fact at multiple locations — keep canonical copy only
|
||||
- Merged/closed PR notes, resolved incidents, superseded info
|
||||
- Verbose entries restatable in fewer words
|
||||
- Overlapping or nested sections covering the same topic
|
||||
|
||||
**Likely delete** (apply judgment):
|
||||
- Same fact at different detail levels — keep most complete version only
|
||||
- Debugging steps unlikely to recur
|
||||
- Ephemeral facts past their useful life
|
||||
- Tool/service details documented upstream
|
||||
- Lines with ``← Nd`` where N>{{ stale_threshold_days }} — closer review, not automatic removal
|
||||
|
||||
**Never delete:**
|
||||
- User preferences and personality traits (permanent regardless of age)
|
||||
- Active project context still referenced in conversations
|
||||
- Behavioral rules in SOUL.md
|
||||
|
||||
When removing: prefer deleting individual items over entire sections.
|
||||
|
||||
## Fact extraction
|
||||
- Atomic facts: "has a cat named Luna" not "discussed pet care"
|
||||
- Corrections: edit the existing entry, don't append a new one
|
||||
- Capture confirmed approaches the user validated
|
||||
|
||||
## Skill discovery & creation
|
||||
Flag [SKILL] only when ALL are true: repeatable workflow appeared 2+ times, involves clear steps (not vague preferences), substantial enough for its own instruction set. Check existing skills to avoid redundancy.
|
||||
|
||||
For [SKILL] entries:
|
||||
- Use write_file to create skills/<name>/SKILL.md; read_file `{{ skill_creator_path }}` for format reference
|
||||
- YAML frontmatter (name, description), under 2000 words: when to use, steps, output format, example
|
||||
- Do NOT overwrite existing skills — if overlapping, merge delta into the existing skill
|
||||
- Skills are instruction sets, not code. Keep concrete values in MEMORY.md; skills use placeholders
|
||||
|
||||
## Editing
|
||||
- Default tool: apply_patch. Use edit_file only for small exact replacements.
|
||||
- File contents provided below — no read_file needed for initial edits.
|
||||
- Batch all changes into a single apply_patch call. Surgical edits only.
|
||||
- dry_run=true to preview. If nothing to update, stop without calling tools.
|
||||
|
||||
Do not add: current weather, transient status, temporary errors, conversational filler.
|
||||
@ -1,40 +0,0 @@
|
||||
You have TWO equally important tasks:
|
||||
1. Extract new facts from conversation history
|
||||
2. Deduplicate existing memory files — find and flag redundant, overlapping, or stale content even if NOT mentioned in history
|
||||
|
||||
Output one line per finding:
|
||||
[FILE] atomic fact (not already in memory)
|
||||
[FILE-REMOVE] reason for removal
|
||||
[SKILL] kebab-case-name: one-line description of the reusable pattern
|
||||
|
||||
Files: USER (identity, preferences), SOUL (bot behavior, tone), MEMORY (knowledge, project context)
|
||||
|
||||
Rules:
|
||||
- Atomic facts: "has a cat named Luna" not "discussed pet care"
|
||||
- Corrections: [USER] location is Tokyo, not Osaka
|
||||
- Capture confirmed approaches the user validated
|
||||
|
||||
Deduplication — scan ALL memory files for these redundancy patterns:
|
||||
- Same fact stated in multiple places (e.g., "communicates in Chinese" in both USER.md and multiple MEMORY.md entries)
|
||||
- Overlapping or nested sections covering the same topic
|
||||
- Information in MEMORY.md that is already captured in USER.md or SOUL.md (MEMORY.md should not duplicate permanent-file content)
|
||||
- Verbose entries that can be condensed without losing information
|
||||
For each duplicate found, output [FILE-REMOVE] for the less authoritative copy (prefer keeping facts in their canonical location)
|
||||
|
||||
Staleness — MEMORY.md lines may have a ``← Nd`` suffix showing days since last modification:
|
||||
- SOUL.md and USER.md have no age annotations — they are permanent, only update with corrections
|
||||
- Age only indicates when content was last touched, not whether it should be removed
|
||||
- Use content judgment: user habits/preferences/personality traits are permanent regardless of age
|
||||
- Only prune content that is objectively outdated: passed events, resolved tracking, superseded approaches
|
||||
- Lines with ``← Nd`` (N>{{ stale_threshold_days }}) deserve closer review but are NOT automatically removable
|
||||
- When removing: prefer deleting individual items over entire sections
|
||||
|
||||
Skill discovery — flag [SKILL] when ALL of these are true:
|
||||
- A specific, repeatable workflow appeared 2+ times in the conversation history
|
||||
- It involves clear steps (not vague preferences like "likes concise answers")
|
||||
- It is substantial enough to warrant its own instruction set (not trivial like "read a file")
|
||||
- Do not worry about duplicates — the next phase will check against existing skills
|
||||
|
||||
Do not add: current weather, transient status, temporary errors, conversational filler.
|
||||
|
||||
[SKIP] if nothing needs updating.
|
||||
@ -1,37 +0,0 @@
|
||||
Update memory files based on the analysis below.
|
||||
- [FILE] entries: add the described content to the appropriate file
|
||||
- [FILE-REMOVE] entries: delete the corresponding content from memory files
|
||||
- [SKILL] entries: create a new skill under skills/<name>/SKILL.md using write_file
|
||||
|
||||
## File paths (relative to workspace root)
|
||||
- SOUL.md
|
||||
- USER.md
|
||||
- memory/MEMORY.md
|
||||
- skills/<name>/SKILL.md (for [SKILL] entries only)
|
||||
|
||||
Do NOT guess paths.
|
||||
|
||||
## Editing rules
|
||||
- Edit directly — file contents provided below, no read_file needed
|
||||
- Use exact text as old_text, include surrounding blank lines for unique match
|
||||
- Batch changes to the same file into one edit_file call
|
||||
- For deletions: section header + all bullets as old_text, new_text empty
|
||||
- Surgical edits only — never rewrite entire files
|
||||
- If nothing to update, stop without calling tools
|
||||
|
||||
## Skill creation rules (for [SKILL] entries)
|
||||
- Use write_file to create skills/<name>/SKILL.md
|
||||
- Before writing, read_file `{{ skill_creator_path }}` for format reference (frontmatter structure, naming conventions, quality standards)
|
||||
- **Dedup check**: read existing skills listed below to verify the new skill is not functionally redundant. Skip creation if an existing skill already covers the same workflow.
|
||||
- Include YAML frontmatter with name and description fields
|
||||
- Keep SKILL.md under 2000 words — concise and actionable
|
||||
- Include: when to use, steps, output format, at least one example
|
||||
- Do NOT overwrite existing skills — skip if the skill directory already exists
|
||||
- Reference specific tools the agent has access to (read_file, write_file, exec, web_search, etc.)
|
||||
- Skills are instruction sets, not code — do not include implementation code
|
||||
|
||||
## Quality
|
||||
- Every line must carry standalone value
|
||||
- Concise bullets under clear headers
|
||||
- When reducing (not deleting): keep essential facts, drop verbose details
|
||||
- If uncertain whether to delete, keep but add "(verify currency)"
|
||||
@ -19,7 +19,8 @@ class CommitInfo:
|
||||
|
||||
def format(self, diff: str = "") -> str:
|
||||
"""Format this commit for display, optionally with a diff."""
|
||||
header = f"## {self.message.splitlines()[0]}\n`{self.sha}` — {self.timestamp}\n"
|
||||
summary = self.message.splitlines()[0] if self.message else "(no message)"
|
||||
header = f"## {summary}\n`{self.sha}` — {self.timestamp}\n"
|
||||
if diff:
|
||||
return f"{header}\n```diff\n{diff}\n```"
|
||||
return f"{header}\n(no file changes)"
|
||||
|
||||
@ -1,19 +1,32 @@
|
||||
"""Tests for the Dream class — two-phase memory consolidation via AgentRunner."""
|
||||
"""Tests for Dream driven through AgentLoop._process_system_message."""
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from nanobot.agent.memory import Dream, MemoryStore
|
||||
from nanobot.agent.loop import AgentLoop
|
||||
from nanobot.agent.runner import AgentRunResult
|
||||
from nanobot.agent.skills import BUILTIN_SKILLS_DIR
|
||||
from nanobot.bus.events import InboundMessage
|
||||
from nanobot.bus.queue import MessageBus
|
||||
from nanobot.utils.gitstore import LineAge
|
||||
|
||||
|
||||
def _provider(default_model: str, max_tokens: int = 123) -> MagicMock:
|
||||
provider = MagicMock()
|
||||
provider.get_default_model.return_value = default_model
|
||||
provider.generation = SimpleNamespace(
|
||||
max_tokens=max_tokens, temperature=0.1, reasoning_effort=None
|
||||
)
|
||||
return provider
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path):
|
||||
from nanobot.agent.memory import MemoryStore
|
||||
|
||||
s = MemoryStore(tmp_path)
|
||||
s.write_soul("# Soul\n- Helpful")
|
||||
s.write_user("# User\n- Developer")
|
||||
@ -23,9 +36,7 @@ def store(tmp_path):
|
||||
|
||||
@pytest.fixture
|
||||
def mock_provider():
|
||||
p = MagicMock()
|
||||
p.chat_with_retry = AsyncMock()
|
||||
return p
|
||||
return _provider("test-model")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@ -34,10 +45,16 @@ def mock_runner():
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dream(store, mock_provider, mock_runner):
|
||||
d = Dream(store=store, provider=mock_provider, model="test-model", max_batch_size=5)
|
||||
d._runner = mock_runner
|
||||
return d
|
||||
def loop(tmp_path, mock_provider, mock_runner):
|
||||
loop = AgentLoop(
|
||||
bus=MessageBus(),
|
||||
provider=mock_provider,
|
||||
workspace=tmp_path,
|
||||
model="test-model",
|
||||
context_window_tokens=1000,
|
||||
)
|
||||
loop.dream._runner = mock_runner
|
||||
return loop
|
||||
|
||||
|
||||
def _make_run_result(
|
||||
@ -56,254 +73,418 @@ def _make_run_result(
|
||||
)
|
||||
|
||||
|
||||
class TestDreamRun:
|
||||
async def test_noop_when_no_unprocessed_history(self, dream, mock_provider, mock_runner, store):
|
||||
"""Dream should not call LLM when there's nothing to process."""
|
||||
result = await dream.run()
|
||||
assert result is False
|
||||
mock_provider.chat_with_retry.assert_not_called()
|
||||
class TestDreamAgentLoopIntegration:
|
||||
async def test_completes_goal_state_after_full_backlog(self, loop, mock_runner, store):
|
||||
"""Goal should be completed after processing all backlog in internal loop."""
|
||||
for i in range(6):
|
||||
store.append_history(f"event {i}")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
session = loop.sessions.get_or_create("system:dream")
|
||||
goal = session.metadata.get("goal_state")
|
||||
assert isinstance(goal, dict)
|
||||
assert goal["status"] == "completed"
|
||||
assert store.get_last_dream_cursor() == 6
|
||||
|
||||
async def test_completes_goal_state_on_finish(self, loop, mock_runner, store):
|
||||
"""Goal should be marked completed when backlog is fully processed."""
|
||||
store.append_history("event 1")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
session = loop.sessions.get_or_create("system:dream")
|
||||
goal = session.metadata.get("goal_state")
|
||||
assert goal["status"] == "completed"
|
||||
assert "completed_at" in goal
|
||||
assert "recap" in goal
|
||||
|
||||
async def test_noop_when_no_unprocessed_history(self, loop, mock_runner):
|
||||
"""Dream should not call runner when there's nothing to process."""
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
result = await loop._process_system_message(msg)
|
||||
assert result is None
|
||||
mock_runner.run.assert_not_called()
|
||||
|
||||
async def test_calls_runner_for_unprocessed_entries(self, dream, mock_provider, mock_runner, store):
|
||||
async def test_calls_runner_for_unprocessed_entries(self, loop, mock_runner, store):
|
||||
"""Dream should call AgentRunner when there are unprocessed history entries."""
|
||||
store.append_history("User prefers dark mode")
|
||||
mock_provider.chat_with_retry.return_value = MagicMock(content="New fact")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result(
|
||||
tool_events=[{"name": "edit_file", "status": "ok", "detail": "memory/MEMORY.md"}],
|
||||
))
|
||||
result = await dream.run()
|
||||
assert result is True
|
||||
mock_runner.run = AsyncMock(
|
||||
return_value=_make_run_result(
|
||||
tool_events=[
|
||||
{"name": "edit_file", "status": "ok", "detail": "memory/MEMORY.md"}
|
||||
],
|
||||
)
|
||||
)
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
mock_runner.run.assert_called_once()
|
||||
spec = mock_runner.run.call_args[0][0]
|
||||
assert spec.max_iterations == 10
|
||||
assert spec.fail_on_tool_error is False
|
||||
|
||||
async def test_advances_dream_cursor(self, dream, mock_provider, mock_runner, store):
|
||||
async def test_advances_dream_cursor(self, loop, mock_runner, store):
|
||||
"""Dream should advance the cursor after processing."""
|
||||
store.append_history("event 1")
|
||||
store.append_history("event 2")
|
||||
mock_provider.chat_with_retry.return_value = MagicMock(content="Nothing new")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
await dream.run()
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
assert store.get_last_dream_cursor() == 2
|
||||
|
||||
async def test_compacts_processed_history(self, dream, mock_provider, mock_runner, store):
|
||||
async def test_compacts_processed_history(self, loop, mock_runner, store):
|
||||
"""Dream should compact history after processing."""
|
||||
store.append_history("event 1")
|
||||
store.append_history("event 2")
|
||||
store.append_history("event 3")
|
||||
mock_provider.chat_with_retry.return_value = MagicMock(content="Nothing new")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
await dream.run()
|
||||
# After Dream, cursor is advanced and 3, compact keeps last max_history_entries
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
entries = store.read_unprocessed_history(since_cursor=0)
|
||||
assert all(e["cursor"] > 0 for e in entries)
|
||||
|
||||
async def test_skill_phase_uses_builtin_skill_creator_path(self, dream, mock_provider, mock_runner, store):
|
||||
"""Dream should point skill creation guidance at the builtin skill-creator template."""
|
||||
async def test_processes_full_backlog_in_one_call(self, loop, mock_runner, store):
|
||||
"""Backlog larger than max_batch_size should be fully processed in one call."""
|
||||
for i in range(12):
|
||||
store.append_history(f"event {i}")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
assert store.get_last_dream_cursor() == 12
|
||||
assert mock_runner.run.call_count == 3 # 5 + 5 + 2
|
||||
|
||||
async def test_single_git_commit_for_multi_batch(self, loop, mock_runner, store):
|
||||
"""Multi-batch run should collapse into exactly one git commit."""
|
||||
store.git.init()
|
||||
store.git.auto_commit("initial")
|
||||
for i in range(12):
|
||||
store.append_history(f"event {i}")
|
||||
mock_runner.run = AsyncMock(
|
||||
return_value=_make_run_result(
|
||||
tool_events=[
|
||||
{"name": "edit_file", "status": "ok", "detail": "memory/MEMORY.md"}
|
||||
],
|
||||
)
|
||||
)
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
commits = store.git.log()
|
||||
dream_commits = [c for c in commits if c.message.startswith("dream:")]
|
||||
assert len(dream_commits) == 1
|
||||
|
||||
async def test_system_prompt_cached(self, loop, mock_runner, store):
|
||||
"""Batches within one run should reuse cached system prompt when template mtime unchanged."""
|
||||
for i in range(6):
|
||||
store.append_history(f"event {i}")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
# Two batches (5 + 1), both should use the same cached prompt
|
||||
assert mock_runner.run.call_count == 2
|
||||
first_prompt = mock_runner.run.call_args_list[0][0][0].initial_messages[0]["content"]
|
||||
second_prompt = mock_runner.run.call_args_list[1][0][0].initial_messages[0]["content"]
|
||||
assert second_prompt is first_prompt
|
||||
|
||||
async def test_noop_when_empty_backlog(self, loop, mock_runner, store):
|
||||
"""Empty backlog should not advance cursor or create a commit."""
|
||||
store.git.init()
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
assert store.get_last_dream_cursor() == 0
|
||||
commits = store.git.log()
|
||||
assert len([c for c in commits if c.message.startswith("dream:")]) == 0
|
||||
|
||||
|
||||
class TestDreamPrompt:
|
||||
async def test_prompt_contains_mece_rules(self, loop, mock_runner, store):
|
||||
store.append_history("some event")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
spec = mock_runner.run.call_args[0][0]
|
||||
system_prompt = spec.initial_messages[0]["content"]
|
||||
assert "Do NOT guess paths" in system_prompt
|
||||
assert "SOUL.md" in system_prompt
|
||||
assert "USER.md" in system_prompt
|
||||
assert "MEMORY.md" in system_prompt
|
||||
|
||||
async def test_skill_phase_uses_builtin_skill_creator_path(self, loop, mock_runner, store):
|
||||
store.append_history("Repeated workflow one")
|
||||
store.append_history("Repeated workflow two")
|
||||
mock_provider.chat_with_retry.return_value = MagicMock(content="[SKILL] test-skill: test description")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
|
||||
await dream.run()
|
||||
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
spec = mock_runner.run.call_args[0][0]
|
||||
system_prompt = spec.initial_messages[0]["content"]
|
||||
expected = str(BUILTIN_SKILLS_DIR / "skill-creator" / "SKILL.md")
|
||||
assert expected in system_prompt
|
||||
|
||||
async def test_skill_write_tool_accepts_workspace_relative_skill_path(self, dream, store):
|
||||
"""Dream skill creation should allow skills/<name>/SKILL.md relative to workspace root."""
|
||||
write_tool = dream._tools.get("write_file")
|
||||
assert write_tool is not None
|
||||
|
||||
result = await write_tool.execute(
|
||||
path="skills/test-skill/SKILL.md",
|
||||
content="---\nname: test-skill\ndescription: Test\n---\n",
|
||||
async def test_system_prompt_uses_threshold_from_template_var(self, loop, mock_runner, store):
|
||||
store.append_history("some event")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
|
||||
assert "Successfully wrote" in result
|
||||
assert (store.workspace / "skills" / "test-skill" / "SKILL.md").exists()
|
||||
|
||||
async def test_phase1_prompt_includes_line_age_annotations(self, dream, mock_provider, mock_runner, store):
|
||||
"""Phase 1 prompt should have per-line age suffixes in MEMORY.md when git is available."""
|
||||
store.append_history("some event")
|
||||
mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
|
||||
# Init git so line_ages works
|
||||
store.git.init()
|
||||
store.git.auto_commit("initial memory state")
|
||||
|
||||
await dream.run()
|
||||
|
||||
# The MEMORY.md section should not crash and should contain the memory content
|
||||
call_args = mock_provider.chat_with_retry.call_args
|
||||
user_msg = call_args.kwargs.get("messages", call_args[1].get("messages"))[1]["content"]
|
||||
assert "## Current MEMORY.md" in user_msg
|
||||
|
||||
async def test_phase1_annotates_only_memory_not_soul_or_user(self, dream, mock_provider, mock_runner, store):
|
||||
"""SOUL.md and USER.md should never have age annotations — they are permanent."""
|
||||
store.append_history("some event")
|
||||
mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
|
||||
store.git.init()
|
||||
store.git.auto_commit("initial state")
|
||||
|
||||
await dream.run()
|
||||
|
||||
call_args = mock_provider.chat_with_retry.call_args
|
||||
user_msg = call_args.kwargs.get("messages", call_args[1].get("messages"))[1]["content"]
|
||||
# The ← suffix should only appear in MEMORY.md section
|
||||
memory_section = user_msg.split("## Current MEMORY.md")[1].split("## Current SOUL.md")[0]
|
||||
soul_section = user_msg.split("## Current SOUL.md")[1].split("## Current USER.md")[0]
|
||||
user_section = user_msg.split("## Current USER.md")[1]
|
||||
# SOUL and USER should not contain age arrows
|
||||
assert "\u2190" not in soul_section
|
||||
assert "\u2190" not in user_section
|
||||
|
||||
async def test_phase1_prompt_works_without_git(self, dream, mock_provider, mock_runner, store):
|
||||
"""Phase 1 should work fine even if git is not initialized (no age annotations)."""
|
||||
store.append_history("some event")
|
||||
mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
|
||||
await dream.run()
|
||||
|
||||
# Should still succeed — just without age annotations
|
||||
mock_provider.chat_with_retry.assert_called_once()
|
||||
call_args = mock_provider.chat_with_retry.call_args
|
||||
user_msg = call_args.kwargs.get("messages", call_args[1].get("messages"))[1]["content"]
|
||||
assert "## Current MEMORY.md" in user_msg
|
||||
|
||||
async def test_phase1_prompt_carries_age_suffix_for_stale_lines(
|
||||
self, dream, mock_provider, mock_runner, store,
|
||||
):
|
||||
"""End-to-end: ages >14d must appear verbatim in the LLM prompt, ages ≤14d must not."""
|
||||
# MEMORY.md fixture has 2 non-blank lines ("# Memory" and "- Project X active").
|
||||
# Inject four ages to cover threshold boundaries: >14 suffix, ==14 no suffix, <14 no suffix.
|
||||
store.write_memory("# Memory\n- Project X active\n- fresh item\n- edge case line")
|
||||
store.append_history("some event")
|
||||
mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
|
||||
fake_ages = [
|
||||
LineAge(age_days=30), # "# Memory" → should get ← 30d
|
||||
LineAge(age_days=20), # "- Project X..." → should get ← 20d
|
||||
LineAge(age_days=14), # "- fresh item" → ==14, threshold is strictly >14, no suffix
|
||||
LineAge(age_days=5), # "- edge case..." → no suffix
|
||||
]
|
||||
with patch.object(store.git, "line_ages", return_value=fake_ages):
|
||||
await dream.run()
|
||||
|
||||
call_args = mock_provider.chat_with_retry.call_args
|
||||
user_msg = call_args.kwargs.get("messages", call_args[1].get("messages"))[1]["content"]
|
||||
memory_section = user_msg.split("## Current MEMORY.md")[1].split("## Current SOUL.md")[0]
|
||||
assert "\u2190 30d" in memory_section
|
||||
assert "\u2190 20d" in memory_section
|
||||
assert "\u2190 14d" not in memory_section
|
||||
assert "\u2190 5d" not in memory_section
|
||||
|
||||
async def test_phase1_skips_annotation_when_disabled(
|
||||
self, dream, mock_provider, mock_runner, store,
|
||||
):
|
||||
"""`annotate_line_ages=False` must bypass the git lookup entirely and keep MEMORY.md raw."""
|
||||
store.append_history("some event")
|
||||
mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
|
||||
dream.annotate_line_ages = False
|
||||
# line_ages must be bypassed entirely — verify with a spy rather than a
|
||||
# raising side_effect, because _annotate_with_ages catches Exception
|
||||
# (which swallows AssertionError) and would hide an accidental call.
|
||||
with patch.object(store.git, "line_ages") as mock_line_ages:
|
||||
await dream.run()
|
||||
mock_line_ages.assert_not_called()
|
||||
|
||||
call_args = mock_provider.chat_with_retry.call_args
|
||||
user_msg = call_args.kwargs.get("messages", call_args[1].get("messages"))[1]["content"]
|
||||
assert "\u2190" not in user_msg
|
||||
|
||||
async def test_phase1_skips_annotation_on_line_ages_length_mismatch(
|
||||
self, dream, mock_provider, mock_runner, store,
|
||||
):
|
||||
"""If ages length != lines length (dirty working tree), skip annotation instead of mis-tagging."""
|
||||
# MEMORY.md has 2 non-blank lines but we hand back only 1 age → mismatch.
|
||||
store.append_history("some event")
|
||||
mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
|
||||
with patch.object(store.git, "line_ages", return_value=[LineAge(age_days=999)]):
|
||||
await dream.run()
|
||||
|
||||
call_args = mock_provider.chat_with_retry.call_args
|
||||
user_msg = call_args.kwargs.get("messages", call_args[1].get("messages"))[1]["content"]
|
||||
memory_section = user_msg.split("## Current MEMORY.md")[1].split("## Current SOUL.md")[0]
|
||||
# No age arrow at all — we refused to annotate rather than tag the wrong line.
|
||||
assert "\u2190" not in memory_section
|
||||
|
||||
async def test_phase1_prompt_uses_threshold_from_template_var(
|
||||
self, dream, mock_provider, mock_runner, store,
|
||||
):
|
||||
"""System prompt should reference the stale-threshold constant, not a hardcoded 14."""
|
||||
store.append_history("some event")
|
||||
mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
|
||||
await dream.run()
|
||||
|
||||
system_msg = mock_provider.chat_with_retry.call_args.kwargs["messages"][0]["content"]
|
||||
# The template renders with stale_threshold_days=14 → LLM must see "N>14"
|
||||
await loop._process_system_message(msg)
|
||||
spec = mock_runner.run.call_args[0][0]
|
||||
system_msg = spec.initial_messages[0]["content"]
|
||||
assert "N>14" in system_msg
|
||||
|
||||
|
||||
class TestDreamPromptCaps:
|
||||
"""Dream's Phase 1/2 prompt must not be poisoned by a legacy oversized
|
||||
history entry or a runaway MEMORY.md. Without caps, a single pre-#3412
|
||||
raw_archive dump in history.jsonl would make every subsequent Dream run
|
||||
exceed the context window and silently advance the cursor past real work.
|
||||
"""
|
||||
|
||||
async def test_phase1_caps_huge_memory_file(
|
||||
self, dream, mock_provider, mock_runner, store,
|
||||
):
|
||||
"""A MEMORY.md much larger than _MEMORY_FILE_MAX_CHARS must be truncated
|
||||
in the prompt preview (full content is still reachable via read_file)."""
|
||||
store.write_memory("M" * (dream._MEMORY_FILE_MAX_CHARS * 5))
|
||||
async def test_caps_huge_memory_file(self, loop, mock_runner, store):
|
||||
store.write_memory("M" * (loop.dream._MEMORY_FILE_MAX_CHARS * 5))
|
||||
store.append_history("some event")
|
||||
mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
spec = mock_runner.run.call_args[0][0]
|
||||
user_msg = spec.initial_messages[1]["content"]
|
||||
memory_section = user_msg.split("## Current MEMORY.md")[1].split(
|
||||
"## Current SOUL.md"
|
||||
)[0]
|
||||
assert len(memory_section) < loop.dream._MEMORY_FILE_MAX_CHARS + 500
|
||||
|
||||
await dream.run()
|
||||
|
||||
user_msg = mock_provider.chat_with_retry.call_args.kwargs["messages"][1]["content"]
|
||||
memory_section = user_msg.split("## Current MEMORY.md")[1].split("## Current SOUL.md")[0]
|
||||
assert len(memory_section) < dream._MEMORY_FILE_MAX_CHARS + 500
|
||||
|
||||
async def test_phase1_caps_huge_history_entry(
|
||||
self, dream, mock_provider, mock_runner, store,
|
||||
):
|
||||
"""A legacy oversized history entry (e.g. pre-#3412 raw_archive dump)
|
||||
must not explode the Phase 1 prompt — each entry is capped in the
|
||||
preview, even though the JSONL record itself stays full-size."""
|
||||
# Bypass the append_history cap by writing directly, simulating a
|
||||
# record that was written by an older nanobot build before any caps.
|
||||
async def test_caps_huge_history_entry(self, loop, mock_runner, store):
|
||||
store.history_file.write_text(
|
||||
json.dumps({
|
||||
"cursor": 1,
|
||||
"timestamp": "2026-04-01 10:00",
|
||||
"content": "H" * (dream._HISTORY_ENTRY_PREVIEW_MAX_CHARS * 8),
|
||||
}) + "\n",
|
||||
json.dumps(
|
||||
{
|
||||
"cursor": 1,
|
||||
"timestamp": "2026-04-01 10:00",
|
||||
"content": "H" * (loop.dream._HISTORY_ENTRY_PREVIEW_MAX_CHARS * 8),
|
||||
}
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
spec = mock_runner.run.call_args[0][0]
|
||||
user_msg = spec.initial_messages[1]["content"]
|
||||
history_section = user_msg.split("## Conversation History\n")[1].split(
|
||||
"\n\n## Current Date"
|
||||
)[0]
|
||||
assert len(history_section) < loop.dream._HISTORY_ENTRY_PREVIEW_MAX_CHARS + 500
|
||||
|
||||
await dream.run()
|
||||
|
||||
user_msg = mock_provider.chat_with_retry.call_args.kwargs["messages"][1]["content"]
|
||||
history_section = user_msg.split("## Conversation History\n")[1].split("\n\n## Current Date")[0]
|
||||
assert len(history_section) < dream._HISTORY_ENTRY_PREVIEW_MAX_CHARS + 500
|
||||
class TestDreamTools:
|
||||
def test_apply_patch_tool_registered(self, loop):
|
||||
tool = loop.dream._tools.get("apply_patch")
|
||||
assert tool is not None
|
||||
|
||||
|
||||
class TestDreamCaps:
|
||||
def test_batch_size_default_is_5(self):
|
||||
from nanobot.config.schema import DreamConfig
|
||||
|
||||
assert DreamConfig().max_batch_size == 5
|
||||
|
||||
def test_memory_cap_is_16k(self, loop):
|
||||
assert loop.dream._MEMORY_FILE_MAX_CHARS == 16_000
|
||||
|
||||
|
||||
class TestDreamSkipFiltering:
|
||||
async def test_skip_entries_removed_from_prompt(self, loop, mock_runner, store):
|
||||
store.append_history("- [skip] greeting\n- [permanent] User prefers dark mode")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
spec = mock_runner.run.call_args[0][0]
|
||||
user_msg = spec.initial_messages[1]["content"]
|
||||
assert "User prefers dark mode" in user_msg
|
||||
assert "[skip]" not in user_msg
|
||||
assert "greeting" not in user_msg
|
||||
|
||||
|
||||
class TestDreamAgeAnnotations:
|
||||
async def test_prompt_includes_line_age_annotations(self, loop, mock_runner, store):
|
||||
store.append_history("some event")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
store.git.init()
|
||||
store.git.auto_commit("initial memory state")
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
spec = mock_runner.run.call_args[0][0]
|
||||
user_msg = spec.initial_messages[1]["content"]
|
||||
assert "## Current MEMORY.md" in user_msg
|
||||
|
||||
async def test_annotates_only_memory_not_soul_or_user(self, loop, mock_runner, store):
|
||||
store.append_history("some event")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
store.git.init()
|
||||
store.git.auto_commit("initial state")
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
spec = mock_runner.run.call_args[0][0]
|
||||
user_msg = spec.initial_messages[1]["content"]
|
||||
soul_section = user_msg.split("## Current SOUL.md")[1].split(
|
||||
"## Current USER.md"
|
||||
)[0]
|
||||
user_section = user_msg.split("## Current USER.md")[1]
|
||||
assert "←" not in soul_section
|
||||
assert "←" not in user_section
|
||||
|
||||
async def test_prompt_works_without_git(self, loop, mock_runner, store):
|
||||
store.append_history("some event")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
mock_runner.run.assert_called_once()
|
||||
spec = mock_runner.run.call_args[0][0]
|
||||
user_msg = spec.initial_messages[1]["content"]
|
||||
assert "## Current MEMORY.md" in user_msg
|
||||
|
||||
async def test_prompt_carries_age_suffix_for_stale_lines(self, loop, mock_runner, store):
|
||||
store.write_memory(
|
||||
"# Memory\n- Project X active\n- fresh item\n- edge case line"
|
||||
)
|
||||
store.append_history("some event")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
fake_ages = [
|
||||
LineAge(age_days=30),
|
||||
LineAge(age_days=20),
|
||||
LineAge(age_days=14),
|
||||
LineAge(age_days=5),
|
||||
]
|
||||
with patch.object(loop.dream.store.git, "line_ages", return_value=fake_ages):
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
spec = mock_runner.run.call_args[0][0]
|
||||
user_msg = spec.initial_messages[1]["content"]
|
||||
memory_section = user_msg.split("## Current MEMORY.md")[1].split(
|
||||
"## Current SOUL.md"
|
||||
)[0]
|
||||
assert "← 30d" in memory_section
|
||||
assert "← 20d" in memory_section
|
||||
assert "← 14d" not in memory_section
|
||||
assert "← 5d" not in memory_section
|
||||
|
||||
async def test_skips_annotation_when_disabled(self, loop, mock_runner, store):
|
||||
store.append_history("some event")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
loop.dream.annotate_line_ages = False
|
||||
with patch.object(loop.dream.store.git, "line_ages") as mock_line_ages:
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
mock_line_ages.assert_not_called()
|
||||
spec = mock_runner.run.call_args[0][0]
|
||||
user_msg = spec.initial_messages[1]["content"]
|
||||
assert "←" not in user_msg
|
||||
|
||||
async def test_skips_annotation_on_line_ages_length_mismatch(self, loop, mock_runner, store):
|
||||
store.append_history("some event")
|
||||
mock_runner.run = AsyncMock(return_value=_make_run_result())
|
||||
with patch.object(
|
||||
loop.dream.store.git, "line_ages", return_value=[LineAge(age_days=999)]
|
||||
):
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
spec = mock_runner.run.call_args[0][0]
|
||||
user_msg = spec.initial_messages[1]["content"]
|
||||
memory_section = user_msg.split("## Current MEMORY.md")[1].split(
|
||||
"## Current SOUL.md"
|
||||
)[0]
|
||||
assert "←" not in memory_section
|
||||
|
||||
|
||||
class TestDreamSessionPersistence:
|
||||
async def test_writes_session_on_success(self, loop, mock_runner, store):
|
||||
store.append_history("event one")
|
||||
store.append_history("event two")
|
||||
mock_runner.run = AsyncMock(
|
||||
return_value=_make_run_result(
|
||||
tool_events=[
|
||||
{"name": "edit_file", "status": "ok", "detail": "memory/MEMORY.md"}
|
||||
],
|
||||
)
|
||||
)
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
session_path = store.memory_dir / ".dream_session.json"
|
||||
assert session_path.exists()
|
||||
data = json.loads(session_path.read_text(encoding="utf-8"))
|
||||
assert data["batch"]["from_cursor"] == 0
|
||||
assert data["batch"]["to_cursor"] == 2
|
||||
assert data["batch"]["count"] == 2
|
||||
assert data["stop_reason"] == "completed"
|
||||
assert data["changelog"] == ["edit_file: memory/MEMORY.md"]
|
||||
assert "timestamp" in data
|
||||
assert "elapsed_seconds" in data
|
||||
assert "messages" in data
|
||||
|
||||
async def test_no_session_record_on_failure(self, loop, mock_runner, store):
|
||||
"""Failed batch should not write a session record (cursor stays put for retry)."""
|
||||
store.append_history("event one")
|
||||
mock_runner.run = AsyncMock(side_effect=RuntimeError("LLM error"))
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
session_path = store.memory_dir / ".dream_session.json"
|
||||
assert not session_path.exists()
|
||||
assert store.get_last_dream_cursor() == 0
|
||||
|
||||
async def test_session_contains_full_messages(self, loop, mock_runner, store):
|
||||
store.append_history("event one")
|
||||
messages = [
|
||||
{"role": "system", "content": "you are a memory bot"},
|
||||
{"role": "user", "content": "history here"},
|
||||
{"role": "assistant", "content": "I will edit MEMORY.md"},
|
||||
]
|
||||
result = _make_run_result()
|
||||
result.messages = messages
|
||||
mock_runner.run = AsyncMock(return_value=result)
|
||||
msg = InboundMessage(
|
||||
channel="system", sender_id="dream", chat_id="dream", content=""
|
||||
)
|
||||
await loop._process_system_message(msg)
|
||||
session_path = store.memory_dir / ".dream_session.json"
|
||||
data = json.loads(session_path.read_text(encoding="utf-8"))
|
||||
assert data["messages"] == messages
|
||||
assert data["prompt_chars"] > 0
|
||||
assert data["commit_sha"] is None
|
||||
|
||||
125
tests/agent/test_memory.py
Normal file
125
tests/agent/test_memory.py
Normal file
@ -0,0 +1,125 @@
|
||||
"""Tests for memory system: Consolidator, token estimation, truncation."""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from nanobot.agent.memory import _TIKTOKEN_ENC, Consolidator, MemoryStore, _estimate_tokens
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path):
|
||||
s = MemoryStore(tmp_path)
|
||||
s.write_soul("# Soul\n- Helpful")
|
||||
s.write_user("# User\n- Developer")
|
||||
s.write_memory("# Memory\n- Project X active")
|
||||
return s
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_provider():
|
||||
p = MagicMock()
|
||||
p.chat_with_retry = AsyncMock()
|
||||
p.generation.max_tokens = 4096
|
||||
return p
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_sessions():
|
||||
return MagicMock()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_build_messages():
|
||||
return MagicMock(return_value=[])
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_get_tool_definitions():
|
||||
return MagicMock(return_value=[])
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def consolidator(store, mock_provider, mock_sessions, mock_build_messages, mock_get_tool_definitions):
|
||||
return Consolidator(
|
||||
store=store,
|
||||
provider=mock_provider,
|
||||
model="test-model",
|
||||
sessions=mock_sessions,
|
||||
context_window_tokens=128_000,
|
||||
build_messages=mock_build_messages,
|
||||
get_tool_definitions=mock_get_tool_definitions,
|
||||
)
|
||||
|
||||
|
||||
class TestEstimateTokens:
|
||||
def test_estimate_tokens_returns_positive(self):
|
||||
assert _estimate_tokens("hello world") > 0
|
||||
|
||||
def test_estimate_tokens_english_approximate(self):
|
||||
# English is roughly 1 token per 4 chars as fallback
|
||||
text = "a " * 100
|
||||
if _TIKTOKEN_ENC is not None:
|
||||
expected = len(_TIKTOKEN_ENC.encode(text))
|
||||
else:
|
||||
expected = len(text) // 4
|
||||
assert _estimate_tokens(text) == expected
|
||||
|
||||
|
||||
class TestTruncateToTokenBudget:
|
||||
def test_reserve_tokens_reduces_budget(self, consolidator):
|
||||
long_text = "word " * 200_000
|
||||
# Without reserve, more text survives
|
||||
no_reserve = consolidator._truncate_to_token_budget(long_text, reserve_tokens=0)
|
||||
with_reserve = consolidator._truncate_to_token_budget(long_text, reserve_tokens=500)
|
||||
assert len(with_reserve) < len(no_reserve)
|
||||
|
||||
def test_reserve_tokens_zero_default(self, consolidator):
|
||||
text = "hello world"
|
||||
result = consolidator._truncate_to_token_budget(text)
|
||||
assert result == text
|
||||
|
||||
|
||||
class TestConsolidatorPrompt:
|
||||
def test_prompt_contains_snip(self):
|
||||
from nanobot.utils.prompt_templates import render_template
|
||||
text = render_template("agent/consolidator_archive.md", strip=True)
|
||||
assert "SNIP" in text
|
||||
assert "[permanent]" in text
|
||||
assert "[skip]" in text
|
||||
|
||||
|
||||
class TestConsolidatorArchive:
|
||||
async def test_archive_injects_dedup_context(self, consolidator, mock_provider, store):
|
||||
store.write_memory("- User prefers dark mode")
|
||||
store.write_user("- Developer")
|
||||
messages = [{"role": "user", "content": "hello", "timestamp": "2026-01-01 10:00"}]
|
||||
|
||||
mock_provider.chat_with_retry.return_value = MagicMock(
|
||||
content="(nothing)", finish_reason="stop"
|
||||
)
|
||||
await consolidator.archive(messages)
|
||||
|
||||
call_args = mock_provider.chat_with_retry.call_args
|
||||
user_msg = call_args.kwargs["messages"][1]["content"]
|
||||
assert "## Current MEMORY.md (for dedup)" in user_msg
|
||||
assert "User prefers dark mode" in user_msg
|
||||
assert "## Current USER.md (for dedup)" in user_msg
|
||||
assert "Developer" in user_msg
|
||||
|
||||
async def test_archive_skips_dedup_when_budget_exhausted(self, consolidator, mock_provider, store):
|
||||
# Shrink token budget so dedup context (always capped at ~6000 chars)
|
||||
# exceeds the available room.
|
||||
consolidator.context_window_tokens = 6_000
|
||||
store.write_memory("word " * 10_000)
|
||||
messages = [{"role": "user", "content": "hello", "timestamp": "2026-01-01 10:00"}]
|
||||
|
||||
mock_provider.chat_with_retry.return_value = MagicMock(
|
||||
content="(nothing)", finish_reason="stop"
|
||||
)
|
||||
await consolidator.archive(messages)
|
||||
|
||||
call_args = mock_provider.chat_with_retry.call_args
|
||||
user_msg = call_args.kwargs["messages"][1]["content"]
|
||||
# Should not contain dedup context when budget is exhausted
|
||||
assert "## Current MEMORY.md (for dedup)" not in user_msg
|
||||
@ -292,3 +292,95 @@ def test_from_config_static_preset_loader_does_not_enable_hot_reload(tmp_path) -
|
||||
loop = AgentLoop.from_config(config)
|
||||
assert loop._provider_snapshot_loader is None
|
||||
assert loop._preset_snapshot_loader is not None
|
||||
|
||||
|
||||
class TestDreamModelOverride:
|
||||
def test_dream_follows_main_when_no_override(self, tmp_path) -> None:
|
||||
provider = _provider("base-model")
|
||||
loop = AgentLoop(
|
||||
bus=MessageBus(),
|
||||
provider=provider,
|
||||
workspace=tmp_path,
|
||||
model="base-model",
|
||||
context_window_tokens=1000,
|
||||
)
|
||||
assert loop.dream.model == "base-model"
|
||||
assert loop.dream.provider is provider
|
||||
|
||||
def test_dream_raw_model_override(self, tmp_path) -> None:
|
||||
provider = _provider("base-model")
|
||||
loop = AgentLoop(
|
||||
bus=MessageBus(),
|
||||
provider=provider,
|
||||
workspace=tmp_path,
|
||||
model="base-model",
|
||||
context_window_tokens=1000,
|
||||
dream_model_override="custom-model-v2",
|
||||
)
|
||||
assert loop.dream.model == "custom-model-v2"
|
||||
assert loop.dream.provider is provider
|
||||
|
||||
def test_dream_preset_override(self, tmp_path) -> None:
|
||||
cheap_provider = _provider("openai/gpt-4.1-mini", max_tokens=2048)
|
||||
preset = ModelPresetConfig(
|
||||
model="openai/gpt-4.1-mini",
|
||||
provider="openai",
|
||||
max_tokens=2048,
|
||||
context_window_tokens=128_000,
|
||||
)
|
||||
loop = AgentLoop(
|
||||
bus=MessageBus(),
|
||||
provider=_provider("base-model"),
|
||||
workspace=tmp_path,
|
||||
model="base-model",
|
||||
context_window_tokens=1000,
|
||||
model_presets={"cheap": preset},
|
||||
dream_model_override="cheap",
|
||||
preset_snapshot_loader=lambda _name: ProviderSnapshot(
|
||||
provider=cheap_provider,
|
||||
model=preset.model,
|
||||
context_window_tokens=preset.context_window_tokens,
|
||||
signature=("cheap", preset.model),
|
||||
),
|
||||
)
|
||||
assert loop.dream.model == "openai/gpt-4.1-mini"
|
||||
assert loop.dream.provider is cheap_provider
|
||||
assert loop.dream._runner.provider is cheap_provider
|
||||
|
||||
def test_dream_override_survives_main_preset_switch(self, tmp_path) -> None:
|
||||
base_provider = _provider("base-model")
|
||||
fast_provider = _provider("openai/gpt-4.1", max_tokens=4096)
|
||||
cheap_provider = _provider("openai/gpt-4.1-mini", max_tokens=2048)
|
||||
loop = AgentLoop(
|
||||
bus=MessageBus(),
|
||||
provider=base_provider,
|
||||
workspace=tmp_path,
|
||||
model="base-model",
|
||||
context_window_tokens=1000,
|
||||
model_presets={
|
||||
"fast": ModelPresetConfig(model="openai/gpt-4.1"),
|
||||
"cheap": ModelPresetConfig(model="openai/gpt-4.1-mini"),
|
||||
},
|
||||
dream_model_override="cheap",
|
||||
preset_snapshot_loader=lambda name: ProviderSnapshot(
|
||||
provider=fast_provider if name == "fast" else cheap_provider,
|
||||
model="openai/gpt-4.1" if name == "fast" else "openai/gpt-4.1-mini",
|
||||
context_window_tokens=32_768 if name == "fast" else 128_000,
|
||||
signature=(name, "model"),
|
||||
),
|
||||
)
|
||||
# Initially dream is on cheap
|
||||
assert loop.dream.model == "openai/gpt-4.1-mini"
|
||||
assert loop.dream.provider is cheap_provider
|
||||
|
||||
# Switch main preset to fast
|
||||
loop.set_model_preset("fast")
|
||||
|
||||
# Main agent should be on fast
|
||||
assert loop.model == "openai/gpt-4.1"
|
||||
assert loop.provider is fast_provider
|
||||
|
||||
# Dream should still be on cheap override
|
||||
assert loop.dream.model == "openai/gpt-4.1-mini"
|
||||
assert loop.dream.provider is cheap_provider
|
||||
assert loop.dream._runner.provider is cheap_provider
|
||||
|
||||
@ -113,6 +113,52 @@ async def test_dream_restore_lists_versions_with_next_steps() -> None:
|
||||
assert "Restore a version with `/dream-restore <sha>`." in out.content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dream_log_shows_summary_and_analysis() -> None:
|
||||
commit = CommitInfo(
|
||||
sha="abcd1234",
|
||||
message="dream: 2026-04-04, 2 change(s)\n\n[ADD] fact A →USER\n[REMOVE] old fact",
|
||||
timestamp="2026-04-04 12:00",
|
||||
)
|
||||
diff = (
|
||||
"diff --git a/SOUL.md b/SOUL.md\n"
|
||||
"--- a/SOUL.md\n"
|
||||
"+++ b/SOUL.md\n"
|
||||
"@@ -1 +1 @@\n"
|
||||
"-old\n"
|
||||
"+new\n"
|
||||
)
|
||||
git = _FakeGit(commits=[commit], diff_map={commit.sha: (commit, diff)})
|
||||
|
||||
out = await cmd_dream_log(_make_ctx("/dream-log", git))
|
||||
|
||||
assert "## Dream Update" in out.content
|
||||
assert "- Summary: dream: 2026-04-04, 2 change(s)" in out.content
|
||||
assert "### Analysis" in out.content
|
||||
assert "[ADD] fact A →USER" in out.content
|
||||
assert "[REMOVE] old fact" in out.content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dream_log_with_empty_commit_message() -> None:
|
||||
commit = CommitInfo(sha="abcd1234", message="", timestamp="2026-04-04 12:00")
|
||||
diff = (
|
||||
"diff --git a/SOUL.md b/SOUL.md\n"
|
||||
"--- a/SOUL.md\n"
|
||||
"+++ b/SOUL.md\n"
|
||||
"@@ -1 +1 @@\n"
|
||||
"-old\n"
|
||||
"+new\n"
|
||||
)
|
||||
git = _FakeGit(commits=[commit], diff_map={commit.sha: (commit, diff)})
|
||||
|
||||
out = await cmd_dream_log(_make_ctx("/dream-log", git))
|
||||
|
||||
assert "## Dream Update" in out.content
|
||||
assert "- Summary:" not in out.content
|
||||
assert "### Analysis" not in out.content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dream_restore_success_mentions_files_and_followup() -> None:
|
||||
commit = CommitInfo(sha="abcd1234", message="dream: latest", timestamp="2026-04-04 12:00")
|
||||
|
||||
@ -1,216 +1,13 @@
|
||||
"""Tests for GitStore — line_ages() and core git operations."""
|
||||
|
||||
import subprocess
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from nanobot.utils.gitstore import GitStore
|
||||
from nanobot.utils.gitstore import CommitInfo
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def git(tmp_path):
|
||||
"""Create an initialized GitStore with tracked MEMORY.md."""
|
||||
g = GitStore(tmp_path, tracked_files=["MEMORY.md", "SOUL.md"])
|
||||
g.init()
|
||||
return g
|
||||
class TestCommitInfo:
|
||||
def test_format_with_empty_message(self):
|
||||
commit = CommitInfo(sha="abcd1234", message="", timestamp="2026-04-04 12:00")
|
||||
result = commit.format()
|
||||
assert "(no message)" in result or "## " in result
|
||||
|
||||
|
||||
class TestLineAges:
|
||||
def test_returns_empty_when_not_initialized(self, tmp_path):
|
||||
"""line_ages should return [] if the git repo is not initialized."""
|
||||
git = GitStore(tmp_path, tracked_files=["MEMORY.md"])
|
||||
assert git.line_ages("MEMORY.md") == []
|
||||
|
||||
def test_returns_empty_for_missing_file(self, git):
|
||||
"""line_ages should return [] for a file that doesn't exist."""
|
||||
assert git.line_ages("SOUL.md") == []
|
||||
|
||||
def test_returns_empty_for_empty_file(self, git, tmp_path):
|
||||
"""line_ages should return [] for an empty tracked file."""
|
||||
(tmp_path / "SOUL.md").write_text("", encoding="utf-8")
|
||||
git.auto_commit("empty soul")
|
||||
assert git.line_ages("SOUL.md") == []
|
||||
|
||||
def test_one_age_per_line(self, git, tmp_path):
|
||||
"""line_ages should return one entry per line in the file."""
|
||||
content = "# Memory\n\n## Section A\n- item 1\n"
|
||||
(tmp_path / "MEMORY.md").write_text(content, encoding="utf-8")
|
||||
git.auto_commit("initial")
|
||||
ages = git.line_ages("MEMORY.md")
|
||||
assert len(ages) == len(content.splitlines())
|
||||
|
||||
def test_fresh_lines_have_age_zero(self, git, tmp_path):
|
||||
"""Lines committed today should have age_days=0."""
|
||||
(tmp_path / "MEMORY.md").write_text("## A\n- x\n", encoding="utf-8")
|
||||
git.auto_commit("initial")
|
||||
ages = git.line_ages("MEMORY.md")
|
||||
assert all(a.age_days == 0 for a in ages)
|
||||
|
||||
def test_age_differentiates_across_days(self, git, tmp_path):
|
||||
"""Lines committed today should show correct age when 'now' is mocked forward."""
|
||||
(tmp_path / "MEMORY.md").write_text("## A\n- x\n", encoding="utf-8")
|
||||
git.auto_commit("initial")
|
||||
|
||||
future_now = datetime.now(tz=timezone.utc) + timedelta(days=30)
|
||||
with patch("nanobot.utils.gitstore.datetime") as mock_dt:
|
||||
mock_dt.now.return_value = future_now
|
||||
mock_dt.fromtimestamp = datetime.fromtimestamp
|
||||
ages = git.line_ages("MEMORY.md")
|
||||
|
||||
assert len(ages) == 2
|
||||
assert all(a.age_days == 30 for a in ages)
|
||||
|
||||
def test_annotate_failure_returns_empty(self, tmp_path):
|
||||
"""If annotate fails, line_ages should return [] gracefully."""
|
||||
git = GitStore(tmp_path, tracked_files=["MEMORY.md"])
|
||||
# Don't init — annotate will fail
|
||||
assert git.line_ages("MEMORY.md") == []
|
||||
|
||||
def test_partial_edit_only_updates_changed_lines(self, git, tmp_path):
|
||||
"""Only modified lines should reflect the new commit's timestamp."""
|
||||
(tmp_path / "MEMORY.md").write_text(
|
||||
"# Memory\n\n## A\n- old\n\n## B\n- keep\n", encoding="utf-8"
|
||||
)
|
||||
git.auto_commit("commit1")
|
||||
time.sleep(1.1)
|
||||
|
||||
# Only modify section A
|
||||
(tmp_path / "MEMORY.md").write_text(
|
||||
"# Memory\n\n## A\n- new\n\n## B\n- keep\n", encoding="utf-8"
|
||||
)
|
||||
git.auto_commit("commit2")
|
||||
|
||||
ages = git.line_ages("MEMORY.md")
|
||||
lines = (tmp_path / "MEMORY.md").read_text(encoding="utf-8").splitlines()
|
||||
# All lines are from today, but verify line-level tracking works
|
||||
assert len(ages) == len(lines)
|
||||
# "- new" line and "- keep" line both age=0 (same day), but
|
||||
# the key point is we get per-line results
|
||||
assert len(ages) == 7
|
||||
|
||||
|
||||
class TestNestedRepoProtection:
|
||||
"""Regression tests for GitHub issue #2980: nested repo protection."""
|
||||
|
||||
def test_init_refuses_inside_git_repo(self, tmp_path):
|
||||
"""init() should detect it's inside an existing git repo and refuse."""
|
||||
project = tmp_path / "project"
|
||||
project.mkdir()
|
||||
(project / ".git").mkdir()
|
||||
|
||||
workspace = project / "workspace"
|
||||
workspace.mkdir()
|
||||
|
||||
g = GitStore(workspace, tracked_files=["MEMORY.md"])
|
||||
result = g.init()
|
||||
|
||||
assert result is False
|
||||
assert not (workspace / ".git").is_dir()
|
||||
|
||||
def test_init_preserves_existing_gitignore(self, tmp_path):
|
||||
"""init() should preserve existing .gitignore entries and append new ones."""
|
||||
workspace = tmp_path / "workspace"
|
||||
workspace.mkdir()
|
||||
|
||||
existing = "*.pyc\n__pycache__/\n"
|
||||
(workspace / ".gitignore").write_text(existing, encoding="utf-8")
|
||||
|
||||
g = GitStore(workspace, tracked_files=["MEMORY.md"])
|
||||
result = g.init()
|
||||
|
||||
assert result is True
|
||||
gitignore = (workspace / ".gitignore").read_text(encoding="utf-8")
|
||||
assert "*.pyc" in gitignore
|
||||
assert "__pycache__/" in gitignore
|
||||
assert "!MEMORY.md" in gitignore
|
||||
assert "!.gitignore" in gitignore
|
||||
|
||||
def test_init_no_gitignore_creates_new(self, tmp_path):
|
||||
"""init() should create .gitignore with Dream content when none exists."""
|
||||
workspace = tmp_path / "workspace"
|
||||
workspace.mkdir()
|
||||
|
||||
g = GitStore(workspace, tracked_files=["MEMORY.md"])
|
||||
result = g.init()
|
||||
|
||||
assert result is True
|
||||
gitignore = (workspace / ".gitignore").read_text(encoding="utf-8")
|
||||
expected = g._build_gitignore()
|
||||
assert gitignore == expected
|
||||
|
||||
def test_init_gitignore_merge_idempotent(self, tmp_path):
|
||||
"""init() should not duplicate Dream entries already in .gitignore."""
|
||||
workspace = tmp_path / "workspace"
|
||||
workspace.mkdir()
|
||||
|
||||
# Pre-existing .gitignore that already has some Dream entries
|
||||
existing = "*.pyc\n/*\n!MEMORY.md\n"
|
||||
(workspace / ".gitignore").write_text(existing, encoding="utf-8")
|
||||
|
||||
g = GitStore(workspace, tracked_files=["MEMORY.md"])
|
||||
result = g.init()
|
||||
|
||||
assert result is True
|
||||
gitignore = (workspace / ".gitignore").read_text(encoding="utf-8")
|
||||
# No duplicate lines
|
||||
lines = gitignore.splitlines()
|
||||
assert lines.count("/*") == 1
|
||||
assert lines.count("!MEMORY.md") == 1
|
||||
# Existing entry preserved, new Dream entries appended
|
||||
assert "*.pyc" in gitignore
|
||||
assert "!.gitignore" in gitignore
|
||||
|
||||
def test_init_outside_git_repo_works_normally(self, tmp_path):
|
||||
"""init() should succeed and create .git when not inside a git repo."""
|
||||
workspace = tmp_path / "workspace"
|
||||
workspace.mkdir()
|
||||
|
||||
g = GitStore(workspace, tracked_files=["MEMORY.md"])
|
||||
result = g.init()
|
||||
|
||||
assert result is True
|
||||
assert (workspace / ".git").is_dir()
|
||||
|
||||
def test_init_refuses_inside_git_worktree(self, tmp_path):
|
||||
"""init() should refuse when the parent checkout is a git worktree."""
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
subprocess.run(["git", "init", "-q", str(repo)], check=True)
|
||||
(repo / "README.md").write_text("x\n", encoding="utf-8")
|
||||
subprocess.run(["git", "-C", str(repo), "add", "README.md"], check=True)
|
||||
subprocess.run(
|
||||
[
|
||||
"git",
|
||||
"-C",
|
||||
str(repo),
|
||||
"-c",
|
||||
"user.name=test",
|
||||
"-c",
|
||||
"user.email=test@example.com",
|
||||
"commit",
|
||||
"-q",
|
||||
"-m",
|
||||
"init",
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(["git", "-C", str(repo), "branch", "wt-branch"], check=True)
|
||||
|
||||
worktree = tmp_path / "worktree"
|
||||
subprocess.run(
|
||||
["git", "-C", str(repo), "worktree", "add", "-q", str(worktree), "wt-branch"],
|
||||
check=True,
|
||||
)
|
||||
assert (worktree / ".git").is_file()
|
||||
|
||||
workspace = worktree / "workspace"
|
||||
workspace.mkdir()
|
||||
|
||||
g = GitStore(workspace, tracked_files=["MEMORY.md"])
|
||||
result = g.init()
|
||||
|
||||
assert result is False
|
||||
assert not (workspace / ".git").exists()
|
||||
def test_format_with_message(self):
|
||||
commit = CommitInfo(sha="abcd1234", message="dream: update", timestamp="2026-04-04 12:00")
|
||||
result = commit.format()
|
||||
assert "dream: update" in result
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user