diff --git a/nanobot/agent/loop.py b/nanobot/agent/loop.py index f50583119..848366ca8 100644 --- a/nanobot/agent/loop.py +++ b/nanobot/agent/loop.py @@ -8,6 +8,7 @@ import os import time from contextlib import AsyncExitStack, nullcontext, suppress from dataclasses import dataclass, field +from datetime import datetime from enum import Enum, auto from pathlib import Path from typing import TYPE_CHECKING, Any, Awaitable, Callable @@ -19,7 +20,13 @@ from nanobot.agent import model_presets as preset_helpers from nanobot.agent.autocompact import AutoCompact from nanobot.agent.context import ContextBuilder from nanobot.agent.hook import AgentHook, CompositeHook -from nanobot.agent.memory import Consolidator, Dream +from nanobot.agent.memory import ( + _STALE_THRESHOLD_DAYS, + Consolidator, + Dream, + _estimate_tokens, + _strip_skip_lines, +) from nanobot.agent.progress_hook import AgentProgressHook from nanobot.agent.runner import _MAX_INJECTIONS_PER_TURN, AgentRunner, AgentRunSpec from nanobot.agent.subagent import SubagentManager @@ -34,6 +41,8 @@ from nanobot.config.schema import AgentDefaults, ModelPresetConfig from nanobot.providers.base import LLMProvider from nanobot.providers.factory import ProviderSnapshot from nanobot.session.goal_state import ( + GOAL_STATE_KEY, + goal_state_runtime_lines, runner_wall_llm_timeout_s, ) from nanobot.session.manager import Session, SessionManager @@ -47,7 +56,11 @@ from nanobot.utils.helpers import image_placeholder_text from nanobot.utils.helpers import truncate_text as truncate_text_fn from nanobot.utils.image_generation_intent import image_generation_prompt from nanobot.utils.llm_runtime import LLMRuntime -from nanobot.utils.runtime import EMPTY_FINAL_RESPONSE_MESSAGE +from nanobot.utils.prompt_templates import _TEMPLATES_ROOT, render_template +from nanobot.utils.runtime import ( + EMPTY_FINAL_RESPONSE_MESSAGE, + SUSTAINED_GOAL_CONTINUE_PROMPT, +) if TYPE_CHECKING: from nanobot.config.schema import ( @@ -191,6 +204,7 @@ class AgentLoop: model_preset: str | None = None, preset_snapshot_loader: preset_helpers.PresetSnapshotLoader | None = None, runtime_model_publisher: Callable[[str, str | None], None] | None = None, + dream_model_override: str | None = None, ): from nanobot.config.schema import ToolsConfig @@ -203,6 +217,7 @@ class AgentLoop: self._preset_snapshot_loader = preset_snapshot_loader self._runtime_model_publisher = runtime_model_publisher self._provider_signature = provider_signature + self._dream_model_override = dream_model_override self._default_selection_signature = preset_helpers.default_selection_signature(provider_signature) self.workspace = workspace self.model = model or provider.get_default_model() @@ -310,6 +325,7 @@ class AgentLoop: self._active_preset: str | None = None if model_preset: self.set_model_preset(model_preset, publish_update=False) + self._configure_dream() self._register_default_tools() self._runtime_vars: dict[str, Any] = {} self._current_iteration: int = 0 @@ -369,6 +385,7 @@ class AgentLoop: model_preset=defaults.model_preset, provider_snapshot_loader=provider_snapshot_loader, preset_snapshot_loader=preset_snapshot_loader, + dream_model_override=config.agents.defaults.dream.model_override, **extra, ) @@ -394,7 +411,7 @@ class AgentLoop: self.runner.provider = provider self.subagents.set_provider(provider, model) self.consolidator.set_provider(provider, model, context_window_tokens) - self.dream.set_provider(provider, model) + self._configure_dream() self._provider_signature = snapshot.signature if publish_update and self._runtime_model_publisher is not None: self._runtime_model_publisher( @@ -403,6 +420,20 @@ class AgentLoop: ) logger.info("Runtime model switched for next turn: {} -> {}", old_model, model) + def _configure_dream(self) -> None: + """Apply dream.model_override, resolving preset names if needed.""" + if not self._dream_model_override: + self.dream.set_provider(self.provider, self.model) + return + + if self._dream_model_override in self.model_presets: + snapshot = self._build_model_preset_snapshot(self._dream_model_override) + self.dream.set_provider(snapshot.provider, snapshot.model) + return + + # Raw model name fallback — same provider, different model + self.dream.set_provider(self.provider, self._dream_model_override) + def _refresh_provider_snapshot(self) -> None: if self._provider_snapshot_loader is None: return @@ -1006,6 +1037,28 @@ class AgentLoop: msg.chat_id.split(":", 1) if ":" in msg.chat_id else ("cli", msg.chat_id) ) logger.info("Processing system message from {}", msg.sender_id) + if msg.sender_id == "dream": + session_key = "system:dream" + session = self.sessions.get_or_create(session_key) + session.metadata["is_dream"] = True + # Capture trigger source on first batch so _dream_finalize_commit + # can notify the user who ran /dream (cron-triggered runs have no trigger). + if "_dream_trigger_channel" not in session.metadata: + trigger_ch = msg.metadata.get("trigger_channel") + trigger_ci = msg.metadata.get("trigger_chat_id") + if trigger_ch and trigger_ci: + session.metadata["_dream_trigger_channel"] = trigger_ch + session.metadata["_dream_trigger_chat_id"] = trigger_ci + if not sustained_goal_active(session.metadata): + session.metadata[GOAL_STATE_KEY] = { + "status": "active", + "objective": "Dream: consolidate unprocessed memory backlog into MEMORY.md, SOUL.md, USER.md", + "started_at": datetime.now().isoformat(), + } + self.sessions.save(session) + await self._process_dream_batch(session, msg) + await self._dream_finalize_commit(session) + return None key = msg.session_key_override or f"{channel}:{chat_id}" session = self.sessions.get_or_create(key) if self._restore_runtime_checkpoint(session): @@ -1082,6 +1135,204 @@ class AgentLoop: metadata=outbound_metadata, ) + async def _process_dream_batch(self, session: Session, msg: InboundMessage) -> None: + """Process the full Dream backlog in batches within a single invocation.""" + from nanobot.agent.skills import BUILTIN_SKILLS_DIR + + # System prompt caching with mtime invalidation + template_path = _TEMPLATES_ROOT / "agent" / "dream.md" + cached_prompt = session.metadata.get("_dream_system_prompt") + cached_mtime = session.metadata.get("_dream_system_prompt_mtime") + current_mtime = template_path.stat().st_mtime if template_path.exists() else None + + if cached_prompt is None or cached_mtime != current_mtime: + skill_creator_path = BUILTIN_SKILLS_DIR / "skill-creator" / "SKILL.md" + workspace = self.dream.store.workspace + cached_prompt = render_template( + "agent/dream.md", + strip=True, + skill_creator_path=str(skill_creator_path), + soul_path=str(workspace / "SOUL.md"), + user_path=str(workspace / "USER.md"), + memory_path=str(workspace / "memory" / "MEMORY.md"), + stale_threshold_days=_STALE_THRESHOLD_DAYS, + ) + session.metadata["_dream_system_prompt"] = cached_prompt + session.metadata["_dream_system_prompt_mtime"] = current_mtime + + while True: + last_cursor = self.dream.store.get_last_dream_cursor() + entries = self.dream.store.read_unprocessed_history(since_cursor=last_cursor) + if not entries: + return + + batch = entries[: self.dream.max_batch_size] + logger.info( + "Dream: processing {}/{} entries (cursor {}→{})", + len(batch), len(entries), last_cursor, batch[-1]["cursor"], + ) + + # Build history text — cap each entry and strip [skip] lines + history_text = "\n".join( + f"[{e['timestamp']}] " + f"{truncate_text_fn(_strip_skip_lines(e['content']), self.dream._HISTORY_ENTRY_PREVIEW_MAX_CHARS)}" + for e in batch + ) + + # Current file contents + per-line age annotations + current_date = datetime.now().strftime("%Y-%m-%d") + annotate = self.dream.annotate_line_ages + raw_memory = self.dream.store.read_memory() or "(empty)" + raw_soul = self.dream.store.read_soul() or "(empty)" + raw_user = self.dream.store.read_user() or "(empty)" + annotated_memory = ( + self.dream._annotate_with_ages(raw_memory, "memory/MEMORY.md") + if annotate else raw_memory + ) + annotated_soul = ( + self.dream._annotate_with_ages(raw_soul, "SOUL.md") + if annotate else raw_soul + ) + annotated_user = ( + self.dream._annotate_with_ages(raw_user, "USER.md") + if annotate else raw_user + ) + current_memory = truncate_text_fn(annotated_memory, self.dream._MEMORY_FILE_MAX_CHARS) + current_soul = truncate_text_fn(annotated_soul, self.dream._SOUL_FILE_MAX_CHARS) + current_user = truncate_text_fn(annotated_user, self.dream._USER_FILE_MAX_CHARS) + + file_context = ( + f"## Current Date\n{current_date}\n\n" + f"## Current MEMORY.md ({len(current_memory)} chars)\n{current_memory}\n\n" + f"## Current SOUL.md ({len(current_soul)} chars)\n{current_soul}\n\n" + f"## Current USER.md ({len(current_user)} chars)\n{current_user}" + ) + + existing_skills = self.dream._list_existing_skills() + skills_section = "" + if existing_skills: + skills_section = ( + "\n\n## Existing Skills\n" + + "\n".join(f"- {s}" for s in existing_skills) + ) + + user_prompt = f"## Conversation History\n{history_text}\n\n{file_context}{skills_section}" + logger.info("Dream prompt: {} chars, ~{} tokens", len(user_prompt), _estimate_tokens(user_prompt)) + + messages: list[dict[str, Any]] = [ + {"role": "system", "content": cached_prompt}, + {"role": "user", "content": user_prompt}, + ] + + t_start = time.perf_counter() + try: + result = await self.dream._runner.run(AgentRunSpec( + initial_messages=messages, + tools=self.dream._tools, + model=self.dream.model, + max_iterations=self.dream.max_iterations, + max_tool_result_chars=self.dream.max_tool_result_chars, + context_window_tokens=self.context_window_tokens, + fail_on_tool_error=False, + )) + elapsed = time.perf_counter() - t_start + logger.info( + "Dream run complete in {:.1f}s: stop_reason={}, tool_events={}", + elapsed, result.stop_reason, len(result.tool_events), + ) + except Exception: + elapsed = time.perf_counter() - t_start + logger.exception("Dream run failed after {:.1f}s", elapsed) + result = None + + # Build changelog from tool events + changelog: list[str] = [] + if result and result.tool_events: + for event in result.tool_events: + if event.get("status") == "ok": + changelog.append(f"{event['name']}: {event['detail']}") + + success = result is not None and result.stop_reason == "completed" + if success: + new_cursor = batch[-1]["cursor"] + self.dream.store.set_last_dream_cursor(new_cursor) + session.metadata.setdefault("_dream_changelog", []).extend(changelog) + self.sessions.save(session) + logger.info( + "Dream done: {} change(s), cursor advanced to {}", + len(changelog), new_cursor, + ) + else: + reason = result.stop_reason if result else "exception" + logger.warning( + "Dream incomplete ({}): cursor NOT advanced, stopping", + reason, + ) + return + + self.dream.store.compact_history() + + # Persist session record for debugging / visualization + record = { + "timestamp": datetime.now().isoformat(), + "batch": { + "from_cursor": last_cursor, + "to_cursor": batch[-1]["cursor"], + "count": len(batch), + }, + "prompt_chars": len(user_prompt), + "elapsed_seconds": elapsed, + "stop_reason": result.stop_reason, + "usage": result.usage, + "tool_events": result.tool_events, + "changelog": changelog, + "commit_sha": None, + "messages": result.messages, + } + self.dream.store.write_dream_session(record) + session.metadata["_dream_last_record"] = record + + + async def _dream_finalize_commit(self, session: Session) -> None: + """Collapse accumulated changelog into a single git commit, clear caches, and complete the goal.""" + changelog = session.metadata.pop("_dream_changelog", []) + sha = None + if changelog and self.dream.store.git.is_initialized(): + ts = datetime.now().strftime("%Y-%m-%d %H:%M") + summary = f"dream: {ts}, {len(changelog)} change(s)" + commit_msg = f"{summary}\n\n" + "\n".join(changelog) + sha = self.dream.store.git.auto_commit(commit_msg) + if sha: + logger.info("Dream commit: {}", sha) + record = session.metadata.pop("_dream_last_record", None) + if record and sha: + record["commit_sha"] = sha + self.dream.store.write_dream_session(record) + session.metadata.pop("_dream_system_prompt", None) + session.metadata.pop("_dream_system_prompt_mtime", None) + trigger_channel = session.metadata.pop("_dream_trigger_channel", None) + trigger_chat_id = session.metadata.pop("_dream_trigger_chat_id", None) + goal = session.metadata.get(GOAL_STATE_KEY) + if isinstance(goal, dict) and goal.get("status") == "active": + session.metadata[GOAL_STATE_KEY] = { + **goal, + "status": "completed", + "completed_at": datetime.now().isoformat(), + "recap": f"Memory backlog consolidated ({len(changelog)} change(s)).", + } + self.sessions.save(session) + session.metadata["_dream_finalized"] = True + # Notify the user who triggered /dream + if trigger_channel and trigger_chat_id: + content = f"Dream completed: {len(changelog)} change(s) committed." + if not changelog: + content = "Dream: nothing to process." + await self.bus.publish_outbound(OutboundMessage( + channel=trigger_channel, + chat_id=trigger_chat_id, + content=content, + )) + async def _process_message( self, msg: InboundMessage, diff --git a/nanobot/agent/memory.py b/nanobot/agent/memory.py index ffc9c5f0e..b88fff786 100644 --- a/nanobot/agent/memory.py +++ b/nanobot/agent/memory.py @@ -6,6 +6,7 @@ import asyncio import json import os import re +import time import weakref from contextlib import suppress from datetime import datetime @@ -15,7 +16,7 @@ from typing import TYPE_CHECKING, Any, Callable, Iterator import tiktoken from loguru import logger -from nanobot.agent.runner import AgentRunner, AgentRunSpec +from nanobot.agent.runner import AgentRunner from nanobot.agent.tools.registry import ToolRegistry from nanobot.session.manager import Session from nanobot.utils.gitstore import GitStore @@ -33,6 +34,20 @@ if TYPE_CHECKING: from nanobot.providers.base import LLMProvider from nanobot.session.manager import SessionManager +# Cache the tiktoken encoding to avoid repeated instantiation on every +# truncate/encode call. Encoding objects are thread-safe and reusable. +try: + _TIKTOKEN_ENC = tiktoken.get_encoding("cl100k_base") +except Exception: # pragma: no cover + _TIKTOKEN_ENC = None + + +def _estimate_tokens(text: str) -> int: + """Approximate token count for a text string.""" + if _TIKTOKEN_ENC is not None: + return len(_TIKTOKEN_ENC.encode(text)) + return len(text) // 4 + # --------------------------------------------------------------------------- # MemoryStore — pure file I/O layer @@ -400,6 +415,26 @@ class MemoryStore: def set_last_dream_cursor(self, cursor: int) -> None: self._dream_cursor_file.write_text(str(cursor), encoding="utf-8") + def write_dream_session(self, data: dict[str, Any]) -> None: + """Atomic overwrite of the latest Dream run record.""" + path = self.memory_dir / ".dream_session.json" + tmp_path = path.with_suffix(".tmp") + try: + with open(tmp_path, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False, indent=2) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp_path, path) + with suppress(PermissionError): + fd = os.open(str(path.parent), os.O_RDONLY) + try: + os.fsync(fd) + finally: + os.close(fd) + except BaseException: + tmp_path.unlink(missing_ok=True) + raise + # -- message formatting utility ------------------------------------------ @staticmethod @@ -618,19 +653,21 @@ class Consolidator: """Available input token budget for consolidation LLM.""" return self.context_window_tokens - self.max_completion_tokens - self._SAFETY_BUFFER - def _truncate_to_token_budget(self, text: str) -> str: - """Truncate text so it fits within the consolidation LLM's token budget.""" - budget = self._input_token_budget + def _truncate_to_token_budget(self, text: str, reserve_tokens: int = 0) -> str: + """Truncate text so it fits within the consolidation LLM's token budget. + + reserve_tokens: additional tokens to reserve for dedup context or other + overhead that will be appended after truncation. + """ + budget = self._input_token_budget - reserve_tokens if budget <= 0: return truncate_text(text, _RAW_ARCHIVE_MAX_CHARS) - try: - enc = tiktoken.get_encoding("cl100k_base") - tokens = enc.encode(text) + if _TIKTOKEN_ENC is not None: + tokens = _TIKTOKEN_ENC.encode(text) if len(tokens) <= budget: return text - return enc.decode(tokens[:budget]) + "\n... (truncated)" - except Exception: - return truncate_text(text, budget * 4) + return _TIKTOKEN_ENC.decode(tokens[:budget]) + "\n... (truncated)" + return truncate_text(text, budget * 4) async def archive(self, messages: list[dict]) -> str | None: """Summarize messages via LLM and append to history.jsonl. @@ -639,9 +676,53 @@ class Consolidator: """ if not messages: return None + t_start = time.perf_counter() try: formatted = MemoryStore._format_messages(messages) - formatted = self._truncate_to_token_budget(formatted) + logger.debug( + "Consolidator: {} messages, formatted={} chars", + len(messages), len(formatted), + ) + + # Inject current memory context for dedup-aware summarization. + memory_preview = self.store.read_memory()[:4000] + user_preview = self.store.read_user()[:2000] + dedup_context = "" + if memory_preview: + dedup_context += f"\n\n## Current MEMORY.md (for dedup)\n{memory_preview}" + if user_preview: + dedup_context += f"\n\n## Current USER.md (for dedup)\n{user_preview}" + + reserve_tokens = 0 + if dedup_context: + if _TIKTOKEN_ENC is not None: + reserve_tokens = len(_TIKTOKEN_ENC.encode(dedup_context)) + 100 + else: + reserve_tokens = len(dedup_context) // 4 + 100 + + if self._input_token_budget <= reserve_tokens: + logger.warning( + "Consolidator: dedup_context ({} tokens) exceeds budget ({}), dropping it", + reserve_tokens, self._input_token_budget, + ) + dedup_context = "" + reserve_tokens = 0 + else: + logger.debug( + "Consolidator: dedup_context={} chars, reserve_tokens={}", + len(dedup_context), reserve_tokens, + ) + + formatted_before = len(formatted) + formatted = self._truncate_to_token_budget( + formatted, reserve_tokens=reserve_tokens + ) + if len(formatted) < formatted_before: + logger.warning( + "Consolidator: truncated formatted messages from {} to {} chars", + formatted_before, len(formatted), + ) + response = await self.provider.chat_with_retry( model=self.model, messages=[ @@ -652,18 +733,31 @@ class Consolidator: strip=True, ), }, - {"role": "user", "content": formatted}, + {"role": "user", "content": formatted + dedup_context}, ], tools=None, tool_choice=None, ) + elapsed = time.perf_counter() - t_start if response.finish_reason == "error": + logger.warning( + "Consolidator LLM error after {:.1f}s: {}", + elapsed, response.content, + ) raise RuntimeError(f"LLM returned error: {response.content}") summary = response.content or "[no summary]" + logger.info( + "Consolidator: {} entries -> {} chars summary in {:.1f}s", + len(messages), len(summary), elapsed, + ) self.store.append_history(summary, max_chars=_ARCHIVE_SUMMARY_MAX_CHARS) return summary except Exception: - logger.warning("Consolidation LLM call failed, raw-dumping to history") + elapsed = time.perf_counter() - t_start + logger.warning( + "Consolidation LLM call failed after {:.1f}s, raw-dumping to history", + elapsed, + ) self.store.raw_archive(messages) return None @@ -851,35 +945,44 @@ class Consolidator: # Single source of truth for the staleness threshold used in _annotate_with_ages -# *and* in the Phase 1 prompt template (passed as `stale_threshold_days`). +# *and* in the system prompt template (passed as `stale_threshold_days`). # Keep code and prompt aligned — if you bump this, the LLM's instruction string # updates automatically. _STALE_THRESHOLD_DAYS = 14 +_SKIP_LINE_RE = re.compile(r"^\s*-\s*\[skip\]\s*.*$", re.MULTILINE | re.IGNORECASE) + + +def _strip_skip_lines(text: str) -> str: + """Remove lines marked [skip] from history content.""" + lines = text.splitlines() + kept = [line for line in lines if not _SKIP_LINE_RE.match(line)] + return "\n".join(kept) + class Dream: - """Two-phase memory processor: analyze history.jsonl, then edit files via AgentRunner. + """Single-phase memory processor: analyze history.jsonl and edit files via AgentRunner. - Phase 1 produces an analysis summary (plain LLM call). - Phase 2 delegates to AgentRunner with read_file / edit_file tools so the - LLM can make targeted, incremental edits instead of replacing entire files. + Delegates to AgentRunner with read_file / edit_file tools so the LLM can + analyze conversation history, extract facts, deduplicate, and make targeted + incremental edits — all in a single agent run. """ # Caps on prompt-bound inputs so Dream's LLM calls never exceed the model's # context window just because a file (or a legacy large history entry) grew # unexpectedly. Each file still appears in full via read_file when the agent - # needs it in Phase 2 — these caps only bound the Phase 1/2 prompt preview. - _MEMORY_FILE_MAX_CHARS = 32_000 - _SOUL_FILE_MAX_CHARS = 16_000 - _USER_FILE_MAX_CHARS = 16_000 - _HISTORY_ENTRY_PREVIEW_MAX_CHARS = 4_000 + # needs it — these caps only bound the prompt preview. + _MEMORY_FILE_MAX_CHARS = 16_000 + _SOUL_FILE_MAX_CHARS = 4_000 + _USER_FILE_MAX_CHARS = 4_000 + _HISTORY_ENTRY_PREVIEW_MAX_CHARS = 2_000 def __init__( self, store: MemoryStore, provider: LLMProvider, model: str, - max_batch_size: int = 20, + max_batch_size: int = 5, max_iterations: int = 10, max_tool_result_chars: int = 16_000, annotate_line_ages: bool = True, @@ -890,9 +993,9 @@ class Dream: self.max_batch_size = max_batch_size self.max_iterations = max_iterations self.max_tool_result_chars = max_tool_result_chars - # Kill switch for the git-blame-based per-line age annotation in Phase 1. - # Default True keeps the #3212 behavior; set False to feed MEMORY.md raw - # (e.g. if a specific LLM reacts poorly to the `← Nd` suffix). + # Kill switch for the git-blame-based per-line age annotation in the prompt. + # Default True keeps the #3212 behavior; set False to feed all memory + # files raw (e.g. if a specific LLM reacts poorly to the `← Nd` suffix). self.annotate_line_ages = annotate_line_ages self._runner = AgentRunner(provider) self._tools = self._build_tools() @@ -907,6 +1010,7 @@ class Dream: def _build_tools(self) -> ToolRegistry: """Build a minimal tool registry for the Dream agent.""" from nanobot.agent.skills import BUILTIN_SKILLS_DIR + from nanobot.agent.tools.apply_patch import ApplyPatchTool from nanobot.agent.tools.file_state import FileStates from nanobot.agent.tools.filesystem import EditFileTool, ReadFileTool, WriteFileTool @@ -924,6 +1028,7 @@ class Dream: file_states=file_states, )) tools.register(EditFileTool(workspace=workspace, allowed_dir=workspace, file_states=file_states)) + tools.register(ApplyPatchTool(workspace=workspace, allowed_dir=workspace, file_states=file_states)) # write_file resolves relative paths from workspace root, but can only # write under skills/ so the prompt can safely use skills//SKILL.md. skills_dir = workspace / "skills" @@ -961,8 +1066,8 @@ class Dream: # -- main entry ---------------------------------------------------------- - def _annotate_with_ages(self, content: str) -> str: - """Append per-line age suffixes to MEMORY.md content. + def _annotate_with_ages(self, content: str, file_path: str = "memory/MEMORY.md") -> str: + """Append per-line age suffixes to file content. Each non-blank line whose age exceeds ``_STALE_THRESHOLD_DAYS`` gets a suffix like ``← 30d`` indicating days since last modification. @@ -970,9 +1075,7 @@ class Dream: annotate fails, or the line count doesn't match the age count (which can happen with an uncommitted working-tree edit — better to skip annotation than to tag the wrong line). - SOUL.md and USER.md are never annotated. """ - file_path = "memory/MEMORY.md" try: ages = self.store.git.line_ages(file_path) except Exception: @@ -1007,156 +1110,3 @@ class Dream: result += "\n" return result - async def run(self) -> bool: - """Process unprocessed history entries. Returns True if work was done.""" - from nanobot.agent.skills import BUILTIN_SKILLS_DIR - - last_cursor = self.store.get_last_dream_cursor() - entries = self.store.read_unprocessed_history(since_cursor=last_cursor) - if not entries: - return False - - batch = entries[: self.max_batch_size] - logger.info( - "Dream: processing {} entries (cursor {}→{}), batch={}", - len(entries), last_cursor, batch[-1]["cursor"], len(batch), - ) - - # Build history text for LLM — cap each entry so a legacy oversized - # record (e.g. pre-#3412 raw_archive dump) can't blow up the prompt. - history_text = "\n".join( - f"[{e['timestamp']}] " - f"{truncate_text(e['content'], self._HISTORY_ENTRY_PREVIEW_MAX_CHARS)}" - for e in batch - ) - - # Current file contents + per-line age annotations (MEMORY.md only). - # Each file is capped in the *prompt preview* only; Phase 2 still sees - # the full file via the read_file tool. - current_date = datetime.now().strftime("%Y-%m-%d") - raw_memory = self.store.read_memory() or "(empty)" - annotated_memory = ( - self._annotate_with_ages(raw_memory) - if self.annotate_line_ages - else raw_memory - ) - current_memory = truncate_text(annotated_memory, self._MEMORY_FILE_MAX_CHARS) - current_soul = truncate_text( - self.store.read_soul() or "(empty)", self._SOUL_FILE_MAX_CHARS, - ) - current_user = truncate_text( - self.store.read_user() or "(empty)", self._USER_FILE_MAX_CHARS, - ) - - file_context = ( - f"## Current Date\n{current_date}\n\n" - f"## Current MEMORY.md ({len(current_memory)} chars)\n{current_memory}\n\n" - f"## Current SOUL.md ({len(current_soul)} chars)\n{current_soul}\n\n" - f"## Current USER.md ({len(current_user)} chars)\n{current_user}" - ) - - # Phase 1: Analyze (no skills list — dedup is Phase 2's job) - phase1_prompt = ( - f"## Conversation History\n{history_text}\n\n{file_context}" - ) - - try: - phase1_response = await self.provider.chat_with_retry( - model=self.model, - messages=[ - { - "role": "system", - "content": render_template( - "agent/dream_phase1.md", - strip=True, - stale_threshold_days=_STALE_THRESHOLD_DAYS, - ), - }, - {"role": "user", "content": phase1_prompt}, - ], - tools=None, - tool_choice=None, - ) - analysis = phase1_response.content or "" - logger.debug("Dream Phase 1 analysis ({} chars): {}", len(analysis), analysis[:500]) - except Exception: - logger.exception("Dream Phase 1 failed") - return False - - # Phase 2: Delegate to AgentRunner with read_file / edit_file - existing_skills = self._list_existing_skills() - skills_section = "" - if existing_skills: - skills_section = ( - "\n\n## Existing Skills\n" - + "\n".join(f"- {s}" for s in existing_skills) - ) - phase2_prompt = f"## Analysis Result\n{analysis}\n\n{file_context}{skills_section}" - - tools = self._tools - skill_creator_path = BUILTIN_SKILLS_DIR / "skill-creator" / "SKILL.md" - messages: list[dict[str, Any]] = [ - { - "role": "system", - "content": render_template( - "agent/dream_phase2.md", - strip=True, - skill_creator_path=str(skill_creator_path), - ), - }, - {"role": "user", "content": phase2_prompt}, - ] - - try: - result = await self._runner.run(AgentRunSpec( - initial_messages=messages, - tools=tools, - model=self.model, - max_iterations=self.max_iterations, - max_tool_result_chars=self.max_tool_result_chars, - fail_on_tool_error=False, - )) - logger.debug( - "Dream Phase 2 complete: stop_reason={}, tool_events={}", - result.stop_reason, len(result.tool_events), - ) - for ev in (result.tool_events or []): - logger.info("Dream tool_event: name={}, status={}, detail={}", ev.get("name"), ev.get("status"), ev.get("detail", "")[:200]) - except Exception: - logger.exception("Dream Phase 2 failed") - result = None - - # Build changelog from tool events - changelog: list[str] = [] - if result and result.tool_events: - for event in result.tool_events: - if event["status"] == "ok": - changelog.append(f"{event['name']}: {event['detail']}") - - # Only advance cursor on successful completion to prevent silent loss - if result and result.stop_reason == "completed": - new_cursor = batch[-1]["cursor"] - self.store.set_last_dream_cursor(new_cursor) - logger.info( - "Dream done: {} change(s), cursor advanced to {}", - len(changelog), new_cursor, - ) - else: - reason = result.stop_reason if result else "exception" - logger.warning( - "Dream incomplete ({}): cursor NOT advanced, will retry next cron cycle", - reason, - ) - - self.store.compact_history() - - # Git auto-commit (only when there are actual changes) - if changelog and self.store.git.is_initialized(): - ts = batch[-1]["timestamp"] - summary = f"dream: {ts}, {len(changelog)} change(s)" - commit_msg = f"{summary}\n\n{analysis.strip()}" - sha = self.store.git.auto_commit(commit_msg) - if sha: - logger.info("Dream commit: {}", sha) - - return True diff --git a/nanobot/cli/commands.py b/nanobot/cli/commands.py index f7bf043a4..2a3d4f484 100644 --- a/nanobot/cli/commands.py +++ b/nanobot/cli/commands.py @@ -762,7 +762,7 @@ def _run_gateway( ) from nanobot.agent.loop import UNIFIED_SESSION_KEY - from nanobot.bus.events import OutboundMessage + from nanobot.bus.events import InboundMessage, OutboundMessage def _channel_session_key(channel: str, chat_id: str) -> str: return ( @@ -810,13 +810,13 @@ def _run_gateway( # Set cron callback (needs agent) async def on_cron_job(job: CronJob) -> str | None: """Execute a cron job through the agent.""" - # Dream is an internal job — run directly, not through the agent loop. if job.name == "dream": - try: - await agent.dream.run() - logger.info("Dream cron job completed") - except Exception: - logger.exception("Dream cron job failed") + await bus.publish_inbound(InboundMessage( + channel="system", + sender_id="dream", + chat_id="dream", + content="", + )) return None from nanobot.utils.evaluator import evaluate_response @@ -1027,8 +1027,6 @@ def _run_gateway( await server.serve_forever() # Register Dream system job (always-on, idempotent on restart) dream_cfg = config.agents.defaults.dream - if dream_cfg.model_override: - agent.dream.model = dream_cfg.model_override agent.dream.max_batch_size = dream_cfg.max_batch_size agent.dream.max_iterations = dream_cfg.max_iterations agent.dream.annotate_line_ages = dream_cfg.annotate_line_ages diff --git a/nanobot/command/builtin.py b/nanobot/command/builtin.py index 4646df38a..95b4791c5 100644 --- a/nanobot/command/builtin.py +++ b/nanobot/command/builtin.py @@ -299,30 +299,22 @@ async def cmd_model(ctx: CommandContext) -> OutboundMessage: async def cmd_dream(ctx: CommandContext) -> OutboundMessage: """Manually trigger a Dream consolidation run.""" - import time + from nanobot.bus.events import InboundMessage - loop = ctx.loop - msg = ctx.msg - - async def _run_dream(): - t0 = time.monotonic() - try: - did_work = await loop.dream.run() - elapsed = time.monotonic() - t0 - if did_work: - content = f"Dream completed in {elapsed:.1f}s." - else: - content = "Dream: nothing to process." - except Exception as e: - elapsed = time.monotonic() - t0 - content = f"Dream failed after {elapsed:.1f}s: {e}" - await loop.bus.publish_outbound(OutboundMessage( - channel=msg.channel, chat_id=msg.chat_id, content=content, - )) - - asyncio.create_task(_run_dream()) + await ctx.loop.bus.publish_inbound(InboundMessage( + channel="system", + sender_id="dream", + chat_id="dream", + content="", + metadata={ + "trigger_channel": ctx.msg.channel, + "trigger_chat_id": ctx.msg.chat_id, + }, + )) return OutboundMessage( - channel=msg.channel, chat_id=msg.chat_id, content="Dreaming...", + channel=ctx.msg.channel, + chat_id=ctx.msg.chat_id, + content="Dream started. It will process memory backlog and report when done.", ) @@ -355,6 +347,18 @@ def _format_changed_files(diff: str) -> str: def _format_dream_log_content(commit, diff: str, *, requested_sha: str | None = None) -> str: files_line = _format_changed_files(diff) + msg_lines = commit.message.splitlines() if commit.message else [] + msg_summary = msg_lines[0] if msg_lines else "" + msg_body = [] + in_body = False + for line in msg_lines[1:]: + if not in_body: + if not line: + in_body = True + continue + msg_body.append(line) + body_text = "\n".join(msg_body).strip() + lines = [ "## Dream Update", "", @@ -362,8 +366,12 @@ def _format_dream_log_content(commit, diff: str, *, requested_sha: str | None = "", f"- Commit: `{commit.sha}`", f"- Time: {commit.timestamp}", - f"- Changed files: {files_line}", ] + if msg_summary: + lines.append(f"- Summary: {msg_summary}") + lines.append(f"- Changed files: {files_line}") + if body_text: + lines.extend(["", "### Analysis", "", body_text]) if diff: lines.extend([ "", @@ -389,7 +397,8 @@ def _format_dream_restore_list(commits: list) -> str: "", ] for c in commits: - lines.append(f"- `{c.sha}` {c.timestamp} - {c.message.splitlines()[0]}") + summary = c.message.splitlines()[0] if c.message else "(no message)" + lines.append(f"- `{c.sha}` {c.timestamp} - {summary}") lines.extend([ "", "Preview a version with `/dream-log ` before restoring it.", diff --git a/nanobot/config/schema.py b/nanobot/config/schema.py index f194820cc..e5aed3494 100644 --- a/nanobot/config/schema.py +++ b/nanobot/config/schema.py @@ -52,13 +52,12 @@ class DreamConfig(Base): model_override: str | None = Field( default=None, validation_alias=AliasChoices("modelOverride", "model", "model_override"), - ) # Optional Dream-specific model override - max_batch_size: int = Field(default=20, ge=1) # Max history entries per run - # Bumped from 10 to 15 in #3212 (exp002: +30% dedup, no accuracy loss; >15 plateaus). - max_iterations: int = Field(default=15, ge=1) # Max tool calls per Phase 2 - # Per-line git-blame age annotation in Phase 1 prompt (see #3212). Default - # on — set to False to feed MEMORY.md raw if a specific LLM reacts poorly - # to the `← Nd` suffix or you want deterministic, git-independent prompts. + ) # Optional Dream-specific model override. Supports preset names (resolved against model_presets) or raw model identifiers. + max_batch_size: int = Field(default=5, ge=1) # Max history entries per run + max_iterations: int = Field(default=15, ge=1) # Max tool calls per Dream run + # Per-line git-blame age annotation in the Dream prompt (see #3212). Default + # on — set to False to feed all memory files raw if a specific LLM reacts + # poorly to the `← Nd` suffix or you want deterministic, git-independent prompts. annotate_line_ages: bool = True def build_schedule(self, timezone: str) -> CronSchedule: diff --git a/nanobot/skills/memory/SKILL.md b/nanobot/skills/memory/SKILL.md index 042ef80ca..837102459 100644 --- a/nanobot/skills/memory/SKILL.md +++ b/nanobot/skills/memory/SKILL.md @@ -34,3 +34,5 @@ Examples (replace `keyword`): - **Do NOT edit SOUL.md, USER.md, or MEMORY.md.** They are automatically managed by Dream. - If you notice outdated information, it will be corrected when Dream runs next. - Users can view Dream's activity with the `/dream-log` command. +- Dream runs as a `system` session inside the AgentLoop, triggered by the `/dream` command or cron. Each turn processes one batch; if backlog remains, Dream automatically chains additional turns until complete. All changes are committed in a single git commit. +- Dream can use a different model than the main agent via `agents.defaults.dream.modelOverride`. Supports preset names or raw model identifiers. diff --git a/nanobot/templates/agent/consolidator_archive.md b/nanobot/templates/agent/consolidator_archive.md index 5073f4f44..8181a2e19 100644 --- a/nanobot/templates/agent/consolidator_archive.md +++ b/nanobot/templates/agent/consolidator_archive.md @@ -1,13 +1,27 @@ -Extract key facts from this conversation. Only output items matching these categories, skip everything else: -- User facts: personal info, preferences, stated opinions, habits -- Decisions: choices made, conclusions reached -- Solutions: working approaches discovered through trial and error, especially non-obvious methods that succeeded after failed attempts -- Events: plans, deadlines, notable occurrences -- Preferences: communication style, tool preferences +Extract key facts from this conversation. For each fact, annotate its memory attributes. -Priority: user corrections and preferences > solutions > decisions > events > environment facts. The most valuable memory prevents the user from having to repeat themselves. +Only SNIP facts deserve a non-[skip] mark: +- Signal: would the user need to repeat this if forgotten? +- Novel: not already in MEMORY.md or USER.md (check context below) +- Important: prevents rework or captures preferences / rules +- Persistent: still relevant after 2 weeks -Skip: code patterns derivable from source, git history, or anything already captured in existing memory. +Output one fact per line in this format: +- [mark] fact content + +Marks (choose the best match): +- [permanent] Core preferences, personal traits, habits — never becomes stale +- [durable] Technical discoveries, project knowledge, config details — valid for months +- [ephemeral] Active task state, temporary decisions — may change in weeks +- [correction] Correction to a previous memory — must state what it replaces +- [skip] Does not meet SNIP criteria — still written to history.jsonl for audit, but Dream will ignore it + +Categories to capture: people/roles, decisions/rationale, solutions, events/dates, preferences. +Decisions must include their motivation. +Write densely. Prefer 'X=A, Y=B' over separate bullets for tightly coupled facts. +Priority: user corrections > decisions with rationale > solutions > specific events > general context. +Output in the same language as the input conversation. +CRITICAL: Never drop person names, team names, or project names. +Skip: code patterns derivable from source, git history, or anything already in existing memory. -Output as concise bullet points, one fact per line. No preamble, no commentary. If nothing noteworthy happened, output: (nothing) diff --git a/nanobot/templates/agent/dream.md b/nanobot/templates/agent/dream.md new file mode 100644 index 000000000..eeb004bc6 --- /dev/null +++ b/nanobot/templates/agent/dream.md @@ -0,0 +1,58 @@ +Update memory files by analyzing conversation history and editing files directly. +Prune before adding — removing stale content is as important as adding new facts. + +## File routing +Do NOT guess paths. Route each fact to its canonical file: + +| File | Full path | Content | +|------|------|---------| +| SOUL.md | `{{ soul_path }}` | Agent behavior, guardrails, tone, interaction patterns | +| USER.md | `{{ user_path }}` | Personal info, preferences, habits, work context, communication style | +| MEMORY.md | `{{ memory_path }}` | Technical knowledge, project context, infrastructure, accounts | +| SKILL.md | `skills//SKILL.md` | Reusable workflow templates ([SKILL] entries only) | + +Cross-boundary rule: no technical configs in USER.md, no user facts in SOUL.md, no preferences in MEMORY.md. If a fact fits multiple files, keep the most specific copy and remove the rest. + +## Delete-or-keep + +**Always delete:** +- Same fact at multiple locations — keep canonical copy only +- Merged/closed PR notes, resolved incidents, superseded info +- Verbose entries restatable in fewer words +- Overlapping or nested sections covering the same topic + +**Likely delete** (apply judgment): +- Same fact at different detail levels — keep most complete version only +- Debugging steps unlikely to recur +- Ephemeral facts past their useful life +- Tool/service details documented upstream +- Lines with ``← Nd`` where N>{{ stale_threshold_days }} — closer review, not automatic removal + +**Never delete:** +- User preferences and personality traits (permanent regardless of age) +- Active project context still referenced in conversations +- Behavioral rules in SOUL.md + +When removing: prefer deleting individual items over entire sections. + +## Fact extraction +- Atomic facts: "has a cat named Luna" not "discussed pet care" +- Corrections: edit the existing entry, don't append a new one +- Capture confirmed approaches the user validated + +## Skill discovery & creation +Flag [SKILL] only when ALL are true: repeatable workflow appeared 2+ times, involves clear steps (not vague preferences), substantial enough for its own instruction set. Check existing skills to avoid redundancy. + +For [SKILL] entries: +- Use write_file to create skills//SKILL.md; read_file `{{ skill_creator_path }}` for format reference +- YAML frontmatter (name, description), under 2000 words: when to use, steps, output format, example +- Do NOT overwrite existing skills — if overlapping, merge delta into the existing skill +- Skills are instruction sets, not code. Keep concrete values in MEMORY.md; skills use placeholders + +## Editing +- Default tool: apply_patch. Use edit_file only for small exact replacements. +- File contents provided below — no read_file needed for initial edits. +- Batch all changes into a single apply_patch call. Surgical edits only. +- dry_run=true to preview. If nothing to update, stop without calling tools. + +Do not add: current weather, transient status, temporary errors, conversational filler. diff --git a/nanobot/templates/agent/dream_phase1.md b/nanobot/templates/agent/dream_phase1.md deleted file mode 100644 index 114db38c5..000000000 --- a/nanobot/templates/agent/dream_phase1.md +++ /dev/null @@ -1,40 +0,0 @@ -You have TWO equally important tasks: -1. Extract new facts from conversation history -2. Deduplicate existing memory files — find and flag redundant, overlapping, or stale content even if NOT mentioned in history - -Output one line per finding: -[FILE] atomic fact (not already in memory) -[FILE-REMOVE] reason for removal -[SKILL] kebab-case-name: one-line description of the reusable pattern - -Files: USER (identity, preferences), SOUL (bot behavior, tone), MEMORY (knowledge, project context) - -Rules: -- Atomic facts: "has a cat named Luna" not "discussed pet care" -- Corrections: [USER] location is Tokyo, not Osaka -- Capture confirmed approaches the user validated - -Deduplication — scan ALL memory files for these redundancy patterns: -- Same fact stated in multiple places (e.g., "communicates in Chinese" in both USER.md and multiple MEMORY.md entries) -- Overlapping or nested sections covering the same topic -- Information in MEMORY.md that is already captured in USER.md or SOUL.md (MEMORY.md should not duplicate permanent-file content) -- Verbose entries that can be condensed without losing information -For each duplicate found, output [FILE-REMOVE] for the less authoritative copy (prefer keeping facts in their canonical location) - -Staleness — MEMORY.md lines may have a ``← Nd`` suffix showing days since last modification: -- SOUL.md and USER.md have no age annotations — they are permanent, only update with corrections -- Age only indicates when content was last touched, not whether it should be removed -- Use content judgment: user habits/preferences/personality traits are permanent regardless of age -- Only prune content that is objectively outdated: passed events, resolved tracking, superseded approaches -- Lines with ``← Nd`` (N>{{ stale_threshold_days }}) deserve closer review but are NOT automatically removable -- When removing: prefer deleting individual items over entire sections - -Skill discovery — flag [SKILL] when ALL of these are true: -- A specific, repeatable workflow appeared 2+ times in the conversation history -- It involves clear steps (not vague preferences like "likes concise answers") -- It is substantial enough to warrant its own instruction set (not trivial like "read a file") -- Do not worry about duplicates — the next phase will check against existing skills - -Do not add: current weather, transient status, temporary errors, conversational filler. - -[SKIP] if nothing needs updating. diff --git a/nanobot/templates/agent/dream_phase2.md b/nanobot/templates/agent/dream_phase2.md deleted file mode 100644 index f833afb6a..000000000 --- a/nanobot/templates/agent/dream_phase2.md +++ /dev/null @@ -1,37 +0,0 @@ -Update memory files based on the analysis below. -- [FILE] entries: add the described content to the appropriate file -- [FILE-REMOVE] entries: delete the corresponding content from memory files -- [SKILL] entries: create a new skill under skills//SKILL.md using write_file - -## File paths (relative to workspace root) -- SOUL.md -- USER.md -- memory/MEMORY.md -- skills//SKILL.md (for [SKILL] entries only) - -Do NOT guess paths. - -## Editing rules -- Edit directly — file contents provided below, no read_file needed -- Use exact text as old_text, include surrounding blank lines for unique match -- Batch changes to the same file into one edit_file call -- For deletions: section header + all bullets as old_text, new_text empty -- Surgical edits only — never rewrite entire files -- If nothing to update, stop without calling tools - -## Skill creation rules (for [SKILL] entries) -- Use write_file to create skills//SKILL.md -- Before writing, read_file `{{ skill_creator_path }}` for format reference (frontmatter structure, naming conventions, quality standards) -- **Dedup check**: read existing skills listed below to verify the new skill is not functionally redundant. Skip creation if an existing skill already covers the same workflow. -- Include YAML frontmatter with name and description fields -- Keep SKILL.md under 2000 words — concise and actionable -- Include: when to use, steps, output format, at least one example -- Do NOT overwrite existing skills — skip if the skill directory already exists -- Reference specific tools the agent has access to (read_file, write_file, exec, web_search, etc.) -- Skills are instruction sets, not code — do not include implementation code - -## Quality -- Every line must carry standalone value -- Concise bullets under clear headers -- When reducing (not deleting): keep essential facts, drop verbose details -- If uncertain whether to delete, keep but add "(verify currency)" diff --git a/nanobot/utils/gitstore.py b/nanobot/utils/gitstore.py index 6e05ca128..b1c5c7bed 100644 --- a/nanobot/utils/gitstore.py +++ b/nanobot/utils/gitstore.py @@ -19,7 +19,8 @@ class CommitInfo: def format(self, diff: str = "") -> str: """Format this commit for display, optionally with a diff.""" - header = f"## {self.message.splitlines()[0]}\n`{self.sha}` — {self.timestamp}\n" + summary = self.message.splitlines()[0] if self.message else "(no message)" + header = f"## {summary}\n`{self.sha}` — {self.timestamp}\n" if diff: return f"{header}\n```diff\n{diff}\n```" return f"{header}\n(no file changes)" diff --git a/tests/agent/test_dream.py b/tests/agent/test_dream.py index 27e49fda5..8ce9f6c9d 100644 --- a/tests/agent/test_dream.py +++ b/tests/agent/test_dream.py @@ -1,19 +1,32 @@ -"""Tests for the Dream class — two-phase memory consolidation via AgentRunner.""" +"""Tests for Dream driven through AgentLoop._process_system_message.""" import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch import pytest -from unittest.mock import AsyncMock, MagicMock, patch - -from nanobot.agent.memory import Dream, MemoryStore +from nanobot.agent.loop import AgentLoop from nanobot.agent.runner import AgentRunResult from nanobot.agent.skills import BUILTIN_SKILLS_DIR +from nanobot.bus.events import InboundMessage +from nanobot.bus.queue import MessageBus from nanobot.utils.gitstore import LineAge +def _provider(default_model: str, max_tokens: int = 123) -> MagicMock: + provider = MagicMock() + provider.get_default_model.return_value = default_model + provider.generation = SimpleNamespace( + max_tokens=max_tokens, temperature=0.1, reasoning_effort=None + ) + return provider + + @pytest.fixture def store(tmp_path): + from nanobot.agent.memory import MemoryStore + s = MemoryStore(tmp_path) s.write_soul("# Soul\n- Helpful") s.write_user("# User\n- Developer") @@ -23,9 +36,7 @@ def store(tmp_path): @pytest.fixture def mock_provider(): - p = MagicMock() - p.chat_with_retry = AsyncMock() - return p + return _provider("test-model") @pytest.fixture @@ -34,10 +45,16 @@ def mock_runner(): @pytest.fixture -def dream(store, mock_provider, mock_runner): - d = Dream(store=store, provider=mock_provider, model="test-model", max_batch_size=5) - d._runner = mock_runner - return d +def loop(tmp_path, mock_provider, mock_runner): + loop = AgentLoop( + bus=MessageBus(), + provider=mock_provider, + workspace=tmp_path, + model="test-model", + context_window_tokens=1000, + ) + loop.dream._runner = mock_runner + return loop def _make_run_result( @@ -56,254 +73,418 @@ def _make_run_result( ) -class TestDreamRun: - async def test_noop_when_no_unprocessed_history(self, dream, mock_provider, mock_runner, store): - """Dream should not call LLM when there's nothing to process.""" - result = await dream.run() - assert result is False - mock_provider.chat_with_retry.assert_not_called() +class TestDreamAgentLoopIntegration: + async def test_completes_goal_state_after_full_backlog(self, loop, mock_runner, store): + """Goal should be completed after processing all backlog in internal loop.""" + for i in range(6): + store.append_history(f"event {i}") + mock_runner.run = AsyncMock(return_value=_make_run_result()) + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) + session = loop.sessions.get_or_create("system:dream") + goal = session.metadata.get("goal_state") + assert isinstance(goal, dict) + assert goal["status"] == "completed" + assert store.get_last_dream_cursor() == 6 + + async def test_completes_goal_state_on_finish(self, loop, mock_runner, store): + """Goal should be marked completed when backlog is fully processed.""" + store.append_history("event 1") + mock_runner.run = AsyncMock(return_value=_make_run_result()) + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) + session = loop.sessions.get_or_create("system:dream") + goal = session.metadata.get("goal_state") + assert goal["status"] == "completed" + assert "completed_at" in goal + assert "recap" in goal + + async def test_noop_when_no_unprocessed_history(self, loop, mock_runner): + """Dream should not call runner when there's nothing to process.""" + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + result = await loop._process_system_message(msg) + assert result is None mock_runner.run.assert_not_called() - async def test_calls_runner_for_unprocessed_entries(self, dream, mock_provider, mock_runner, store): + async def test_calls_runner_for_unprocessed_entries(self, loop, mock_runner, store): """Dream should call AgentRunner when there are unprocessed history entries.""" store.append_history("User prefers dark mode") - mock_provider.chat_with_retry.return_value = MagicMock(content="New fact") - mock_runner.run = AsyncMock(return_value=_make_run_result( - tool_events=[{"name": "edit_file", "status": "ok", "detail": "memory/MEMORY.md"}], - )) - result = await dream.run() - assert result is True + mock_runner.run = AsyncMock( + return_value=_make_run_result( + tool_events=[ + {"name": "edit_file", "status": "ok", "detail": "memory/MEMORY.md"} + ], + ) + ) + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) mock_runner.run.assert_called_once() spec = mock_runner.run.call_args[0][0] assert spec.max_iterations == 10 assert spec.fail_on_tool_error is False - async def test_advances_dream_cursor(self, dream, mock_provider, mock_runner, store): + async def test_advances_dream_cursor(self, loop, mock_runner, store): """Dream should advance the cursor after processing.""" store.append_history("event 1") store.append_history("event 2") - mock_provider.chat_with_retry.return_value = MagicMock(content="Nothing new") mock_runner.run = AsyncMock(return_value=_make_run_result()) - await dream.run() + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) assert store.get_last_dream_cursor() == 2 - async def test_compacts_processed_history(self, dream, mock_provider, mock_runner, store): + async def test_compacts_processed_history(self, loop, mock_runner, store): """Dream should compact history after processing.""" store.append_history("event 1") store.append_history("event 2") store.append_history("event 3") - mock_provider.chat_with_retry.return_value = MagicMock(content="Nothing new") mock_runner.run = AsyncMock(return_value=_make_run_result()) - await dream.run() - # After Dream, cursor is advanced and 3, compact keeps last max_history_entries + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) entries = store.read_unprocessed_history(since_cursor=0) assert all(e["cursor"] > 0 for e in entries) - async def test_skill_phase_uses_builtin_skill_creator_path(self, dream, mock_provider, mock_runner, store): - """Dream should point skill creation guidance at the builtin skill-creator template.""" + async def test_processes_full_backlog_in_one_call(self, loop, mock_runner, store): + """Backlog larger than max_batch_size should be fully processed in one call.""" + for i in range(12): + store.append_history(f"event {i}") + mock_runner.run = AsyncMock(return_value=_make_run_result()) + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) + assert store.get_last_dream_cursor() == 12 + assert mock_runner.run.call_count == 3 # 5 + 5 + 2 + + async def test_single_git_commit_for_multi_batch(self, loop, mock_runner, store): + """Multi-batch run should collapse into exactly one git commit.""" + store.git.init() + store.git.auto_commit("initial") + for i in range(12): + store.append_history(f"event {i}") + mock_runner.run = AsyncMock( + return_value=_make_run_result( + tool_events=[ + {"name": "edit_file", "status": "ok", "detail": "memory/MEMORY.md"} + ], + ) + ) + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) + commits = store.git.log() + dream_commits = [c for c in commits if c.message.startswith("dream:")] + assert len(dream_commits) == 1 + + async def test_system_prompt_cached(self, loop, mock_runner, store): + """Batches within one run should reuse cached system prompt when template mtime unchanged.""" + for i in range(6): + store.append_history(f"event {i}") + mock_runner.run = AsyncMock(return_value=_make_run_result()) + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) + # Two batches (5 + 1), both should use the same cached prompt + assert mock_runner.run.call_count == 2 + first_prompt = mock_runner.run.call_args_list[0][0][0].initial_messages[0]["content"] + second_prompt = mock_runner.run.call_args_list[1][0][0].initial_messages[0]["content"] + assert second_prompt is first_prompt + + async def test_noop_when_empty_backlog(self, loop, mock_runner, store): + """Empty backlog should not advance cursor or create a commit.""" + store.git.init() + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) + assert store.get_last_dream_cursor() == 0 + commits = store.git.log() + assert len([c for c in commits if c.message.startswith("dream:")]) == 0 + + +class TestDreamPrompt: + async def test_prompt_contains_mece_rules(self, loop, mock_runner, store): + store.append_history("some event") + mock_runner.run = AsyncMock(return_value=_make_run_result()) + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) + spec = mock_runner.run.call_args[0][0] + system_prompt = spec.initial_messages[0]["content"] + assert "Do NOT guess paths" in system_prompt + assert "SOUL.md" in system_prompt + assert "USER.md" in system_prompt + assert "MEMORY.md" in system_prompt + + async def test_skill_phase_uses_builtin_skill_creator_path(self, loop, mock_runner, store): store.append_history("Repeated workflow one") store.append_history("Repeated workflow two") - mock_provider.chat_with_retry.return_value = MagicMock(content="[SKILL] test-skill: test description") mock_runner.run = AsyncMock(return_value=_make_run_result()) - - await dream.run() - + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) spec = mock_runner.run.call_args[0][0] system_prompt = spec.initial_messages[0]["content"] expected = str(BUILTIN_SKILLS_DIR / "skill-creator" / "SKILL.md") assert expected in system_prompt - async def test_skill_write_tool_accepts_workspace_relative_skill_path(self, dream, store): - """Dream skill creation should allow skills//SKILL.md relative to workspace root.""" - write_tool = dream._tools.get("write_file") - assert write_tool is not None - - result = await write_tool.execute( - path="skills/test-skill/SKILL.md", - content="---\nname: test-skill\ndescription: Test\n---\n", + async def test_system_prompt_uses_threshold_from_template_var(self, loop, mock_runner, store): + store.append_history("some event") + mock_runner.run = AsyncMock(return_value=_make_run_result()) + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" ) - - assert "Successfully wrote" in result - assert (store.workspace / "skills" / "test-skill" / "SKILL.md").exists() - - async def test_phase1_prompt_includes_line_age_annotations(self, dream, mock_provider, mock_runner, store): - """Phase 1 prompt should have per-line age suffixes in MEMORY.md when git is available.""" - store.append_history("some event") - mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]") - mock_runner.run = AsyncMock(return_value=_make_run_result()) - - # Init git so line_ages works - store.git.init() - store.git.auto_commit("initial memory state") - - await dream.run() - - # The MEMORY.md section should not crash and should contain the memory content - call_args = mock_provider.chat_with_retry.call_args - user_msg = call_args.kwargs.get("messages", call_args[1].get("messages"))[1]["content"] - assert "## Current MEMORY.md" in user_msg - - async def test_phase1_annotates_only_memory_not_soul_or_user(self, dream, mock_provider, mock_runner, store): - """SOUL.md and USER.md should never have age annotations — they are permanent.""" - store.append_history("some event") - mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]") - mock_runner.run = AsyncMock(return_value=_make_run_result()) - - store.git.init() - store.git.auto_commit("initial state") - - await dream.run() - - call_args = mock_provider.chat_with_retry.call_args - user_msg = call_args.kwargs.get("messages", call_args[1].get("messages"))[1]["content"] - # The ← suffix should only appear in MEMORY.md section - memory_section = user_msg.split("## Current MEMORY.md")[1].split("## Current SOUL.md")[0] - soul_section = user_msg.split("## Current SOUL.md")[1].split("## Current USER.md")[0] - user_section = user_msg.split("## Current USER.md")[1] - # SOUL and USER should not contain age arrows - assert "\u2190" not in soul_section - assert "\u2190" not in user_section - - async def test_phase1_prompt_works_without_git(self, dream, mock_provider, mock_runner, store): - """Phase 1 should work fine even if git is not initialized (no age annotations).""" - store.append_history("some event") - mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]") - mock_runner.run = AsyncMock(return_value=_make_run_result()) - - await dream.run() - - # Should still succeed — just without age annotations - mock_provider.chat_with_retry.assert_called_once() - call_args = mock_provider.chat_with_retry.call_args - user_msg = call_args.kwargs.get("messages", call_args[1].get("messages"))[1]["content"] - assert "## Current MEMORY.md" in user_msg - - async def test_phase1_prompt_carries_age_suffix_for_stale_lines( - self, dream, mock_provider, mock_runner, store, - ): - """End-to-end: ages >14d must appear verbatim in the LLM prompt, ages ≤14d must not.""" - # MEMORY.md fixture has 2 non-blank lines ("# Memory" and "- Project X active"). - # Inject four ages to cover threshold boundaries: >14 suffix, ==14 no suffix, <14 no suffix. - store.write_memory("# Memory\n- Project X active\n- fresh item\n- edge case line") - store.append_history("some event") - mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]") - mock_runner.run = AsyncMock(return_value=_make_run_result()) - - fake_ages = [ - LineAge(age_days=30), # "# Memory" → should get ← 30d - LineAge(age_days=20), # "- Project X..." → should get ← 20d - LineAge(age_days=14), # "- fresh item" → ==14, threshold is strictly >14, no suffix - LineAge(age_days=5), # "- edge case..." → no suffix - ] - with patch.object(store.git, "line_ages", return_value=fake_ages): - await dream.run() - - call_args = mock_provider.chat_with_retry.call_args - user_msg = call_args.kwargs.get("messages", call_args[1].get("messages"))[1]["content"] - memory_section = user_msg.split("## Current MEMORY.md")[1].split("## Current SOUL.md")[0] - assert "\u2190 30d" in memory_section - assert "\u2190 20d" in memory_section - assert "\u2190 14d" not in memory_section - assert "\u2190 5d" not in memory_section - - async def test_phase1_skips_annotation_when_disabled( - self, dream, mock_provider, mock_runner, store, - ): - """`annotate_line_ages=False` must bypass the git lookup entirely and keep MEMORY.md raw.""" - store.append_history("some event") - mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]") - mock_runner.run = AsyncMock(return_value=_make_run_result()) - - dream.annotate_line_ages = False - # line_ages must be bypassed entirely — verify with a spy rather than a - # raising side_effect, because _annotate_with_ages catches Exception - # (which swallows AssertionError) and would hide an accidental call. - with patch.object(store.git, "line_ages") as mock_line_ages: - await dream.run() - mock_line_ages.assert_not_called() - - call_args = mock_provider.chat_with_retry.call_args - user_msg = call_args.kwargs.get("messages", call_args[1].get("messages"))[1]["content"] - assert "\u2190" not in user_msg - - async def test_phase1_skips_annotation_on_line_ages_length_mismatch( - self, dream, mock_provider, mock_runner, store, - ): - """If ages length != lines length (dirty working tree), skip annotation instead of mis-tagging.""" - # MEMORY.md has 2 non-blank lines but we hand back only 1 age → mismatch. - store.append_history("some event") - mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]") - mock_runner.run = AsyncMock(return_value=_make_run_result()) - - with patch.object(store.git, "line_ages", return_value=[LineAge(age_days=999)]): - await dream.run() - - call_args = mock_provider.chat_with_retry.call_args - user_msg = call_args.kwargs.get("messages", call_args[1].get("messages"))[1]["content"] - memory_section = user_msg.split("## Current MEMORY.md")[1].split("## Current SOUL.md")[0] - # No age arrow at all — we refused to annotate rather than tag the wrong line. - assert "\u2190" not in memory_section - - async def test_phase1_prompt_uses_threshold_from_template_var( - self, dream, mock_provider, mock_runner, store, - ): - """System prompt should reference the stale-threshold constant, not a hardcoded 14.""" - store.append_history("some event") - mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]") - mock_runner.run = AsyncMock(return_value=_make_run_result()) - - await dream.run() - - system_msg = mock_provider.chat_with_retry.call_args.kwargs["messages"][0]["content"] - # The template renders with stale_threshold_days=14 → LLM must see "N>14" + await loop._process_system_message(msg) + spec = mock_runner.run.call_args[0][0] + system_msg = spec.initial_messages[0]["content"] assert "N>14" in system_msg class TestDreamPromptCaps: - """Dream's Phase 1/2 prompt must not be poisoned by a legacy oversized - history entry or a runaway MEMORY.md. Without caps, a single pre-#3412 - raw_archive dump in history.jsonl would make every subsequent Dream run - exceed the context window and silently advance the cursor past real work. - """ - - async def test_phase1_caps_huge_memory_file( - self, dream, mock_provider, mock_runner, store, - ): - """A MEMORY.md much larger than _MEMORY_FILE_MAX_CHARS must be truncated - in the prompt preview (full content is still reachable via read_file).""" - store.write_memory("M" * (dream._MEMORY_FILE_MAX_CHARS * 5)) + async def test_caps_huge_memory_file(self, loop, mock_runner, store): + store.write_memory("M" * (loop.dream._MEMORY_FILE_MAX_CHARS * 5)) store.append_history("some event") - mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]") mock_runner.run = AsyncMock(return_value=_make_run_result()) + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) + spec = mock_runner.run.call_args[0][0] + user_msg = spec.initial_messages[1]["content"] + memory_section = user_msg.split("## Current MEMORY.md")[1].split( + "## Current SOUL.md" + )[0] + assert len(memory_section) < loop.dream._MEMORY_FILE_MAX_CHARS + 500 - await dream.run() - - user_msg = mock_provider.chat_with_retry.call_args.kwargs["messages"][1]["content"] - memory_section = user_msg.split("## Current MEMORY.md")[1].split("## Current SOUL.md")[0] - assert len(memory_section) < dream._MEMORY_FILE_MAX_CHARS + 500 - - async def test_phase1_caps_huge_history_entry( - self, dream, mock_provider, mock_runner, store, - ): - """A legacy oversized history entry (e.g. pre-#3412 raw_archive dump) - must not explode the Phase 1 prompt — each entry is capped in the - preview, even though the JSONL record itself stays full-size.""" - # Bypass the append_history cap by writing directly, simulating a - # record that was written by an older nanobot build before any caps. + async def test_caps_huge_history_entry(self, loop, mock_runner, store): store.history_file.write_text( - json.dumps({ - "cursor": 1, - "timestamp": "2026-04-01 10:00", - "content": "H" * (dream._HISTORY_ENTRY_PREVIEW_MAX_CHARS * 8), - }) + "\n", + json.dumps( + { + "cursor": 1, + "timestamp": "2026-04-01 10:00", + "content": "H" * (loop.dream._HISTORY_ENTRY_PREVIEW_MAX_CHARS * 8), + } + ) + + "\n", encoding="utf-8", ) - mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]") mock_runner.run = AsyncMock(return_value=_make_run_result()) + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) + spec = mock_runner.run.call_args[0][0] + user_msg = spec.initial_messages[1]["content"] + history_section = user_msg.split("## Conversation History\n")[1].split( + "\n\n## Current Date" + )[0] + assert len(history_section) < loop.dream._HISTORY_ENTRY_PREVIEW_MAX_CHARS + 500 - await dream.run() - user_msg = mock_provider.chat_with_retry.call_args.kwargs["messages"][1]["content"] - history_section = user_msg.split("## Conversation History\n")[1].split("\n\n## Current Date")[0] - assert len(history_section) < dream._HISTORY_ENTRY_PREVIEW_MAX_CHARS + 500 +class TestDreamTools: + def test_apply_patch_tool_registered(self, loop): + tool = loop.dream._tools.get("apply_patch") + assert tool is not None + +class TestDreamCaps: + def test_batch_size_default_is_5(self): + from nanobot.config.schema import DreamConfig + + assert DreamConfig().max_batch_size == 5 + + def test_memory_cap_is_16k(self, loop): + assert loop.dream._MEMORY_FILE_MAX_CHARS == 16_000 + + +class TestDreamSkipFiltering: + async def test_skip_entries_removed_from_prompt(self, loop, mock_runner, store): + store.append_history("- [skip] greeting\n- [permanent] User prefers dark mode") + mock_runner.run = AsyncMock(return_value=_make_run_result()) + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) + spec = mock_runner.run.call_args[0][0] + user_msg = spec.initial_messages[1]["content"] + assert "User prefers dark mode" in user_msg + assert "[skip]" not in user_msg + assert "greeting" not in user_msg + + +class TestDreamAgeAnnotations: + async def test_prompt_includes_line_age_annotations(self, loop, mock_runner, store): + store.append_history("some event") + mock_runner.run = AsyncMock(return_value=_make_run_result()) + store.git.init() + store.git.auto_commit("initial memory state") + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) + spec = mock_runner.run.call_args[0][0] + user_msg = spec.initial_messages[1]["content"] + assert "## Current MEMORY.md" in user_msg + + async def test_annotates_only_memory_not_soul_or_user(self, loop, mock_runner, store): + store.append_history("some event") + mock_runner.run = AsyncMock(return_value=_make_run_result()) + store.git.init() + store.git.auto_commit("initial state") + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) + spec = mock_runner.run.call_args[0][0] + user_msg = spec.initial_messages[1]["content"] + soul_section = user_msg.split("## Current SOUL.md")[1].split( + "## Current USER.md" + )[0] + user_section = user_msg.split("## Current USER.md")[1] + assert "←" not in soul_section + assert "←" not in user_section + + async def test_prompt_works_without_git(self, loop, mock_runner, store): + store.append_history("some event") + mock_runner.run = AsyncMock(return_value=_make_run_result()) + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) + mock_runner.run.assert_called_once() + spec = mock_runner.run.call_args[0][0] + user_msg = spec.initial_messages[1]["content"] + assert "## Current MEMORY.md" in user_msg + + async def test_prompt_carries_age_suffix_for_stale_lines(self, loop, mock_runner, store): + store.write_memory( + "# Memory\n- Project X active\n- fresh item\n- edge case line" + ) + store.append_history("some event") + mock_runner.run = AsyncMock(return_value=_make_run_result()) + fake_ages = [ + LineAge(age_days=30), + LineAge(age_days=20), + LineAge(age_days=14), + LineAge(age_days=5), + ] + with patch.object(loop.dream.store.git, "line_ages", return_value=fake_ages): + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) + spec = mock_runner.run.call_args[0][0] + user_msg = spec.initial_messages[1]["content"] + memory_section = user_msg.split("## Current MEMORY.md")[1].split( + "## Current SOUL.md" + )[0] + assert "← 30d" in memory_section + assert "← 20d" in memory_section + assert "← 14d" not in memory_section + assert "← 5d" not in memory_section + + async def test_skips_annotation_when_disabled(self, loop, mock_runner, store): + store.append_history("some event") + mock_runner.run = AsyncMock(return_value=_make_run_result()) + loop.dream.annotate_line_ages = False + with patch.object(loop.dream.store.git, "line_ages") as mock_line_ages: + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) + mock_line_ages.assert_not_called() + spec = mock_runner.run.call_args[0][0] + user_msg = spec.initial_messages[1]["content"] + assert "←" not in user_msg + + async def test_skips_annotation_on_line_ages_length_mismatch(self, loop, mock_runner, store): + store.append_history("some event") + mock_runner.run = AsyncMock(return_value=_make_run_result()) + with patch.object( + loop.dream.store.git, "line_ages", return_value=[LineAge(age_days=999)] + ): + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) + spec = mock_runner.run.call_args[0][0] + user_msg = spec.initial_messages[1]["content"] + memory_section = user_msg.split("## Current MEMORY.md")[1].split( + "## Current SOUL.md" + )[0] + assert "←" not in memory_section + + +class TestDreamSessionPersistence: + async def test_writes_session_on_success(self, loop, mock_runner, store): + store.append_history("event one") + store.append_history("event two") + mock_runner.run = AsyncMock( + return_value=_make_run_result( + tool_events=[ + {"name": "edit_file", "status": "ok", "detail": "memory/MEMORY.md"} + ], + ) + ) + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) + session_path = store.memory_dir / ".dream_session.json" + assert session_path.exists() + data = json.loads(session_path.read_text(encoding="utf-8")) + assert data["batch"]["from_cursor"] == 0 + assert data["batch"]["to_cursor"] == 2 + assert data["batch"]["count"] == 2 + assert data["stop_reason"] == "completed" + assert data["changelog"] == ["edit_file: memory/MEMORY.md"] + assert "timestamp" in data + assert "elapsed_seconds" in data + assert "messages" in data + + async def test_no_session_record_on_failure(self, loop, mock_runner, store): + """Failed batch should not write a session record (cursor stays put for retry).""" + store.append_history("event one") + mock_runner.run = AsyncMock(side_effect=RuntimeError("LLM error")) + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) + session_path = store.memory_dir / ".dream_session.json" + assert not session_path.exists() + assert store.get_last_dream_cursor() == 0 + + async def test_session_contains_full_messages(self, loop, mock_runner, store): + store.append_history("event one") + messages = [ + {"role": "system", "content": "you are a memory bot"}, + {"role": "user", "content": "history here"}, + {"role": "assistant", "content": "I will edit MEMORY.md"}, + ] + result = _make_run_result() + result.messages = messages + mock_runner.run = AsyncMock(return_value=result) + msg = InboundMessage( + channel="system", sender_id="dream", chat_id="dream", content="" + ) + await loop._process_system_message(msg) + session_path = store.memory_dir / ".dream_session.json" + data = json.loads(session_path.read_text(encoding="utf-8")) + assert data["messages"] == messages + assert data["prompt_chars"] > 0 + assert data["commit_sha"] is None diff --git a/tests/agent/test_memory.py b/tests/agent/test_memory.py new file mode 100644 index 000000000..234c0dc70 --- /dev/null +++ b/tests/agent/test_memory.py @@ -0,0 +1,125 @@ +"""Tests for memory system: Consolidator, token estimation, truncation.""" + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from nanobot.agent.memory import _TIKTOKEN_ENC, Consolidator, MemoryStore, _estimate_tokens + + +@pytest.fixture +def store(tmp_path): + s = MemoryStore(tmp_path) + s.write_soul("# Soul\n- Helpful") + s.write_user("# User\n- Developer") + s.write_memory("# Memory\n- Project X active") + return s + + +@pytest.fixture +def mock_provider(): + p = MagicMock() + p.chat_with_retry = AsyncMock() + p.generation.max_tokens = 4096 + return p + + +@pytest.fixture +def mock_sessions(): + return MagicMock() + + +@pytest.fixture +def mock_build_messages(): + return MagicMock(return_value=[]) + + +@pytest.fixture +def mock_get_tool_definitions(): + return MagicMock(return_value=[]) + + +@pytest.fixture +def consolidator(store, mock_provider, mock_sessions, mock_build_messages, mock_get_tool_definitions): + return Consolidator( + store=store, + provider=mock_provider, + model="test-model", + sessions=mock_sessions, + context_window_tokens=128_000, + build_messages=mock_build_messages, + get_tool_definitions=mock_get_tool_definitions, + ) + + +class TestEstimateTokens: + def test_estimate_tokens_returns_positive(self): + assert _estimate_tokens("hello world") > 0 + + def test_estimate_tokens_english_approximate(self): + # English is roughly 1 token per 4 chars as fallback + text = "a " * 100 + if _TIKTOKEN_ENC is not None: + expected = len(_TIKTOKEN_ENC.encode(text)) + else: + expected = len(text) // 4 + assert _estimate_tokens(text) == expected + + +class TestTruncateToTokenBudget: + def test_reserve_tokens_reduces_budget(self, consolidator): + long_text = "word " * 200_000 + # Without reserve, more text survives + no_reserve = consolidator._truncate_to_token_budget(long_text, reserve_tokens=0) + with_reserve = consolidator._truncate_to_token_budget(long_text, reserve_tokens=500) + assert len(with_reserve) < len(no_reserve) + + def test_reserve_tokens_zero_default(self, consolidator): + text = "hello world" + result = consolidator._truncate_to_token_budget(text) + assert result == text + + +class TestConsolidatorPrompt: + def test_prompt_contains_snip(self): + from nanobot.utils.prompt_templates import render_template + text = render_template("agent/consolidator_archive.md", strip=True) + assert "SNIP" in text + assert "[permanent]" in text + assert "[skip]" in text + + +class TestConsolidatorArchive: + async def test_archive_injects_dedup_context(self, consolidator, mock_provider, store): + store.write_memory("- User prefers dark mode") + store.write_user("- Developer") + messages = [{"role": "user", "content": "hello", "timestamp": "2026-01-01 10:00"}] + + mock_provider.chat_with_retry.return_value = MagicMock( + content="(nothing)", finish_reason="stop" + ) + await consolidator.archive(messages) + + call_args = mock_provider.chat_with_retry.call_args + user_msg = call_args.kwargs["messages"][1]["content"] + assert "## Current MEMORY.md (for dedup)" in user_msg + assert "User prefers dark mode" in user_msg + assert "## Current USER.md (for dedup)" in user_msg + assert "Developer" in user_msg + + async def test_archive_skips_dedup_when_budget_exhausted(self, consolidator, mock_provider, store): + # Shrink token budget so dedup context (always capped at ~6000 chars) + # exceeds the available room. + consolidator.context_window_tokens = 6_000 + store.write_memory("word " * 10_000) + messages = [{"role": "user", "content": "hello", "timestamp": "2026-01-01 10:00"}] + + mock_provider.chat_with_retry.return_value = MagicMock( + content="(nothing)", finish_reason="stop" + ) + await consolidator.archive(messages) + + call_args = mock_provider.chat_with_retry.call_args + user_msg = call_args.kwargs["messages"][1]["content"] + # Should not contain dedup context when budget is exhausted + assert "## Current MEMORY.md (for dedup)" not in user_msg diff --git a/tests/agent/test_self_model_preset.py b/tests/agent/test_self_model_preset.py index 0f52f777b..08828ca88 100644 --- a/tests/agent/test_self_model_preset.py +++ b/tests/agent/test_self_model_preset.py @@ -292,3 +292,95 @@ def test_from_config_static_preset_loader_does_not_enable_hot_reload(tmp_path) - loop = AgentLoop.from_config(config) assert loop._provider_snapshot_loader is None assert loop._preset_snapshot_loader is not None + + +class TestDreamModelOverride: + def test_dream_follows_main_when_no_override(self, tmp_path) -> None: + provider = _provider("base-model") + loop = AgentLoop( + bus=MessageBus(), + provider=provider, + workspace=tmp_path, + model="base-model", + context_window_tokens=1000, + ) + assert loop.dream.model == "base-model" + assert loop.dream.provider is provider + + def test_dream_raw_model_override(self, tmp_path) -> None: + provider = _provider("base-model") + loop = AgentLoop( + bus=MessageBus(), + provider=provider, + workspace=tmp_path, + model="base-model", + context_window_tokens=1000, + dream_model_override="custom-model-v2", + ) + assert loop.dream.model == "custom-model-v2" + assert loop.dream.provider is provider + + def test_dream_preset_override(self, tmp_path) -> None: + cheap_provider = _provider("openai/gpt-4.1-mini", max_tokens=2048) + preset = ModelPresetConfig( + model="openai/gpt-4.1-mini", + provider="openai", + max_tokens=2048, + context_window_tokens=128_000, + ) + loop = AgentLoop( + bus=MessageBus(), + provider=_provider("base-model"), + workspace=tmp_path, + model="base-model", + context_window_tokens=1000, + model_presets={"cheap": preset}, + dream_model_override="cheap", + preset_snapshot_loader=lambda _name: ProviderSnapshot( + provider=cheap_provider, + model=preset.model, + context_window_tokens=preset.context_window_tokens, + signature=("cheap", preset.model), + ), + ) + assert loop.dream.model == "openai/gpt-4.1-mini" + assert loop.dream.provider is cheap_provider + assert loop.dream._runner.provider is cheap_provider + + def test_dream_override_survives_main_preset_switch(self, tmp_path) -> None: + base_provider = _provider("base-model") + fast_provider = _provider("openai/gpt-4.1", max_tokens=4096) + cheap_provider = _provider("openai/gpt-4.1-mini", max_tokens=2048) + loop = AgentLoop( + bus=MessageBus(), + provider=base_provider, + workspace=tmp_path, + model="base-model", + context_window_tokens=1000, + model_presets={ + "fast": ModelPresetConfig(model="openai/gpt-4.1"), + "cheap": ModelPresetConfig(model="openai/gpt-4.1-mini"), + }, + dream_model_override="cheap", + preset_snapshot_loader=lambda name: ProviderSnapshot( + provider=fast_provider if name == "fast" else cheap_provider, + model="openai/gpt-4.1" if name == "fast" else "openai/gpt-4.1-mini", + context_window_tokens=32_768 if name == "fast" else 128_000, + signature=(name, "model"), + ), + ) + # Initially dream is on cheap + assert loop.dream.model == "openai/gpt-4.1-mini" + assert loop.dream.provider is cheap_provider + + # Switch main preset to fast + loop.set_model_preset("fast") + + # Main agent should be on fast + assert loop.model == "openai/gpt-4.1" + assert loop.provider is fast_provider + + # Dream should still be on cheap override + assert loop.dream.model == "openai/gpt-4.1-mini" + assert loop.dream.provider is cheap_provider + assert loop.dream._runner.provider is cheap_provider diff --git a/tests/command/test_builtin_dream.py b/tests/command/test_builtin_dream.py index 7b1835feb..f5a5589ab 100644 --- a/tests/command/test_builtin_dream.py +++ b/tests/command/test_builtin_dream.py @@ -113,6 +113,52 @@ async def test_dream_restore_lists_versions_with_next_steps() -> None: assert "Restore a version with `/dream-restore `." in out.content +@pytest.mark.asyncio +async def test_dream_log_shows_summary_and_analysis() -> None: + commit = CommitInfo( + sha="abcd1234", + message="dream: 2026-04-04, 2 change(s)\n\n[ADD] fact A →USER\n[REMOVE] old fact", + timestamp="2026-04-04 12:00", + ) + diff = ( + "diff --git a/SOUL.md b/SOUL.md\n" + "--- a/SOUL.md\n" + "+++ b/SOUL.md\n" + "@@ -1 +1 @@\n" + "-old\n" + "+new\n" + ) + git = _FakeGit(commits=[commit], diff_map={commit.sha: (commit, diff)}) + + out = await cmd_dream_log(_make_ctx("/dream-log", git)) + + assert "## Dream Update" in out.content + assert "- Summary: dream: 2026-04-04, 2 change(s)" in out.content + assert "### Analysis" in out.content + assert "[ADD] fact A →USER" in out.content + assert "[REMOVE] old fact" in out.content + + +@pytest.mark.asyncio +async def test_dream_log_with_empty_commit_message() -> None: + commit = CommitInfo(sha="abcd1234", message="", timestamp="2026-04-04 12:00") + diff = ( + "diff --git a/SOUL.md b/SOUL.md\n" + "--- a/SOUL.md\n" + "+++ b/SOUL.md\n" + "@@ -1 +1 @@\n" + "-old\n" + "+new\n" + ) + git = _FakeGit(commits=[commit], diff_map={commit.sha: (commit, diff)}) + + out = await cmd_dream_log(_make_ctx("/dream-log", git)) + + assert "## Dream Update" in out.content + assert "- Summary:" not in out.content + assert "### Analysis" not in out.content + + @pytest.mark.asyncio async def test_dream_restore_success_mentions_files_and_followup() -> None: commit = CommitInfo(sha="abcd1234", message="dream: latest", timestamp="2026-04-04 12:00") diff --git a/tests/utils/test_gitstore.py b/tests/utils/test_gitstore.py index b431bf719..3cd6ad2a4 100644 --- a/tests/utils/test_gitstore.py +++ b/tests/utils/test_gitstore.py @@ -1,216 +1,13 @@ -"""Tests for GitStore — line_ages() and core git operations.""" - -import subprocess -import time -from datetime import datetime, timedelta, timezone -from unittest.mock import patch - -import pytest - -from nanobot.utils.gitstore import GitStore +from nanobot.utils.gitstore import CommitInfo -@pytest.fixture -def git(tmp_path): - """Create an initialized GitStore with tracked MEMORY.md.""" - g = GitStore(tmp_path, tracked_files=["MEMORY.md", "SOUL.md"]) - g.init() - return g +class TestCommitInfo: + def test_format_with_empty_message(self): + commit = CommitInfo(sha="abcd1234", message="", timestamp="2026-04-04 12:00") + result = commit.format() + assert "(no message)" in result or "## " in result - -class TestLineAges: - def test_returns_empty_when_not_initialized(self, tmp_path): - """line_ages should return [] if the git repo is not initialized.""" - git = GitStore(tmp_path, tracked_files=["MEMORY.md"]) - assert git.line_ages("MEMORY.md") == [] - - def test_returns_empty_for_missing_file(self, git): - """line_ages should return [] for a file that doesn't exist.""" - assert git.line_ages("SOUL.md") == [] - - def test_returns_empty_for_empty_file(self, git, tmp_path): - """line_ages should return [] for an empty tracked file.""" - (tmp_path / "SOUL.md").write_text("", encoding="utf-8") - git.auto_commit("empty soul") - assert git.line_ages("SOUL.md") == [] - - def test_one_age_per_line(self, git, tmp_path): - """line_ages should return one entry per line in the file.""" - content = "# Memory\n\n## Section A\n- item 1\n" - (tmp_path / "MEMORY.md").write_text(content, encoding="utf-8") - git.auto_commit("initial") - ages = git.line_ages("MEMORY.md") - assert len(ages) == len(content.splitlines()) - - def test_fresh_lines_have_age_zero(self, git, tmp_path): - """Lines committed today should have age_days=0.""" - (tmp_path / "MEMORY.md").write_text("## A\n- x\n", encoding="utf-8") - git.auto_commit("initial") - ages = git.line_ages("MEMORY.md") - assert all(a.age_days == 0 for a in ages) - - def test_age_differentiates_across_days(self, git, tmp_path): - """Lines committed today should show correct age when 'now' is mocked forward.""" - (tmp_path / "MEMORY.md").write_text("## A\n- x\n", encoding="utf-8") - git.auto_commit("initial") - - future_now = datetime.now(tz=timezone.utc) + timedelta(days=30) - with patch("nanobot.utils.gitstore.datetime") as mock_dt: - mock_dt.now.return_value = future_now - mock_dt.fromtimestamp = datetime.fromtimestamp - ages = git.line_ages("MEMORY.md") - - assert len(ages) == 2 - assert all(a.age_days == 30 for a in ages) - - def test_annotate_failure_returns_empty(self, tmp_path): - """If annotate fails, line_ages should return [] gracefully.""" - git = GitStore(tmp_path, tracked_files=["MEMORY.md"]) - # Don't init — annotate will fail - assert git.line_ages("MEMORY.md") == [] - - def test_partial_edit_only_updates_changed_lines(self, git, tmp_path): - """Only modified lines should reflect the new commit's timestamp.""" - (tmp_path / "MEMORY.md").write_text( - "# Memory\n\n## A\n- old\n\n## B\n- keep\n", encoding="utf-8" - ) - git.auto_commit("commit1") - time.sleep(1.1) - - # Only modify section A - (tmp_path / "MEMORY.md").write_text( - "# Memory\n\n## A\n- new\n\n## B\n- keep\n", encoding="utf-8" - ) - git.auto_commit("commit2") - - ages = git.line_ages("MEMORY.md") - lines = (tmp_path / "MEMORY.md").read_text(encoding="utf-8").splitlines() - # All lines are from today, but verify line-level tracking works - assert len(ages) == len(lines) - # "- new" line and "- keep" line both age=0 (same day), but - # the key point is we get per-line results - assert len(ages) == 7 - - -class TestNestedRepoProtection: - """Regression tests for GitHub issue #2980: nested repo protection.""" - - def test_init_refuses_inside_git_repo(self, tmp_path): - """init() should detect it's inside an existing git repo and refuse.""" - project = tmp_path / "project" - project.mkdir() - (project / ".git").mkdir() - - workspace = project / "workspace" - workspace.mkdir() - - g = GitStore(workspace, tracked_files=["MEMORY.md"]) - result = g.init() - - assert result is False - assert not (workspace / ".git").is_dir() - - def test_init_preserves_existing_gitignore(self, tmp_path): - """init() should preserve existing .gitignore entries and append new ones.""" - workspace = tmp_path / "workspace" - workspace.mkdir() - - existing = "*.pyc\n__pycache__/\n" - (workspace / ".gitignore").write_text(existing, encoding="utf-8") - - g = GitStore(workspace, tracked_files=["MEMORY.md"]) - result = g.init() - - assert result is True - gitignore = (workspace / ".gitignore").read_text(encoding="utf-8") - assert "*.pyc" in gitignore - assert "__pycache__/" in gitignore - assert "!MEMORY.md" in gitignore - assert "!.gitignore" in gitignore - - def test_init_no_gitignore_creates_new(self, tmp_path): - """init() should create .gitignore with Dream content when none exists.""" - workspace = tmp_path / "workspace" - workspace.mkdir() - - g = GitStore(workspace, tracked_files=["MEMORY.md"]) - result = g.init() - - assert result is True - gitignore = (workspace / ".gitignore").read_text(encoding="utf-8") - expected = g._build_gitignore() - assert gitignore == expected - - def test_init_gitignore_merge_idempotent(self, tmp_path): - """init() should not duplicate Dream entries already in .gitignore.""" - workspace = tmp_path / "workspace" - workspace.mkdir() - - # Pre-existing .gitignore that already has some Dream entries - existing = "*.pyc\n/*\n!MEMORY.md\n" - (workspace / ".gitignore").write_text(existing, encoding="utf-8") - - g = GitStore(workspace, tracked_files=["MEMORY.md"]) - result = g.init() - - assert result is True - gitignore = (workspace / ".gitignore").read_text(encoding="utf-8") - # No duplicate lines - lines = gitignore.splitlines() - assert lines.count("/*") == 1 - assert lines.count("!MEMORY.md") == 1 - # Existing entry preserved, new Dream entries appended - assert "*.pyc" in gitignore - assert "!.gitignore" in gitignore - - def test_init_outside_git_repo_works_normally(self, tmp_path): - """init() should succeed and create .git when not inside a git repo.""" - workspace = tmp_path / "workspace" - workspace.mkdir() - - g = GitStore(workspace, tracked_files=["MEMORY.md"]) - result = g.init() - - assert result is True - assert (workspace / ".git").is_dir() - - def test_init_refuses_inside_git_worktree(self, tmp_path): - """init() should refuse when the parent checkout is a git worktree.""" - repo = tmp_path / "repo" - repo.mkdir() - subprocess.run(["git", "init", "-q", str(repo)], check=True) - (repo / "README.md").write_text("x\n", encoding="utf-8") - subprocess.run(["git", "-C", str(repo), "add", "README.md"], check=True) - subprocess.run( - [ - "git", - "-C", - str(repo), - "-c", - "user.name=test", - "-c", - "user.email=test@example.com", - "commit", - "-q", - "-m", - "init", - ], - check=True, - ) - subprocess.run(["git", "-C", str(repo), "branch", "wt-branch"], check=True) - - worktree = tmp_path / "worktree" - subprocess.run( - ["git", "-C", str(repo), "worktree", "add", "-q", str(worktree), "wt-branch"], - check=True, - ) - assert (worktree / ".git").is_file() - - workspace = worktree / "workspace" - workspace.mkdir() - - g = GitStore(workspace, tracked_files=["MEMORY.md"]) - result = g.init() - - assert result is False - assert not (workspace / ".git").exists() + def test_format_with_message(self): + commit = CommitInfo(sha="abcd1234", message="dream: update", timestamp="2026-04-04 12:00") + result = commit.format() + assert "dream: update" in result