feat: add demand-driven document retrieval (#5525)

tests/tools/test_read_enhancements.py

@@ -8,6 +8,19 @@ import pytest
 
 from nanobot.agent.tools import file_state
 from nanobot.agent.tools.filesystem import ReadFileTool, WriteFileTool
+from nanobot.utils.document import (
+    DocumentExtractionError,
+    DocumentLineSource,
+    LocatedDocumentLine,
+)
+
+
+def _document_source(text: str) -> DocumentLineSource:
+    lines = (
+        LocatedDocumentLine(line, line_no, "")
+        for line_no, line in enumerate(text.splitlines(), 1)
+    )
+    return DocumentLineSource(lines)
 
 
 @pytest.fixture(autouse=True)
@@ -220,6 +233,12 @@ class TestReadPdf:
 
         assert "Invalid page range" in result
 
+        out_of_bounds = await tool.execute(path=str(pdf_path), pages="99")
+        assert out_of_bounds == (
+            "Error: Invalid page range '99': document has 1 page; "
+            "use a page number or range within 1-1."
+        )
+
     @pytest.mark.asyncio
     async def test_pdf_file_not_found_error(self, tool, tmp_path):
         result = await tool.execute(path=str(tmp_path / "nope.pdf"))
@@ -345,7 +364,10 @@ class TestReadOfficeDocuments:
 
     @pytest.mark.asyncio
     async def test_docx_returns_extracted_text(self, tool, tmp_path):
-        with patch("nanobot.utils.document.extract_text", return_value="Title\n\nParagraph 1"):
+        with patch(
+            "nanobot.utils.document.open_document_line_source",
+            return_value=_document_source("Title\n\nParagraph 1"),
+        ):
             f = tmp_path / "test.docx"
             f.write_bytes(b"PK")
             result = await tool.execute(path=str(f))
@@ -355,16 +377,69 @@ class TestReadOfficeDocuments:
 
     @pytest.mark.asyncio
     async def test_xlsx_returns_extracted_text(self, tool, tmp_path):
-        with patch("nanobot.utils.document.extract_text", return_value="--- Sheet: Sheet1 ---\nName\tAge\nAlice\t30"):
+        with patch(
+            "nanobot.utils.document.open_document_line_source",
+            return_value=_document_source("--- Sheet: Sheet1 ---\nName\tAge\nAlice\t30"),
+        ):
             f = tmp_path / "test.xlsx"
             f.write_bytes(b"PK")
             result = await tool.execute(path=str(f))
         assert "Sheet1" in result
         assert "Alice" in result
 
+    @pytest.mark.asyncio
+    async def test_office_documents_support_extracted_line_ranges(self, tool, tmp_path):
+        extracted = "--- Sheet: Sheet1 ---\nName\tAge\nAlice\t30\nBob\t25"
+        with patch(
+            "nanobot.utils.document.open_document_line_source",
+            return_value=_document_source(extracted),
+        ):
+            f = tmp_path / "test.xlsx"
+            f.write_bytes(b"PK")
+            result = await tool.execute(path=str(f), offset=3, limit=1)
+
+        assert "3| Alice\t30" in result
+        assert "Name\tAge" not in result
+        assert "Use offset=4 to continue" in result
+
+    @pytest.mark.asyncio
+    async def test_office_range_reaches_beyond_attachment_preview_limit(
+        self,
+        tool,
+        tmp_path,
+        monkeypatch,
+    ):
+        from openpyxl import Workbook
+
+        from nanobot.utils import document as document_utils
+
+        workbook_path = tmp_path / "long.xlsx"
+        workbook = Workbook()
+        sheet = workbook.active
+        for row in range(1, 20):
+            sheet.append([f"ordinary-row-{row}"])
+        sheet.append(["late-content"])
+        workbook.save(workbook_path)
+        workbook.close()
+        monkeypatch.setattr(document_utils, "_MAX_TEXT_LENGTH", 50)
+
+        preview = document_utils.extract_text(workbook_path)
+        assert preview is not None
+        assert "late-content" not in preview
+
+        result = await tool.execute(path=str(workbook_path), offset=21, limit=1)
+
+        assert "21| late-content" in result
+        assert "beyond end" not in result
+
     @pytest.mark.asyncio
     async def test_pptx_returns_extracted_text(self, tool, tmp_path):
-        with patch("nanobot.utils.document.extract_text", return_value="--- Slide 1 ---\nWelcome\n--- Slide 2 ---\nContent"):
+        with patch(
+            "nanobot.utils.document.open_document_line_source",
+            return_value=_document_source(
+                "--- Slide 1 ---\nWelcome\n--- Slide 2 ---\nContent"
+            ),
+        ):
             f = tmp_path / "test.pptx"
             f.write_bytes(b"PK")
             result = await tool.execute(path=str(f))
@@ -373,7 +448,10 @@ class TestReadOfficeDocuments:
 
     @pytest.mark.asyncio
     async def test_docx_missing_library(self, tool, tmp_path):
-        with patch("nanobot.utils.document.extract_text", return_value="[error: python-docx not installed]"):
+        with patch(
+            "nanobot.utils.document.open_document_line_source",
+            side_effect=DocumentExtractionError("python-docx not installed"),
+        ):
             f = tmp_path / "test.docx"
             f.write_bytes(b"PK")
             result = await tool.execute(path=str(f))
@@ -382,7 +460,10 @@ class TestReadOfficeDocuments:
 
     @pytest.mark.asyncio
     async def test_docx_corrupt_file(self, tool, tmp_path):
-        with patch("nanobot.utils.document.extract_text", return_value="[error: failed to extract DOCX: bad zip]"):
+        with patch(
+            "nanobot.utils.document.open_document_line_source",
+            side_effect=DocumentExtractionError("failed to extract DOCX: bad zip"),
+        ):
             f = tmp_path / "test.docx"
             f.write_bytes(b"not-a-zip")
             result = await tool.execute(path=str(f))
@@ -391,7 +472,7 @@ class TestReadOfficeDocuments:
 
     @pytest.mark.asyncio
     async def test_unsupported_extension(self, tool, tmp_path):
-        with patch("nanobot.utils.document.extract_text", return_value=None):
+        with patch("nanobot.utils.document.open_document_line_source", return_value=None):
             f = tmp_path / "test.docx"
             f.write_bytes(b"PK")
             result = await tool.execute(path=str(f))
@@ -400,7 +481,10 @@ class TestReadOfficeDocuments:
 
     @pytest.mark.asyncio
     async def test_empty_document_returns_descriptive_message(self, tool, tmp_path):
-        with patch("nanobot.utils.document.extract_text", return_value=""):
+        with patch(
+            "nanobot.utils.document.open_document_line_source",
+            return_value=_document_source(""),
+        ):
             f = tmp_path / "empty.docx"
             f.write_bytes(b"PK")
             result = await tool.execute(path=str(f))
@@ -415,7 +499,10 @@ class TestOfficeDocTruncation:
 
     @pytest.mark.asyncio
     async def test_large_document_truncated(self, tool, tmp_path):
-        with patch("nanobot.utils.document.extract_text", return_value="x" * 200_000):
+        with patch(
+            "nanobot.utils.document.open_document_line_source",
+            return_value=_document_source("x" * 200_000),
+        ):
             f = tmp_path / "large.docx"
             f.write_bytes(b"PK")
             result = await tool.execute(path=str(f))
@@ -424,7 +511,10 @@ class TestOfficeDocTruncation:
 
     @pytest.mark.asyncio
     async def test_small_document_not_truncated(self, tool, tmp_path):
-        with patch("nanobot.utils.document.extract_text", return_value="Hello world"):
+        with patch(
+            "nanobot.utils.document.open_document_line_source",
+            return_value=_document_source("Hello world"),
+        ):
             f = tmp_path / "small.docx"
             f.write_bytes(b"PK")
             result = await tool.execute(path=str(f))
@@ -433,7 +523,12 @@ class TestOfficeDocTruncation:
 
     @pytest.mark.asyncio
     async def test_error_response_not_truncated(self, tool, tmp_path):
-        with patch("nanobot.utils.document.extract_text", return_value="[error: failed to extract DOCX: something went wrong]"):
+        with patch(
+            "nanobot.utils.document.open_document_line_source",
+            side_effect=DocumentExtractionError(
+                "failed to extract DOCX: something went wrong"
+            ),
+        ):
             f = tmp_path / "bad.docx"
             f.write_bytes(b"PK")
             result = await tool.execute(path=str(f))

tests/tools/test_search_tools.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import os
+import re
 import time
 from pathlib import Path
 from types import SimpleNamespace
@@ -124,9 +125,14 @@ async def test_grep_respects_glob_filter_and_context(tmp_path: Path) -> None:
 
 
 @pytest.mark.asyncio
-async def test_grep_defaults_to_files_with_matches(tmp_path: Path) -> None:
+async def test_grep_defaults_to_match_context(tmp_path: Path) -> None:
     (tmp_path / "src").mkdir()
-    (tmp_path / "src" / "main.py").write_text("match_here\n", encoding="utf-8")
+    (tmp_path / "src" / "main.py").write_text(
+        "\n".join(f"line {line}" for line in range(1, 6))
+        + "\nmatch_here\n"
+        + "\n".join(f"line {line}" for line in range(7, 13)),
+        encoding="utf-8",
+    )
 
     tool = GrepTool(workspace=tmp_path, allowed_dir=tmp_path)
     result = await tool.execute(
@@ -134,8 +140,230 @@ async def test_grep_defaults_to_files_with_matches(tmp_path: Path) -> None:
         path="src",
     )
 
-    assert result.splitlines() == ["src/main.py"]
-    assert "1|" not in result
+    assert "src/main.py:6" in result
+    assert "  1| line 1" in result
+    assert "> 6| match_here" in result
+    assert "  11| line 11" in result
+    assert "line 12" not in result
+
+
+@pytest.mark.asyncio
+async def test_grep_searches_xlsx_with_sheet_cell_locator(tmp_path: Path) -> None:
+    from openpyxl import Workbook
+
+    workbook_path = tmp_path / "people.xlsx"
+    workbook = Workbook()
+    sheet = workbook.active
+    sheet.title = "People"
+    sheet.append(["Name", "Role"])
+    sheet.append(["Ada", "Engineer"])
+    workbook.save(workbook_path)
+    workbook.close()
+
+    tool = GrepTool(workspace=tmp_path, allowed_dir=tmp_path)
+    result = await tool.execute(
+        pattern="Engineer",
+        path="people.xlsx",
+        fixed_strings=True,
+    )
+
+    assert "people.xlsx:3" in result
+    assert "sheet='People',row=2,cell=B2" in result
+    assert "Ada\tEngineer" in result
+
+
+@pytest.mark.asyncio
+async def test_grep_searches_docx_with_paragraph_locator(tmp_path: Path) -> None:
+    from docx import Document
+
+    document_path = tmp_path / "notes.docx"
+    document = Document()
+    document.add_paragraph("Introduction")
+    document.add_paragraph("late needle")
+    document.save(document_path)
+
+    tool = GrepTool(workspace=tmp_path, allowed_dir=tmp_path)
+    result = await tool.execute(
+        pattern="late needle",
+        path="notes.docx",
+        fixed_strings=True,
+    )
+
+    assert "notes.docx:3 [paragraph=2]" in result
+    assert "late needle" in result
+
+
+@pytest.mark.asyncio
+async def test_grep_searches_pptx_with_slide_locator(tmp_path: Path) -> None:
+    from pptx import Presentation
+    from pptx.util import Inches
+
+    presentation_path = tmp_path / "deck.pptx"
+    presentation = Presentation()
+    slide = presentation.slides.add_slide(presentation.slide_layouts[6])
+    textbox = slide.shapes.add_textbox(
+        Inches(1), Inches(1), Inches(4), Inches(1)
+    )
+    textbox.text_frame.text = "slide needle"
+    presentation.save(presentation_path)
+
+    tool = GrepTool(workspace=tmp_path, allowed_dir=tmp_path)
+    result = await tool.execute(
+        pattern="slide needle",
+        path="deck.pptx",
+        fixed_strings=True,
+    )
+
+    assert "deck.pptx:2 [slide=1,line=1]" in result
+    assert "slide needle" in result
+
+
+@pytest.mark.asyncio
+async def test_grep_searches_pdf_with_page_locator(tmp_path: Path) -> None:
+    import fitz
+
+    pdf_path = tmp_path / "notes.pdf"
+    document = fitz.open()
+    page = document.new_page()
+    page.insert_text((72, 72), "pdf needle")
+    document.save(pdf_path)
+    document.close()
+
+    tool = GrepTool(workspace=tmp_path, allowed_dir=tmp_path)
+    result = await tool.execute(
+        pattern="pdf needle",
+        path="notes.pdf",
+        fixed_strings=True,
+    )
+
+    assert "notes.pdf:2 [page=1,line=1]" in result
+    assert "pdf needle" in result
+
+
+@pytest.mark.asyncio
+async def test_grep_searches_xlsx_beyond_attachment_preview_limit(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from openpyxl import Workbook
+
+    from nanobot.utils import document as document_utils
+
+    workbook_path = tmp_path / "long.xlsx"
+    workbook = Workbook()
+    sheet = workbook.active
+    sheet.title = "Data"
+    for row in range(1, 20):
+        sheet.append([f"ordinary-row-{row}"])
+    sheet.append(["late-needle"])
+    workbook.save(workbook_path)
+    workbook.close()
+    monkeypatch.setattr(document_utils, "_MAX_TEXT_LENGTH", 50)
+
+    preview = document_utils.extract_text(workbook_path)
+    assert preview is not None
+    assert "late-needle" not in preview
+
+    tool = GrepTool(workspace=tmp_path, allowed_dir=tmp_path)
+    result = await tool.execute(
+        pattern="late-needle",
+        path="long.xlsx",
+        fixed_strings=True,
+    )
+
+    assert "late-needle" in result
+    assert "sheet='Data',row=20,cell=A20" in result
+    assert "No matches found" not in result
+
+
+@pytest.mark.asyncio
+async def test_grep_keeps_an_oversized_matching_line_visible(tmp_path: Path) -> None:
+    long_line = "x" * 130_000 + "needle" + "y" * 10_000
+    (tmp_path / "huge-line.txt").write_text(long_line, encoding="utf-8")
+    tool = GrepTool(workspace=tmp_path, allowed_dir=tmp_path)
+
+    result = await tool.execute(
+        pattern="needle",
+        path="huge-line.txt",
+        fixed_strings=True,
+        context_before=0,
+        context_after=0,
+    )
+
+    assert "huge-line.txt:1" in result
+    assert "needle" in result
+    assert "No matches found" not in result
+    assert len(result) < GrepTool._MAX_RESULT_CHARS
+
+
+@pytest.mark.asyncio
+async def test_grep_size_limit_returns_a_resumable_offset(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    content = "\n".join(f"needle-{line}-" + "x" * 80 for line in range(1, 11))
+    (tmp_path / "many.txt").write_text(content, encoding="utf-8")
+    monkeypatch.setattr(GrepTool, "_MAX_RESULT_CHARS", 350)
+    tool = GrepTool(workspace=tmp_path, allowed_dir=tmp_path)
+
+    first = await tool.execute(
+        pattern="needle",
+        path="many.txt",
+        fixed_strings=True,
+        context_before=0,
+        context_after=0,
+        head_limit=10,
+    )
+    continuation = re.search(r"use offset=(\d+) to continue", first)
+
+    assert continuation is not None
+    next_offset = int(continuation.group(1))
+    assert next_offset > 0
+
+    second = await tool.execute(
+        pattern="needle",
+        path="many.txt",
+        fixed_strings=True,
+        context_before=0,
+        context_after=0,
+        head_limit=10,
+        offset=next_offset,
+    )
+    first_headers = {
+        line for line in first.splitlines() if line.startswith("many.txt:")
+    }
+    second_headers = {
+        line for line in second.splitlines() if line.startswith("many.txt:")
+    }
+    assert second_headers
+    assert first_headers.isdisjoint(second_headers)
+
+
+@pytest.mark.asyncio
+async def test_grep_reports_an_invalid_pdf_page_range(tmp_path: Path) -> None:
+    from pypdf import PdfWriter
+
+    pdf_path = tmp_path / "one-page.pdf"
+    writer = PdfWriter()
+    writer.add_blank_page(width=100, height=100)
+    with pdf_path.open("wb") as output:
+        writer.write(output)
+
+    tool = GrepTool(workspace=tmp_path, allowed_dir=tmp_path)
+    result = await tool.execute(pattern="needle", path="one-page.pdf", pages="bad")
+
+    assert result.startswith("Error: Invalid PDF page range 'bad'")
+    assert "binary/unreadable" not in result
+
+    out_of_bounds = await tool.execute(
+        pattern="needle",
+        path="one-page.pdf",
+        pages="99",
+    )
+    assert out_of_bounds == (
+        "Error: Invalid PDF page range '99': document has 1 page; "
+        "use a page number or range within 1-1."
+    )
 
 
 @pytest.mark.asyncio
@@ -169,6 +397,7 @@ async def test_grep_type_filter_limits_files(tmp_path: Path) -> None:
         pattern="needle",
         path="src",
         type="py",
+        output_mode="files_with_matches",
     )
 
     assert result.splitlines() == ["src/a.py"]
@@ -224,6 +453,7 @@ async def test_grep_files_with_matches_supports_head_limit_and_offset(tmp_path:
     result = await tool.execute(
         pattern="needle",
         path="src",
+        output_mode="files_with_matches",
         head_limit=1,
         offset=1,
     )
@@ -319,12 +549,16 @@ async def test_grep_uses_a_larger_bounded_limit_for_an_explicit_file(
     assert "skipped 1 large files" in capped_result
 
 
-def test_grep_description_keeps_size_thresholds_implementation_specific(tmp_path: Path) -> None:
+def test_grep_schema_is_concise_and_keeps_legacy_aliases_hidden(tmp_path: Path) -> None:
     tool = GrepTool(workspace=tmp_path, allowed_dir=tmp_path)
+    properties = tool.parameters["properties"]
 
-    assert "limits are enforced by the tool" in tool.description
+    assert len(tool.description) < 150
     assert "2 MB" not in tool.description
     assert "100 MB" not in tool.description
+    assert "head_limit" in properties
+    assert "max_matches" not in properties
+    assert "max_results" not in properties
 
 
 @pytest.mark.asyncio

tests/tools/test_tool_descriptions.py

@@ -30,12 +30,24 @@ def test_coding_tool_descriptions_steer_discovery() -> None:
     find_files = FindFilesTool().description.lower()
     grep = GrepTool().description.lower()
 
-    assert "find_files/list_dir first" in read_file
-    assert "before editing" in read_file
-    assert "uploaded non-image attachments are referenced by path" in read_file
-    assert "only when their contents are needed" in read_file
-    assert "prefer it over shell find/ls" in find_files
-    assert "prefer this over shell grep" in grep
+    assert "text, images, pdfs, and office documents" in read_file
+    assert "line-numbered" in read_file
+    assert "targeted ranges" in read_file
+    assert len(read_file) < 160
+
+    assert "workspace paths" in find_files
+    assert "relative paths" in find_files
+    assert len(find_files) < 140
+
+    assert "pdf, docx, xlsx, and pptx" in grep
+    assert "five context lines" in grep
+    assert "source locators" in grep
+    assert len(grep) < 150
+
+    read_pages = ReadFileTool().parameters["properties"]["pages"]["description"].lower()
+    grep_pages = GrepTool().parameters["properties"]["pages"]["description"].lower()
+    assert "page number or range" in read_pages
+    assert "page number or range" in grep_pages
 
 
 def test_exec_tool_descriptions_are_concise() -> None: