nanobot/tests/agent/test_verification_state.py

from __future__ import annotations

from nanobot.agent.verification_state import (
    analyze_verification_result,
    append_verification_feedback,
)


def test_analyze_pytest_failure_extracts_actionable_summary():
    output = """\
FAILED ../tests/test_outputs.py::test_regex_matches_dates - AssertionError: Expected dates
E       AssertionError: Expected ['2025-01-09'], but got ['bad']
E       FileNotFoundError: [Errno 2] No such file or directory: '/app/out.txt'
============================== 1 failed in 0.05s ===============================
Exit code: 1
"""

    analysis = analyze_verification_result(
        command="pytest /tests/test_outputs.py",
        output=output,
        exit_code=1,
    )

    assert analysis is not None
    assert analysis.status == "failed"
    assert analysis.failed_tests == ("../tests/test_outputs.py::test_regex_matches_dates",)
    assert any("AssertionError" in item for item in analysis.primary_errors)
    assert "/app/out.txt" in analysis.missing_artifacts


def test_append_verification_feedback_tells_agent_not_to_finish():
    analysis = analyze_verification_result(
        command="python /app/test_outputs.py",
        output="FAILED test_outputs.py::test_file\nAssertionError: missing\nExit code: 1",
        exit_code=1,
    )

    feedback = append_verification_feedback("raw output\nExit code: 1", analysis)

    assert "[Verification Feedback]" in feedback
    assert "Do not call complete_goal" in feedback
    assert "Next action" in feedback


def test_analyze_passing_test_records_success_without_feedback():
    analysis = analyze_verification_result(
        command="pytest",
        output="============================== 3 passed in 0.10s ==============================\nExit code: 0",
        exit_code=0,
    )

    assert analysis is not None
    assert analysis.status == "passed"
    assert append_verification_feedback("ok", analysis) == "ok"


def test_analyze_command_not_found_as_failed_check():
    output = """\
STDERR:
/usr/bin/bash: line 1: python3: command not found

Exit code: 127
"""

    analysis = analyze_verification_result(
        command="python3 - <<'PY'\nprint('quick verification')\nPY",
        output=output,
        exit_code=127,
    )

    assert analysis is not None
    assert analysis.status == "failed"
    assert any("command not found" in item for item in analysis.primary_errors)


def test_analyze_artifact_comparison_success_records_pass():
    output = """\
run_exit:0
0d115b98  /app/image.ppm
0d115b98  /tmp/orig.ppm
cmp_exit:0
      7      21    1024

Exit code: 0
"""

    analysis = analyze_verification_result(
        command=(
            "cd /usr/bin && gcc -static -o /app/reversed_final /app/mystery.c -lm "
            "&& (cd /app && ./reversed_final >/tmp/final_out 2>/tmp/final_err); "
            "sha256sum /app/image.ppm /tmp/orig.ppm; "
            "cmp -s /app/image.ppm /tmp/orig.ppm; echo cmp_exit:$?"
        ),
        output=output,
        exit_code=0,
    )

    assert analysis is not None
    assert analysis.status == "passed"
    assert append_verification_feedback("ok", analysis) == "ok"


def test_analyze_plain_checksum_without_success_marker_is_ignored():
    analysis = analyze_verification_result(
        command="sha256sum /app/image.ppm /tmp/orig.ppm",
        output="0d115b98  /app/image.ppm\n0d115b98  /tmp/orig.ppm\nExit code: 0",
        exit_code=0,
    )

    assert analysis is None


def test_analyze_named_comparison_markers_record_pass():
    output = """\
ppm:0
stderr:0
stdout:0
      4      26    1011
1821 mystery.c

Exit code: 0
"""

    analysis = analyze_verification_result(
        command=(
            "gcc -static -O2 -o reversed mystery.c -lm\n"
            "./reversed > vrout.txt 2> vrerr.txt\n"
            "cp image.ppm rev.ppm\n"
            "./mystery > voout.txt 2> voerr.txt\n"
            "cmp image.ppm rev.ppm\n"
            "printf 'ppm:%s\\n' $?\n"
            "cmp voerr.txt vrerr.txt\n"
            "printf 'stderr:%s\\n' $?\n"
            "cmp voout.txt vrout.txt\n"
            "printf 'stdout:%s\\n' $?"
        ),
        output=output,
        exit_code=0,
    )

    assert analysis is not None
    assert analysis.status == "passed"


def test_analyze_named_comparison_marker_failure_records_failed():
    output = """\
ppm:0
stderr:1
stdout:0

Exit code: 0
"""

    analysis = analyze_verification_result(
        command=(
            "cmp image.ppm rev.ppm; printf 'ppm:%s\\n' $?; "
            "cmp voerr.txt vrerr.txt; printf 'stderr:%s\\n' $?; "
            "cmp voout.txt vrout.txt; printf 'stdout:%s\\n' $?"
        ),
        output=output,
        exit_code=0,
    )

    assert analysis is not None
    assert analysis.status == "failed"


def test_analyze_plain_run_status_marker_without_comparison_is_ignored():
    analysis = analyze_verification_result(
        command="gcc -static -O2 -o reversed mystery.c -lm && ./reversed",
        output="rc:0\nExit code: 0",
        exit_code=0,
    )

    assert analysis is None