mirror of
https://github.com/HKUDS/nanobot.git
synced 2026-09-04 02:01:48 +03:00
fix(gateway): recover degraded WebSocket listener (#5544)
* fix(gateway): recover degraded WebSocket listener * test(websocket): model listener readiness in startup stub * fix(tui): keep launcher alive during gateway recovery * test(websocket): colocate listener lifecycle coverage
This commit is contained in:
@@ -3618,7 +3618,12 @@ def test_gateway_health_endpoint_binds_and_serves_expected_responses(
|
||||
assert health_writer.closed is True
|
||||
assert "HTTP/1.0 200 OK" in health_response
|
||||
health_body = json.loads(health_response.split("\r\n\r\n", 1)[1])
|
||||
assert health_body == {"status": "ok"}
|
||||
assert health_body == {
|
||||
"status": "ok",
|
||||
"process": "alive",
|
||||
"ready": True,
|
||||
"websocket": "disabled",
|
||||
}
|
||||
|
||||
missing_response, missing_writer = _call_handler("/missing")
|
||||
assert missing_writer.closed is True
|
||||
|
||||
@@ -14,7 +14,11 @@ from contextlib import suppress
|
||||
from nanobot.agent.hook import AgentRunHookContext
|
||||
from nanobot.agent.tools.mcp import MCPProvider
|
||||
from nanobot.agent.tools.registry import ToolRegistry
|
||||
from nanobot.cli.gateway_runtime import _close_gateway_runtime, _MCPReadinessHook
|
||||
from nanobot.cli.gateway_runtime import (
|
||||
_close_gateway_runtime,
|
||||
_gateway_readiness_payload,
|
||||
_MCPReadinessHook,
|
||||
)
|
||||
|
||||
|
||||
class _FakeAgent:
|
||||
@@ -65,6 +69,33 @@ class _TrackingMCPProvider(MCPProvider):
|
||||
self.connect_calls += 1
|
||||
|
||||
|
||||
def test_gateway_readiness_is_degraded_when_required_websocket_is_unavailable() -> None:
|
||||
channels = type(
|
||||
"Channels",
|
||||
(),
|
||||
{
|
||||
"enabled_channels": ["websocket"],
|
||||
"get_status": lambda self: {
|
||||
"websocket": {
|
||||
"enabled": True,
|
||||
"running": False,
|
||||
"state": "starting",
|
||||
}
|
||||
},
|
||||
},
|
||||
)()
|
||||
|
||||
ready, payload = _gateway_readiness_payload(channels)
|
||||
|
||||
assert ready is False
|
||||
assert payload == {
|
||||
"status": "degraded",
|
||||
"process": "alive",
|
||||
"ready": False,
|
||||
"websocket": "starting",
|
||||
}
|
||||
|
||||
|
||||
async def test_mcp_readiness_hook_delegates_to_application_provider() -> None:
|
||||
provider = _TrackingMCPProvider()
|
||||
hook = _MCPReadinessHook(provider)
|
||||
|
||||
@@ -191,6 +191,78 @@ def test_launcher_terminates_the_tui_when_gateway_start_fails(
|
||||
assert terminated == [True]
|
||||
|
||||
|
||||
def test_launcher_keeps_the_tui_alive_while_an_existing_gateway_recovers(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
config = Config()
|
||||
events: list[str] = []
|
||||
status_calls = 0
|
||||
|
||||
class FakeRuntime:
|
||||
def __init__(self, *, paths: object) -> None:
|
||||
self.paths = paths
|
||||
|
||||
def status(self) -> SimpleNamespace:
|
||||
nonlocal status_calls
|
||||
status_calls += 1
|
||||
return SimpleNamespace(
|
||||
running=True,
|
||||
port=config.gateway.port,
|
||||
ready=False,
|
||||
log_path=tmp_path / "gateway.log",
|
||||
)
|
||||
|
||||
class FakeProcess:
|
||||
def poll(self) -> None:
|
||||
return None
|
||||
|
||||
def terminate(self) -> None:
|
||||
events.append("terminated")
|
||||
|
||||
def wait(self, timeout: float | None = None) -> int:
|
||||
assert timeout is None
|
||||
events.append("waited")
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr("nanobot.gateway.GatewayRuntime", FakeRuntime)
|
||||
monkeypatch.setattr("nanobot.cli.tui_launcher._resolve_tui_command", lambda: ["nanobot-tui"])
|
||||
monkeypatch.setattr(
|
||||
"nanobot.cli.tui_launcher.subprocess.Popen",
|
||||
lambda *args, **kwargs: FakeProcess(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"nanobot.cli.tui_launcher._webui_endpoint_reachable",
|
||||
lambda _url: pytest.fail(
|
||||
"launcher must not probe readiness for a live recovering gateway"
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
tui_launcher,
|
||||
"time",
|
||||
SimpleNamespace(
|
||||
monotonic=lambda: pytest.fail(
|
||||
"launcher must not wait for a live gateway to recover"
|
||||
),
|
||||
sleep=lambda _seconds: pytest.fail(
|
||||
"launcher must not sleep for gateway recovery"
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
result = launch_tui(
|
||||
config,
|
||||
config_path=tmp_path / "config.json",
|
||||
workspace_override=None,
|
||||
session_id=None,
|
||||
theme="auto",
|
||||
)
|
||||
|
||||
assert result == 0
|
||||
assert status_calls == 1
|
||||
assert events == ["waited"]
|
||||
|
||||
|
||||
def test_launcher_promotes_the_gateway_when_the_tui_detaches(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
@@ -684,6 +756,10 @@ def test_gateway_reuses_the_matching_managed_instance(
|
||||
|
||||
monkeypatch.setattr("nanobot.gateway.GatewayRuntime", FakeRuntime)
|
||||
monkeypatch.setattr("nanobot.cli.tui_launcher._webui_endpoint_reachable", lambda _url: True)
|
||||
monkeypatch.setattr(
|
||||
"nanobot.cli.tui_launcher._gateway_health_ready",
|
||||
lambda *_args, **_kwargs: True,
|
||||
)
|
||||
|
||||
gateway = _ensure_gateway(
|
||||
config,
|
||||
@@ -694,21 +770,42 @@ def test_gateway_reuses_the_matching_managed_instance(
|
||||
assert gateway.base_url == "http://127.0.0.1:8765"
|
||||
|
||||
|
||||
def test_gateway_reuse_can_return_before_the_webui_endpoint_is_ready(
|
||||
def test_gateway_reuse_returns_a_degraded_live_gateway_without_waiting(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
config = Config()
|
||||
status_calls = 0
|
||||
|
||||
class FakeRuntime:
|
||||
def __init__(self, *, paths: object) -> None:
|
||||
self.paths = paths
|
||||
|
||||
def status(self) -> SimpleNamespace:
|
||||
return SimpleNamespace(running=True, port=config.gateway.port)
|
||||
nonlocal status_calls
|
||||
status_calls += 1
|
||||
return SimpleNamespace(
|
||||
running=True,
|
||||
port=config.gateway.port,
|
||||
ready=False,
|
||||
log_path=tmp_path / "gateway.log",
|
||||
)
|
||||
|
||||
monkeypatch.setattr("nanobot.gateway.GatewayRuntime", FakeRuntime)
|
||||
monkeypatch.setattr("nanobot.cli.tui_launcher._webui_endpoint_reachable", lambda _url: False)
|
||||
monkeypatch.setattr(
|
||||
"nanobot.cli.tui_launcher._webui_endpoint_reachable",
|
||||
lambda _url: pytest.fail("non-blocking reuse must not probe readiness"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
tui_launcher,
|
||||
"time",
|
||||
SimpleNamespace(
|
||||
monotonic=lambda: pytest.fail(
|
||||
"non-blocking reuse must not enter the readiness wait"
|
||||
),
|
||||
sleep=lambda _seconds: pytest.fail("non-blocking reuse must not sleep"),
|
||||
),
|
||||
)
|
||||
|
||||
gateway = _ensure_gateway(
|
||||
config,
|
||||
@@ -719,6 +816,95 @@ def test_gateway_reuse_can_return_before_the_webui_endpoint_is_ready(
|
||||
|
||||
assert gateway.base_url == "http://127.0.0.1:8765"
|
||||
assert gateway.lease is not None
|
||||
assert status_calls == 1
|
||||
gateway.lease.release(wait_for_stop=False)
|
||||
|
||||
|
||||
def test_gateway_reuse_waits_for_a_live_gateway_to_recover_its_webui_listener(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
config = Config()
|
||||
endpoint_results = iter((False, False, True))
|
||||
|
||||
class FakeRuntime:
|
||||
def __init__(self, *, paths: object) -> None:
|
||||
self.paths = paths
|
||||
|
||||
def status(self) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
running=True,
|
||||
port=config.gateway.port,
|
||||
log_path=tmp_path / "gateway.log",
|
||||
)
|
||||
|
||||
monkeypatch.setattr("nanobot.gateway.GatewayRuntime", FakeRuntime)
|
||||
monkeypatch.setattr(
|
||||
"nanobot.cli.tui_launcher._webui_endpoint_reachable",
|
||||
lambda _url: next(endpoint_results),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"nanobot.cli.tui_launcher._gateway_health_ready",
|
||||
lambda *_args, **_kwargs: True,
|
||||
)
|
||||
clock = iter((0.0, 0.0, 0.1))
|
||||
sleeps: list[float] = []
|
||||
monkeypatch.setattr(
|
||||
tui_launcher,
|
||||
"time",
|
||||
SimpleNamespace(monotonic=lambda: next(clock), sleep=sleeps.append),
|
||||
)
|
||||
|
||||
gateway = _ensure_gateway(
|
||||
config,
|
||||
config_path=tmp_path / "config.json",
|
||||
workspace_override=None,
|
||||
)
|
||||
|
||||
assert gateway.base_url == "http://127.0.0.1:8765"
|
||||
assert gateway.lease is not None
|
||||
assert sleeps == [tui_launcher._GATEWAY_READY_POLL_S]
|
||||
gateway.lease.release(wait_for_stop=False)
|
||||
|
||||
|
||||
def test_gateway_reuse_with_explicit_wait_rejects_a_live_but_unready_gateway(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
config = Config()
|
||||
|
||||
class FakeRuntime:
|
||||
def __init__(self, *, paths: object) -> None:
|
||||
self.paths = paths
|
||||
|
||||
def status(self) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
running=True,
|
||||
port=config.gateway.port,
|
||||
log_path=tmp_path / "gateway.log",
|
||||
)
|
||||
|
||||
monkeypatch.setattr("nanobot.gateway.GatewayRuntime", FakeRuntime)
|
||||
monkeypatch.setattr(
|
||||
"nanobot.cli.tui_launcher._webui_endpoint_reachable",
|
||||
lambda _url: False,
|
||||
)
|
||||
clock = iter((0.0, tui_launcher._GATEWAY_READY_TIMEOUT_S))
|
||||
monkeypatch.setattr(
|
||||
tui_launcher,
|
||||
"time",
|
||||
SimpleNamespace(
|
||||
monotonic=lambda: next(clock),
|
||||
sleep=lambda _seconds: pytest.fail("expired readiness wait must not sleep"),
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(TuiUnavailableError, match="process is running.*listener is unavailable"):
|
||||
_ensure_gateway(
|
||||
config,
|
||||
config_path=tmp_path / "config.json",
|
||||
workspace_override=None,
|
||||
)
|
||||
|
||||
|
||||
def test_gateway_started_for_tui_stops_when_its_last_lease_exits(
|
||||
@@ -767,6 +953,10 @@ def test_gateway_started_for_tui_stops_when_its_last_lease_exits(
|
||||
"nanobot.cli.tui_launcher._webui_endpoint_reachable",
|
||||
lambda _url: started,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"nanobot.cli.tui_launcher._gateway_health_ready",
|
||||
lambda *_args, **_kwargs: started,
|
||||
)
|
||||
|
||||
gateway = _ensure_gateway(
|
||||
config,
|
||||
|
||||
@@ -994,6 +994,37 @@ def test_status_keeps_live_state_when_identity_probe_is_temporarily_unavailable(
|
||||
assert runtime.paths.state_path.exists()
|
||||
|
||||
|
||||
def test_status_distinguishes_live_process_from_degraded_gateway_readiness(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
runtime = GatewayRuntime(paths=_paths(tmp_path), platform_name="Linux")
|
||||
runtime.paths.run_dir.mkdir(parents=True)
|
||||
runtime.paths.state_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"pid": 12345,
|
||||
"identity": 42,
|
||||
"port": 18791,
|
||||
"health_host": "127.0.0.1",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(runtime, "_is_pid_running", lambda _pid: True)
|
||||
monkeypatch.setattr(runtime, "_process_identity", lambda _pid: 42)
|
||||
monkeypatch.setattr(
|
||||
"nanobot.gateway.runtime._gateway_health_ready",
|
||||
lambda _host, _port: False,
|
||||
)
|
||||
|
||||
status = runtime.status()
|
||||
|
||||
assert status.running is True
|
||||
assert status.ready is False
|
||||
assert status.reason == "websocket_unavailable"
|
||||
|
||||
|
||||
def test_stop_refuses_to_signal_a_process_when_identity_cannot_be_verified(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
|
||||
Reference in New Issue
Block a user