fix(gateway): recover degraded WebSocket listener (#5544)

* fix(gateway): recover degraded WebSocket listener

* test(websocket): model listener readiness in startup stub

* fix(tui): keep launcher alive during gateway recovery

* test(websocket): colocate listener lifecycle coverage
This commit is contained in:
chengyongru
2026-08-27 10:23:35 +08:00
committed by GitHub
parent f9d449ef6c
commit d8b4f612f2
10 changed files with 747 additions and 98 deletions
+6 -1
View File
@@ -3618,7 +3618,12 @@ def test_gateway_health_endpoint_binds_and_serves_expected_responses(
assert health_writer.closed is True
assert "HTTP/1.0 200 OK" in health_response
health_body = json.loads(health_response.split("\r\n\r\n", 1)[1])
assert health_body == {"status": "ok"}
assert health_body == {
"status": "ok",
"process": "alive",
"ready": True,
"websocket": "disabled",
}
missing_response, missing_writer = _call_handler("/missing")
assert missing_writer.closed is True
+32 -1
View File
@@ -14,7 +14,11 @@ from contextlib import suppress
from nanobot.agent.hook import AgentRunHookContext
from nanobot.agent.tools.mcp import MCPProvider
from nanobot.agent.tools.registry import ToolRegistry
from nanobot.cli.gateway_runtime import _close_gateway_runtime, _MCPReadinessHook
from nanobot.cli.gateway_runtime import (
_close_gateway_runtime,
_gateway_readiness_payload,
_MCPReadinessHook,
)
class _FakeAgent:
@@ -65,6 +69,33 @@ class _TrackingMCPProvider(MCPProvider):
self.connect_calls += 1
def test_gateway_readiness_is_degraded_when_required_websocket_is_unavailable() -> None:
channels = type(
"Channels",
(),
{
"enabled_channels": ["websocket"],
"get_status": lambda self: {
"websocket": {
"enabled": True,
"running": False,
"state": "starting",
}
},
},
)()
ready, payload = _gateway_readiness_payload(channels)
assert ready is False
assert payload == {
"status": "degraded",
"process": "alive",
"ready": False,
"websocket": "starting",
}
async def test_mcp_readiness_hook_delegates_to_application_provider() -> None:
provider = _TrackingMCPProvider()
hook = _MCPReadinessHook(provider)
+193 -3
View File
@@ -191,6 +191,78 @@ def test_launcher_terminates_the_tui_when_gateway_start_fails(
assert terminated == [True]
def test_launcher_keeps_the_tui_alive_while_an_existing_gateway_recovers(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
config = Config()
events: list[str] = []
status_calls = 0
class FakeRuntime:
def __init__(self, *, paths: object) -> None:
self.paths = paths
def status(self) -> SimpleNamespace:
nonlocal status_calls
status_calls += 1
return SimpleNamespace(
running=True,
port=config.gateway.port,
ready=False,
log_path=tmp_path / "gateway.log",
)
class FakeProcess:
def poll(self) -> None:
return None
def terminate(self) -> None:
events.append("terminated")
def wait(self, timeout: float | None = None) -> int:
assert timeout is None
events.append("waited")
return 0
monkeypatch.setattr("nanobot.gateway.GatewayRuntime", FakeRuntime)
monkeypatch.setattr("nanobot.cli.tui_launcher._resolve_tui_command", lambda: ["nanobot-tui"])
monkeypatch.setattr(
"nanobot.cli.tui_launcher.subprocess.Popen",
lambda *args, **kwargs: FakeProcess(),
)
monkeypatch.setattr(
"nanobot.cli.tui_launcher._webui_endpoint_reachable",
lambda _url: pytest.fail(
"launcher must not probe readiness for a live recovering gateway"
),
)
monkeypatch.setattr(
tui_launcher,
"time",
SimpleNamespace(
monotonic=lambda: pytest.fail(
"launcher must not wait for a live gateway to recover"
),
sleep=lambda _seconds: pytest.fail(
"launcher must not sleep for gateway recovery"
),
),
)
result = launch_tui(
config,
config_path=tmp_path / "config.json",
workspace_override=None,
session_id=None,
theme="auto",
)
assert result == 0
assert status_calls == 1
assert events == ["waited"]
def test_launcher_promotes_the_gateway_when_the_tui_detaches(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
@@ -684,6 +756,10 @@ def test_gateway_reuses_the_matching_managed_instance(
monkeypatch.setattr("nanobot.gateway.GatewayRuntime", FakeRuntime)
monkeypatch.setattr("nanobot.cli.tui_launcher._webui_endpoint_reachable", lambda _url: True)
monkeypatch.setattr(
"nanobot.cli.tui_launcher._gateway_health_ready",
lambda *_args, **_kwargs: True,
)
gateway = _ensure_gateway(
config,
@@ -694,21 +770,42 @@ def test_gateway_reuses_the_matching_managed_instance(
assert gateway.base_url == "http://127.0.0.1:8765"
def test_gateway_reuse_can_return_before_the_webui_endpoint_is_ready(
def test_gateway_reuse_returns_a_degraded_live_gateway_without_waiting(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
config = Config()
status_calls = 0
class FakeRuntime:
def __init__(self, *, paths: object) -> None:
self.paths = paths
def status(self) -> SimpleNamespace:
return SimpleNamespace(running=True, port=config.gateway.port)
nonlocal status_calls
status_calls += 1
return SimpleNamespace(
running=True,
port=config.gateway.port,
ready=False,
log_path=tmp_path / "gateway.log",
)
monkeypatch.setattr("nanobot.gateway.GatewayRuntime", FakeRuntime)
monkeypatch.setattr("nanobot.cli.tui_launcher._webui_endpoint_reachable", lambda _url: False)
monkeypatch.setattr(
"nanobot.cli.tui_launcher._webui_endpoint_reachable",
lambda _url: pytest.fail("non-blocking reuse must not probe readiness"),
)
monkeypatch.setattr(
tui_launcher,
"time",
SimpleNamespace(
monotonic=lambda: pytest.fail(
"non-blocking reuse must not enter the readiness wait"
),
sleep=lambda _seconds: pytest.fail("non-blocking reuse must not sleep"),
),
)
gateway = _ensure_gateway(
config,
@@ -719,6 +816,95 @@ def test_gateway_reuse_can_return_before_the_webui_endpoint_is_ready(
assert gateway.base_url == "http://127.0.0.1:8765"
assert gateway.lease is not None
assert status_calls == 1
gateway.lease.release(wait_for_stop=False)
def test_gateway_reuse_waits_for_a_live_gateway_to_recover_its_webui_listener(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
config = Config()
endpoint_results = iter((False, False, True))
class FakeRuntime:
def __init__(self, *, paths: object) -> None:
self.paths = paths
def status(self) -> SimpleNamespace:
return SimpleNamespace(
running=True,
port=config.gateway.port,
log_path=tmp_path / "gateway.log",
)
monkeypatch.setattr("nanobot.gateway.GatewayRuntime", FakeRuntime)
monkeypatch.setattr(
"nanobot.cli.tui_launcher._webui_endpoint_reachable",
lambda _url: next(endpoint_results),
)
monkeypatch.setattr(
"nanobot.cli.tui_launcher._gateway_health_ready",
lambda *_args, **_kwargs: True,
)
clock = iter((0.0, 0.0, 0.1))
sleeps: list[float] = []
monkeypatch.setattr(
tui_launcher,
"time",
SimpleNamespace(monotonic=lambda: next(clock), sleep=sleeps.append),
)
gateway = _ensure_gateway(
config,
config_path=tmp_path / "config.json",
workspace_override=None,
)
assert gateway.base_url == "http://127.0.0.1:8765"
assert gateway.lease is not None
assert sleeps == [tui_launcher._GATEWAY_READY_POLL_S]
gateway.lease.release(wait_for_stop=False)
def test_gateway_reuse_with_explicit_wait_rejects_a_live_but_unready_gateway(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
config = Config()
class FakeRuntime:
def __init__(self, *, paths: object) -> None:
self.paths = paths
def status(self) -> SimpleNamespace:
return SimpleNamespace(
running=True,
port=config.gateway.port,
log_path=tmp_path / "gateway.log",
)
monkeypatch.setattr("nanobot.gateway.GatewayRuntime", FakeRuntime)
monkeypatch.setattr(
"nanobot.cli.tui_launcher._webui_endpoint_reachable",
lambda _url: False,
)
clock = iter((0.0, tui_launcher._GATEWAY_READY_TIMEOUT_S))
monkeypatch.setattr(
tui_launcher,
"time",
SimpleNamespace(
monotonic=lambda: next(clock),
sleep=lambda _seconds: pytest.fail("expired readiness wait must not sleep"),
),
)
with pytest.raises(TuiUnavailableError, match="process is running.*listener is unavailable"):
_ensure_gateway(
config,
config_path=tmp_path / "config.json",
workspace_override=None,
)
def test_gateway_started_for_tui_stops_when_its_last_lease_exits(
@@ -767,6 +953,10 @@ def test_gateway_started_for_tui_stops_when_its_last_lease_exits(
"nanobot.cli.tui_launcher._webui_endpoint_reachable",
lambda _url: started,
)
monkeypatch.setattr(
"nanobot.cli.tui_launcher._gateway_health_ready",
lambda *_args, **_kwargs: started,
)
gateway = _ensure_gateway(
config,
+31
View File
@@ -994,6 +994,37 @@ def test_status_keeps_live_state_when_identity_probe_is_temporarily_unavailable(
assert runtime.paths.state_path.exists()
def test_status_distinguishes_live_process_from_degraded_gateway_readiness(
tmp_path,
monkeypatch,
):
runtime = GatewayRuntime(paths=_paths(tmp_path), platform_name="Linux")
runtime.paths.run_dir.mkdir(parents=True)
runtime.paths.state_path.write_text(
json.dumps(
{
"pid": 12345,
"identity": 42,
"port": 18791,
"health_host": "127.0.0.1",
}
),
encoding="utf-8",
)
monkeypatch.setattr(runtime, "_is_pid_running", lambda _pid: True)
monkeypatch.setattr(runtime, "_process_identity", lambda _pid: 42)
monkeypatch.setattr(
"nanobot.gateway.runtime._gateway_health_ready",
lambda _host, _port: False,
)
status = runtime.status()
assert status.running is True
assert status.ready is False
assert status.reason == "websocket_unavailable"
def test_stop_refuses_to_signal_a_process_when_identity_cannot_be_verified(
tmp_path,
monkeypatch,