From 3a62b0b744026758e5d747eec12e8ce76eeeb12c Mon Sep 17 00:00:00 2001 From: chengyongru <61816729+chengyongru@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:37:30 +0800 Subject: [PATCH] fix(tui): surface chat connection failures (#5543) * fix(tui): surface chat connection failures * fix(tui): keep connection copy implementation-agnostic * test(tui): avoid millisecond timing boundary * fix(tui): use gateway health for connection status * fix(tui): use product-level readiness copy --- nanobot/cli/tui_launcher.py | 5 + tests/cli/test_tui_launcher.py | 1 + tui/src/app.test.ts | 112 ++++++++++++++- tui/src/app.ts | 105 ++++++++++---- tui/src/index.ts | 2 + tui/src/protocol.test.ts | 197 ++++++++++++++++++++++++++- tui/src/protocol.ts | 241 ++++++++++++++++++++++++++++++--- 7 files changed, 615 insertions(+), 48 deletions(-) diff --git a/nanobot/cli/tui_launcher.py b/nanobot/cli/tui_launcher.py index 4d4075709..b0ddfb487 100644 --- a/nanobot/cli/tui_launcher.py +++ b/nanobot/cli/tui_launcher.py @@ -21,6 +21,7 @@ from nanobot.cli.process_identity import named_executable from nanobot.cli.runtime_config import _model_display from nanobot.cli.webui_support import ( _gateway_health_ready, + _gateway_health_url, _gateway_instance_command, _host_for_local_browser, _webui_endpoint_reachable, @@ -96,6 +97,10 @@ def launch_tui( env.update( { "NANOBOT_TUI_BOOTSTRAP_URL": f"{base_url}/webui/bootstrap", + "NANOBOT_TUI_HEALTH_URL": _gateway_health_url( + config.gateway.host, + config.gateway.port, + ), "NANOBOT_TUI_API_URL": base_url, "NANOBOT_TUI_MODEL": _model_display(config)[0], "NANOBOT_TUI_MODEL_PRESET": config.agents.defaults.model_preset or "default", diff --git a/tests/cli/test_tui_launcher.py b/tests/cli/test_tui_launcher.py index a4b7fede2..5cc1f4a65 100644 --- a/tests/cli/test_tui_launcher.py +++ b/tests/cli/test_tui_launcher.py @@ -142,6 +142,7 @@ def test_launcher_passes_the_canonical_model_preset_to_the_tui( assert captured["NANOBOT_TUI_BOOTSTRAP_URL"] == ( "http://127.0.0.1:8765/webui/bootstrap" ) + assert captured["NANOBOT_TUI_HEALTH_URL"] == "http://127.0.0.1:18790/health" assert captured["NANOBOT_TUI_BOOTSTRAP_SECRET"] == "bootstrap-secret" assert "NANOBOT_TUI_WS_URL" not in captured assert "NANOBOT_TUI_API_TOKEN" not in captured diff --git a/tui/src/app.test.ts b/tui/src/app.test.ts index b5a1b47ac..b170efb9b 100644 --- a/tui/src/app.test.ts +++ b/tui/src/app.test.ts @@ -182,7 +182,7 @@ describe("NanobotTui layout", () => { expect(setup.renderer.height).toBe(height) expect(occurrences(frame, "Ask nanobot anything")).toBe(1) expect(occurrences(frame, "Ready")).toBe(0) - expect(occurrences(frame, "Connecting…")).toBe(1) + expect(occurrences(frame, "Getting ready…")).toBe(1) expect(occurrences(frame, "nanobot · test/model")).toBe(1) } @@ -2384,6 +2384,85 @@ describe("NanobotTui layout", () => { expect(state()).toBe(false) }) + test("shows actionable connection states without implementation details", async () => { + setup = await createRenderer({ width: 100, height: 20, screenMode: "alternate-screen" }) + const app = mount(setup) + const ui = app as unknown as { + status: TextRenderable + handleStatus( + status: "starting" | "connecting" | "connected" | "reconnecting" | "unavailable" | "error", + detail?: string, + info?: { + endpoint: string + attempt: number + elapsedMs: number + health?: "ready" | "degraded" | "unreachable" + }, + ): void + } + + ui.handleStatus("starting", undefined, { + endpoint: "127.0.0.1:8769", + attempt: 1, + elapsedMs: 0, + }) + expect(ui.status.plainText).toBe("Getting ready…") + + ui.handleStatus("connecting") + expect(ui.status.plainText).toBe("Getting ready…") + + ui.handleStatus("connected") + expect(ui.status.plainText).toBe("Getting ready…") + + ui.handleStatus("error", "gateway sent an invalid event") + expect(ui.status.plainText).toBe("Getting ready…") + expect(ui.status.plainText).not.toContain("Unable") + + ui.handleStatus("reconnecting", "connection closed", { + endpoint: "127.0.0.1:8769", + attempt: 2, + elapsedMs: 800, + }) + expect(ui.status.plainText).toBe("Resuming…") + + ui.handleStatus("reconnecting", "connection closed", { + endpoint: "127.0.0.1:8769", + attempt: 2, + elapsedMs: 900, + health: "degraded", + }) + expect(ui.status.plainText).toBe("Resuming…") + + ui.handleStatus("unavailable", "connection refused", { + endpoint: "127.0.0.1:8769", + attempt: 7, + elapsedMs: 3_200, + health: "degraded", + }) + expect(ui.status.plainText).toBe("Still getting ready…") + expect(ui.status.plainText).not.toContain("Unable") + + ui.handleStatus("unavailable", "connection refused", { + endpoint: "127.0.0.1:8769", + attempt: 8, + elapsedMs: 3_500, + health: "unreachable", + }) + expect(ui.status.plainText).toBe("Nanobot is taking longer to respond…") + expect(ui.status.plainText).not.toContain("Unable") + + ui.handleStatus("error", "gateway bootstrap failed: HTTP 401", { + endpoint: "127.0.0.1:8769", + attempt: 9, + elapsedMs: 3_800, + }) + expect(ui.status.plainText).toBe("Nanobot unavailable · restart nanobot") + expect(ui.status.plainText).not.toContain("gateway") + expect(ui.status.plainText).not.toContain("127.0.0.1") + expect(ui.status.plainText).not.toContain("HTTP") + expect(ui.status.plainText).not.toContain("attempt") + }) + test("replays events after asynchronous history hydration", async () => { setup = await createRenderer({ width: 80, height: 22, screenMode: "alternate-screen" }) const original = globalThis.fetch @@ -2444,7 +2523,12 @@ describe("NanobotTui layout", () => { client(sent), new MockTreeSitterClient({ autoResolveTimeout: 0 }), ) - const composer = (app as unknown as { composer: TextareaRenderable }).composer + const ui = app as unknown as { + composer: TextareaRenderable + ready: boolean + status: TextRenderable + } + const composer = ui.composer try { app.accept({ event: "attached", chat_id: "chat" }) @@ -2452,16 +2536,18 @@ describe("NanobotTui layout", () => { app.accept({ event: "attached", chat_id: "chat" }) composer.setText("sent during reconnect") composer.submit() - await Bun.sleep(5) + await waitUntil(() => ui.status.plainText.includes("Not sent")) expect(sent).toEqual([]) expect(composer.plainText).toBe("sent during reconnect") + expect(ui.status.plainText).toContain("Not sent · press Enter to retry when ready") resolveReconnect(new Response(JSON.stringify({ messages: [{ role: "assistant", content: "restored history" }], page: { has_more_before: false }, }))) - await waitUntil(() => (app as unknown as { ready: boolean }).ready) + await waitUntil(() => ui.ready) + expect(ui.status.plainText).toBe("Not sent · press Enter to retry") composer.submit() await waitUntil(() => sent.length === 1) await setup.flush() @@ -2480,24 +2566,38 @@ describe("NanobotTui layout", () => { const app = mount(setup, sent) const composer = (app as unknown as { composer: TextareaRenderable }).composer const connection = app as unknown as { - handleStatus(status: "connecting" | "connected", detail?: string): void + handleStatus( + status: "reconnecting" | "connected", + detail?: string, + info?: { endpoint: string; attempt: number; elapsedMs: number }, + ): void } app.accept({ event: "attached", chat_id: "chat" }) await Bun.sleep(1) - connection.handleStatus("connecting", "reconnecting") + connection.handleStatus("reconnecting", "connection closed", { + endpoint: "127.0.0.1:8769", + attempt: 1, + elapsedMs: 0, + }) connection.handleStatus("connected") composer.setText("draft before attach") composer.submit() await Bun.sleep(5) + composer.submit() + await Bun.sleep(5) expect(sent).toEqual([]) expect(composer.plainText).toBe("draft before attach") + app.accept({ event: "attached", chat_id: "chat" }) app.accept({ event: "attached", chat_id: "chat" }) await waitUntil(() => (app as unknown as { ready: boolean }).ready) + expect(sent).toEqual([]) composer.submit() await waitUntil(() => sent.length === 1) + app.accept({ event: "attached", chat_id: "chat" }) + await Bun.sleep(5) expect(sent).toEqual(["draft before attach"]) }) diff --git a/tui/src/app.ts b/tui/src/app.ts index f12716c63..20689a14e 100644 --- a/tui/src/app.ts +++ b/tui/src/app.ts @@ -20,7 +20,9 @@ import { import { NanobotClient, + connectionEndpoint, fetchAvailableSkills, + fetchGatewayHealth, fetchHistory, fetchGatewayConnection, fetchMentionCandidates, @@ -29,6 +31,7 @@ import { fetchSlashCommands, type ApiReauthenticator, type ConnectionStatus, + type ConnectionStatusInfo, type FileEditEvent, type GatewayApiConnection, type HistoryMessage, @@ -93,6 +96,7 @@ interface AppOptions { wsUrl?: string bootstrapUrl?: string bootstrapSecret?: string + healthUrl?: string apiUrl: string apiToken: string chatId?: string @@ -374,6 +378,21 @@ function formatElapsed(milliseconds: number): string { return `${Math.floor(seconds / 60)}m ${String(seconds % 60).padStart(2, "0")}s` } +function connectionStatusText( + status: ConnectionStatus, + info?: ConnectionStatusInfo, +): string { + if (["starting", "connecting", "connected"].includes(status)) return "Getting ready…" + if (status === "reconnecting") return "Resuming…" + if (status === "unavailable") { + return info?.health === "degraded" + ? "Still getting ready…" + : "Nanobot is taking longer to respond…" + } + if (status === "error") return "Nanobot unavailable · restart nanobot" + return "Session ended" +} + function singleLine(value: string, limit = 120): string { return value.replace(/\s+/gu, " ").trim().slice(0, limit) } @@ -449,6 +468,8 @@ export class NanobotTui { private shimmerTimer: ReturnType | null = null private submitPending = false private submitGeneration = 0 + private unsentSubmit = false + private connectionMessage = "Getting ready…" private readonly promptHistory: string[] = [] private historyCursor = 0 private historyDraft = "" @@ -556,11 +577,14 @@ export class NanobotTui { options.apiUrl, `tui-${process.pid}`, ), + ...(options.healthUrl + ? { checkHealth: () => fetchGatewayHealth(options.healthUrl || "") } + : {}), onConnection: (connection) => this.useGatewayConnection( connection.apiUrl, connection.apiToken, ), - connectionRetryLabel: "Starting local gateway", + targetEndpoint: connectionEndpoint(options.bootstrapUrl), reconnectDelayMs: 100, startupRetryMaxDelayMs: 250, } @@ -571,7 +595,7 @@ export class NanobotTui { access_mode: options.access.toLocaleLowerCase().includes("full") ? "full" : "restricted", }, onEvent: (event) => this.accept(event), - onStatus: (status, detail) => this.handleStatus(status, detail), + onStatus: (status, detail, info) => this.handleStatus(status, detail, info), }) // The terminal owns its canvas. Keeping the default-background intent is @@ -723,6 +747,8 @@ export class NanobotTui { }, onContentChange: () => { this.draft.prune(this.composer.plainText) + const clearedUnsent = this.unsentSubmit && !this.composer.plainText.trim() + if (clearedUnsent) this.unsentSubmit = false this.runtimeControls.hide() if (this.contextPanel.visible && this.composer.plainText) this.contextPanel.hide() this.syncComposerPlaceholder() @@ -730,6 +756,9 @@ export class NanobotTui { else if (this.branchMenu.visible) this.syncBranchMenu() else this.syncComposerMenus() this.resizeComposer() + if (clearedUnsent && !this.activeTurn) { + this.status.content = this.ready ? this.readyStatus() : this.connectionMessage + } }, // IMEs may commit their final composed glyph after Enter. Matching the // OpenCode/OpenTUI integration, defer twice before reading plainText. @@ -738,7 +767,7 @@ export class NanobotTui { }) this.status = new TextRenderable(renderer, { id: "nanobot-tui-status", - content: "Connecting…", + content: "Getting ready…", fg: this.palette.muted, height: 1, width: "auto", @@ -824,7 +853,7 @@ export class NanobotTui { // Network setup and small menu payloads do not depend on terminal colors. // Start them while OSC theme detection is in flight instead of serializing // up to one second of otherwise independent startup work. - this.host.reportState("unknown", "Connecting") + this.host.reportState("unknown", "Getting ready") this.client.connect() void this.loadCommands() void this.loadMentions() @@ -899,6 +928,10 @@ export class NanobotTui { return } if (["/continue", "/dismiss"].includes(visibleContent.toLowerCase())) { + if (!this.ready) { + this.markSubmitUnsent() + return + } this.clearComposer() this.commandMenu.hide() void this.updateRecovery(visibleContent.toLowerCase() === "/continue" ? "continue" : "dismiss") @@ -932,7 +965,7 @@ export class NanobotTui { return } if (!this.ready) { - this.status.content = "Preparing chat…" + this.markSubmitUnsent() return } const prompt = { content, options: mentionOptions(content, this.availableMentions()) } @@ -947,10 +980,11 @@ export class NanobotTui { let turnId: string try { turnId = this.client.send(prompt.content, prompt.options) - } catch (error) { - this.status.content = error instanceof Error ? error.message : String(error) + } catch { + this.markSubmitUnsent(true) return false } + this.unsentSubmit = false this.clearComposer() this.commandMenu.hide() this.mentionMenu.hide() @@ -1392,36 +1426,58 @@ export class NanobotTui { } } - private handleStatus(status: ConnectionStatus, detail?: string): void { + private handleStatus( + status: ConnectionStatus, + _detail?: string, + info?: ConnectionStatusInfo, + ): void { + // Invalid frames do not mean the transport is unavailable. Keep the last + // accurate user-facing state unless the protocol supplied connection diagnostics. + if (status === "error" && !info) return + this.connectionMessage = connectionStatusText(status, info) if (status === "connected") { this.ready = false - this.host.reportState("unknown", "Connecting") - this.status.content = "Connected · preparing chat…" + this.host.reportState("unknown", "Getting ready") + this.renderConnectionMessage() return } - if (status === "connecting") { + if (["starting", "connecting", "reconnecting", "unavailable"].includes(status)) { this.ready = false - const label = detail === "Starting local gateway" - ? detail - : detail ? "Reconnecting" : "Connecting" - this.host.reportState("unknown", label) - if (detail) this.setActive(false) - this.status.content = `${label}…` + this.host.reportState("unknown", this.connectionMessage) + if (status === "reconnecting" || status === "unavailable") this.setActive(false) + this.renderConnectionMessage() return } if (status === "error") { + if (info) this.ready = false this.setActive(false) - this.host.reportState("unknown", detail || "Connection error") - this.status.content = detail || "Connection error" + this.host.reportState("unknown", this.connectionMessage) + this.renderConnectionMessage() return } if (!this.quitting) { + this.ready = false this.setActive(false) this.host.reportState("unknown", "Disconnected") - this.status.content = "Disconnected" + this.renderConnectionMessage() } } + private renderConnectionMessage(): void { + this.status.content = this.unsentSubmit + ? `Not sent · press Enter to retry when ready · ${this.connectionMessage}` + : this.connectionMessage + } + + private markSubmitUnsent(sendFailed = false): void { + this.unsentSubmit = true + if (sendFailed) { + this.status.content = "Not sent · send failed; press Enter to retry when ready" + return + } + this.renderConnectionMessage() + } + private setActive(active: boolean, startedAt?: number): void { if (this.activeTurn === active) { if (active && startedAt !== undefined) this.activeStartedAt = startedAt @@ -1460,6 +1516,7 @@ export class NanobotTui { } private readyStatus(detail = this.readyDetail): string { + if (this.unsentSubmit) return "Not sent · press Enter to retry" if (this.transcriptNavigation.awayFromBottom) { return this.transcriptNavigation.unseenOutput ? "New output · Ctrl+End latest" @@ -2083,6 +2140,7 @@ export class NanobotTui { } private clearComposer(): void { + this.unsentSubmit = false this.draft.clear() this.composer.setText("") } @@ -2383,7 +2441,7 @@ export class NanobotTui { options: MessageOptions = {}, ): void { if (!this.ready) { - this.status.content = "Preparing chat…" + this.markSubmitUnsent() return } if (this.activeTurn && lifecycle === "agent_turn") { @@ -2393,10 +2451,11 @@ export class NanobotTui { let turnId: string try { turnId = this.client.send(content, options) - } catch (error) { - this.status.content = error instanceof Error ? error.message : String(error) + } catch { + this.markSubmitUnsent(true) return } + this.unsentSubmit = false this.commandTurns.set(turnId, lifecycle) if (silent) this.silentCommandTurns.add(turnId) if (/^\/model(?:\s|$)/iu.test(content)) this.modelCommandTurns.add(turnId) diff --git a/tui/src/index.ts b/tui/src/index.ts index f9b2d07fe..e78eb5112 100644 --- a/tui/src/index.ts +++ b/tui/src/index.ts @@ -14,6 +14,7 @@ const workspace = process.env.NANOBOT_TUI_WORKSPACE?.trim() || "" const hostWorkspace = process.cwd() const bootstrapUrl = process.env.NANOBOT_TUI_BOOTSTRAP_URL?.trim() || "" const wsUrl = process.env.NANOBOT_TUI_WS_URL?.trim() || "" +const healthUrl = process.env.NANOBOT_TUI_HEALTH_URL?.trim() || "" const gatewayStopCommand = process.env.NANOBOT_TUI_GATEWAY_STOP_COMMAND?.trim() || "nanobot gateway stop" if (!bootstrapUrl && !wsUrl) { @@ -24,6 +25,7 @@ const options: AppOptions = { ? { bootstrapUrl, bootstrapSecret: process.env.NANOBOT_TUI_BOOTSTRAP_SECRET?.trim() || "", + healthUrl: healthUrl || undefined, } : { wsUrl }), apiUrl: process.env.NANOBOT_TUI_API_URL?.trim() || "", diff --git a/tui/src/protocol.test.ts b/tui/src/protocol.test.ts index edb5c4e50..bb3d8412e 100644 --- a/tui/src/protocol.test.ts +++ b/tui/src/protocol.test.ts @@ -3,14 +3,19 @@ import { describe, expect, test } from "bun:test" import { NanobotClient, GatewayConnectionError, + connectionEndpoint, fetchAvailableSkills, fetchGatewayConnection, + fetchGatewayHealth, fetchHistory, fetchMentionCandidates, fetchRuntimeControls, fetchSessionContext, fetchSessions, fetchSlashCommands, + sanitizeConnectionFailure, + type ConnectionStatus, + type ConnectionStatusInfo, type InboundEvent, } from "./protocol" @@ -39,6 +44,12 @@ class FakeSocket { } } +async function waitUntil(predicate: () => boolean, timeout = 1_000): Promise { + const deadline = Date.now() + timeout + while (!predicate() && Date.now() < deadline) await Bun.sleep(2) + if (!predicate()) throw new Error(`condition was not met within ${timeout}ms`) +} + describe("gateway protocol", () => { test("bootstraps fresh websocket and API credentials", async () => { const original = globalThis.fetch @@ -70,6 +81,41 @@ describe("gateway protocol", () => { } }) + test("classifies gateway health without sending credentials", async () => { + const original = globalThis.fetch + const requests: Array<{ url: string; headers: Headers }> = [] + const responses = [ + new Response(JSON.stringify({ + status: "degraded", + process: "alive", + ready: false, + websocket: "unavailable", + }), { status: 503 }), + new Response(JSON.stringify({ + status: "ok", + process: "alive", + ready: true, + websocket: "running", + })), + new Response("not json"), + ] + globalThis.fetch = ((input: string | URL | Request, init?: RequestInit) => { + requests.push({ url: String(input), headers: new Headers(init?.headers) }) + return Promise.resolve(responses.shift() || new Response("missing", { status: 500 })) + }) as typeof fetch + + try { + const healthUrl = "http://127.0.0.1:18790/health" + expect(await fetchGatewayHealth(healthUrl)).toBe("degraded") + expect(await fetchGatewayHealth(healthUrl)).toBe("ready") + expect(await fetchGatewayHealth(healthUrl)).toBe("unreachable") + expect(requests.map(({ url }) => url)).toEqual([healthUrl, healthUrl, healthUrl]) + expect(requests.every(({ headers }) => [...headers].length === 0)).toBe(true) + } finally { + globalThis.fetch = original + } + }) + test("rejects malformed bootstrap responses without retrying", async () => { const original = globalThis.fetch globalThis.fetch = (() => Promise.resolve(new Response("not json"))) as unknown as typeof fetch @@ -138,8 +184,13 @@ describe("gateway protocol", () => { try { const connections: string[] = [] + let healthChecks = 0 const client = new NanobotClient({ resolveConnection: () => new Promise((resolve) => { resolveConnection = resolve }), + checkHealth: async () => { + healthChecks += 1 + return "ready" + }, onConnection: (connection) => connections.push(connection.apiToken), onEvent: () => undefined, onStatus: () => undefined, @@ -155,6 +206,7 @@ describe("gateway protocol", () => { await Bun.sleep(1) expect(requestedUrl).toBe("ws://nanobot.test/ws?token=fresh") expect(connections).toEqual(["fresh-api-token"]) + expect(healthChecks).toBe(0) client.close() } finally { Object.defineProperty(globalThis, "WebSocket", { configurable: true, value: original }) @@ -200,6 +252,133 @@ describe("gateway protocol", () => { } }) + test("escalates a refused bootstrap with safe endpoint and retry diagnostics", async () => { + const original = globalThis.fetch + const bootstrapUrl = "http://bootstrap-user:bootstrap-pass@127.0.0.1:8769" + + "/webui/bootstrap?token=socket-secret" + const bootstrapSecret = "bootstrap-secret" + const statuses: Array<{ + status: ConnectionStatus + detail?: string + info?: ConnectionStatusInfo + }> = [] + const refused = new TypeError( + `fetch failed for ${bootstrapUrl}&api_token=api-secret`, + { + cause: Object.assign(new Error("connect ECONNREFUSED 127.0.0.1:8769"), { + code: "ECONNREFUSED", + }), + }, + ) + globalThis.fetch = (() => Promise.reject(refused)) as unknown as typeof fetch + const client = new NanobotClient({ + resolveConnection: () => fetchGatewayConnection( + bootstrapUrl, + bootstrapSecret, + "http://127.0.0.1:8769", + "tui-42", + ), + targetEndpoint: connectionEndpoint(bootstrapUrl), + checkHealth: async () => "degraded", + startupFailureDelayMs: 8, + reconnectDelayMs: 100, + onEvent: () => undefined, + onStatus: (status, detail, info) => statuses.push({ status, detail, info }), + }) + + try { + client.connect() + await waitUntil(() => statuses.some(({ status }) => status === "unavailable")) + const failure = [...statuses].reverse().find(({ status }) => status === "unavailable") + + expect(statuses[0]?.status).toBe("starting") + expect(failure?.detail).toBe("connection refused") + expect(failure?.info).toMatchObject({ + endpoint: "127.0.0.1:8769", + attempt: 1, + elapsedMs: expect.any(Number), + health: "degraded", + }) + const visible = JSON.stringify(statuses) + expect(visible).not.toContain("bootstrap-user") + expect(visible).not.toContain("bootstrap-pass") + expect(visible).not.toContain(bootstrapSecret) + expect(visible).not.toContain("socket-secret") + expect(visible).not.toContain("api-secret") + expect(visible).not.toContain("/webui/bootstrap") + } finally { + client.close() + globalThis.fetch = original + } + }) + + test("recovers after sustained bootstrap failures without hiding the outage", async () => { + const original = globalThis.WebSocket + const sockets: FakeSocket[] = [] + let available = false + let attempts = 0 + const statuses: ConnectionStatus[] = [] + Object.defineProperty(globalThis, "WebSocket", { + configurable: true, + value: class extends FakeSocket { + constructor() { + super() + sockets.push(this) + } + }, + }) + const client = new NanobotClient({ + resolveConnection: async () => { + attempts += 1 + if (!available) { + throw Object.assign(new Error("connect ECONNREFUSED"), { code: "ECONNREFUSED" }) + } + return { + wsUrl: "ws://127.0.0.1:8769/ws?token=fresh", + apiUrl: "http://127.0.0.1:8769", + apiToken: "fresh-api-token", + } + }, + targetEndpoint: "127.0.0.1:8769", + checkHealth: async () => available ? "ready" : "degraded", + startupFailureDelayMs: 8, + reconnectDelayMs: 2, + startupRetryMaxDelayMs: 2, + onEvent: () => undefined, + onStatus: (status) => statuses.push(status), + }) + + try { + client.connect() + await waitUntil(() => statuses.includes("unavailable")) + available = true + await waitUntil(() => sockets.length === 1) + sockets[0]?.emit("open") + await waitUntil(() => statuses.at(-1) === "connected") + + expect(attempts).toBeGreaterThan(1) + expect(statuses.indexOf("starting")).toBeLessThan(statuses.indexOf("unavailable")) + expect(statuses.indexOf("unavailable")).toBeLessThan(statuses.lastIndexOf("connected")) + } finally { + client.close() + Object.defineProperty(globalThis, "WebSocket", { configurable: true, value: original }) + } + }) + + test("sanitizes arbitrary connection errors and authenticated URLs", () => { + const authenticated = "wss://user:password@127.0.0.1:8769/ws" + + "?token=socket-secret&api_token=api-secret" + const unknown = new Error(`could not reach ${authenticated}`) + const refused = Object.assign(new Error(`ECONNREFUSED ${authenticated}`), { + code: "ECONNREFUSED", + }) + + expect(connectionEndpoint(authenticated)).toBe("127.0.0.1:8769") + expect(sanitizeConnectionFailure(unknown)).toBe("connection failed") + expect(sanitizeConnectionFailure(refused)).toBe("connection refused") + expect(sanitizeConnectionFailure(unknown)).not.toContain("socket-secret") + }) + test("reports a permanent bootstrap rejection without retrying", async () => { let attempts = 0 const statuses: string[] = [] @@ -588,13 +767,19 @@ describe("gateway protocol", () => { }) try { + const statuses: Array<{ + status: ConnectionStatus + detail?: string + info?: ConnectionStatusInfo + }> = [] const client = new NanobotClient({ url: "ws://nanobot.test/ws", reconnectDelayMs: 1, onEvent: () => undefined, - onStatus: () => undefined, + onStatus: (status, detail, info) => statuses.push({ status, detail, info }), }) client.connect() + sockets[0]?.emit("open") sockets[0]?.emit("message", { data: JSON.stringify({ event: "ready", chat_id: "", client_id: "client" }), }) @@ -605,11 +790,21 @@ describe("gateway protocol", () => { await Bun.sleep(5) expect(sockets).toHaveLength(2) + const reconnecting = [...statuses].reverse().find( + ({ status }) => status === "reconnecting", + ) + expect(reconnecting).toMatchObject({ + status: "reconnecting", + detail: "connection closed", + info: { endpoint: "nanobot.test", attempt: 1 }, + }) + sockets[1]?.emit("open") sockets[1]?.emit("message", { data: JSON.stringify({ event: "ready", chat_id: "", client_id: "client-2" }), }) const outbound = sockets[1]?.sent.map((value) => JSON.parse(value)) || [] expect(outbound).toEqual([{ type: "attach", chat_id: "generated-chat" }]) + expect(statuses.at(-1)?.status).toBe("connected") client.close() } finally { Object.defineProperty(globalThis, "WebSocket", { configurable: true, value: original }) diff --git a/tui/src/protocol.ts b/tui/src/protocol.ts index de5e6c0b1..f895e9515 100644 --- a/tui/src/protocol.ts +++ b/tui/src/protocol.ts @@ -1,4 +1,21 @@ -export type ConnectionStatus = "connecting" | "connected" | "closed" | "error" +export type ConnectionStatus = + | "starting" + | "connecting" + | "connected" + | "reconnecting" + | "unavailable" + | "closed" + | "error" + +export interface ConnectionStatusInfo { + endpoint: string + attempt: number + elapsedMs: number + retryInMs?: number + health?: GatewayHealthStatus +} + +export type GatewayHealthStatus = "ready" | "degraded" | "unreachable" export interface ToolProgressEvent { version?: number @@ -172,14 +189,16 @@ type OutboundEvent = export interface ClientOptions { url?: string resolveConnection?: () => Promise + checkHealth?: () => Promise onConnection?: (connection: GatewayConnection) => void - connectionRetryLabel?: string + targetEndpoint?: string + startupFailureDelayMs?: number startupRetryMaxDelayMs?: number chatId?: string initialWorkspaceScope?: WorkspaceScopePayload reconnectDelayMs?: number onEvent: (event: InboundEvent) => void - onStatus: (status: ConnectionStatus, detail?: string) => void + onStatus: (status: ConnectionStatus, detail?: string, info?: ConnectionStatusInfo) => void } export interface GatewayApiConnection { @@ -940,6 +959,92 @@ export async function fetchGatewayConnection( } } +/** Read gateway readiness without sending bootstrap or API credentials. */ +export async function fetchGatewayHealth( + healthUrl: string, + timeoutMs = 400, +): Promise { + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), timeoutMs) + try { + const response = await fetch(healthUrl, { signal: controller.signal }) + if (response.status !== 200 && response.status !== 503) return "unreachable" + const payload: unknown = await response.json() + if (!isRecord(payload)) return "unreachable" + if ( + response.status === 503 + && payload.status === "degraded" + && payload.ready === false + && payload.process === "alive" + ) return "degraded" + if (response.status === 200 && payload.status === "ok" && payload.ready !== false) { + return "ready" + } + return "unreachable" + } catch { + return "unreachable" + } finally { + clearTimeout(timer) + } +} + +/** Return only the authority users can act on, never credentials or an authenticated path. */ +export function connectionEndpoint(value: string | undefined): string { + if (!value) return "local gateway" + try { + return new URL(value).host || "local gateway" + } catch { + return "local gateway" + } +} + +/** Reduce arbitrary fetch/WebSocket errors to a small set of credential-safe reasons. */ +export function sanitizeConnectionFailure(error: unknown): string { + const signals: string[] = [] + const seen = new Set() + const collect = (value: unknown): void => { + if (value === null || value === undefined || seen.has(value)) return + if (typeof value === "object") seen.add(value) + if (typeof value === "string") { + signals.push(value) + return + } + if (value instanceof Error) { + signals.push(value.name, value.message) + collect(value.cause) + if (value instanceof AggregateError) { + for (const nested of value.errors) collect(nested) + } + return + } + if (!isRecord(value)) return + if (typeof value.code === "string") signals.push(value.code) + collect(value.cause) + if (Array.isArray(value.errors)) { + for (const nested of value.errors) collect(nested) + } + } + collect(error) + const signal = signals.join(" ") + if (/ECONNREFUSED|connection refused/iu.test(signal)) return "connection refused" + if (/ETIMEDOUT|timed? out|timeout/iu.test(signal)) return "connection timed out" + if (/ENOTFOUND|EAI_AGAIN|name not resolved|host not found/iu.test(signal)) { + return "host not found" + } + if (/certificate|TLS|SSL/iu.test(signal)) return "secure connection failed" + const bootstrapStatus = signal.match(/gateway bootstrap failed:\s*HTTP\s*(\d{3})/iu) + if (bootstrapStatus?.[1]) return `gateway bootstrap failed: HTTP ${bootstrapStatus[1]}` + if (/bootstrap response is missing ws_url/iu.test(signal)) { + return "gateway bootstrap response is missing ws_url" + } + if (/bootstrap response (?:has an invalid ws_url|is invalid)/iu.test(signal)) { + return "gateway bootstrap response is invalid" + } + if (/gateway is still starting/iu.test(signal)) return "gateway is still starting" + if (/fetch failed|failed to fetch|network error/iu.test(signal)) return "network request failed" + return "connection failed" +} + export class NanobotClient { private socket: WebSocket | null = null private chatId = "" @@ -949,13 +1054,22 @@ export class NanobotClient { private closedByClient = false private opening = false private connectedOnce = false + private connectionAttempt = 0 + private retryStartedAt = 0 + private nextRetryAt = 0 + private lastFailure = "" + private healthStatus: GatewayHealthStatus | undefined + private failureEscalationTimer: ReturnType | null = null + private readonly endpoint: string private readonly pendingMutations = new Map void reject: (error: Error) => void timer: ReturnType }>() - constructor(private readonly options: ClientOptions) {} + constructor(private readonly options: ClientOptions) { + this.endpoint = options.targetEndpoint || connectionEndpoint(options.url) + } get activeChatId(): string { return this.chatId @@ -963,13 +1077,21 @@ export class NanobotClient { connect(): void { this.closedByClient = false + this.connectionAttempt = 0 + this.reconnectAttempt = 0 + this.retryStartedAt = Date.now() + this.nextRetryAt = 0 + this.lastFailure = "" + this.healthStatus = undefined void this.open() } private async open(): Promise { if (this.socket || this.opening || this.closedByClient) return this.opening = true - this.options.onStatus("connecting") + this.nextRetryAt = 0 + this.connectionAttempt += 1 + this.reportConnectionProgress() let url = this.options.url try { if (this.options.resolveConnection) { @@ -980,37 +1102,52 @@ export class NanobotClient { } } catch (error) { if (!this.closedByClient) { + this.lastFailure = sanitizeConnectionFailure(error) if (error instanceof GatewayConnectionError && !error.retryable) { - this.options.onStatus("error", error.message) + this.clearFailureEscalation() + this.options.onStatus("error", this.lastFailure, this.connectionInfo()) return } - this.options.onStatus( - "connecting", - this.options.connectionRetryLabel || "gateway unavailable", - ) - this.scheduleReconnect(false) + await this.checkHealthAndScheduleReconnect() } return } finally { this.opening = false } if (!url) { - this.options.onStatus("error", "gateway URL is not configured") + this.options.onStatus("error", "gateway URL is not configured", this.connectionInfo()) return } - const socket = new WebSocket(url) + let socket: WebSocket + try { + socket = new WebSocket(url) + } catch (error) { + this.lastFailure = sanitizeConnectionFailure(error) + await this.checkHealthAndScheduleReconnect() + return + } + let opened = false this.socket = socket socket.addEventListener("open", () => { if (this.socket !== socket) return + opened = true this.connectedOnce = true + this.connectionAttempt = 0 this.reconnectAttempt = 0 - this.options.onStatus("connected") + this.retryStartedAt = 0 + this.nextRetryAt = 0 + this.lastFailure = "" + this.healthStatus = "ready" + this.clearFailureEscalation() + this.options.onStatus("connected", undefined, this.connectionInfo()) }) socket.addEventListener("message", (message) => { if (this.socket === socket) this.handleMessage(String(message.data)) }) socket.addEventListener("error", () => { - if (this.socket === socket) this.options.onStatus("error", "connection failed") + if (this.socket !== socket) return + this.lastFailure = "connection failed" + this.reportRetryState() }) socket.addEventListener("close", () => { if (this.socket !== socket) return @@ -1020,7 +1157,14 @@ export class NanobotClient { this.options.onStatus("closed") return } - this.scheduleReconnect() + if (opened) { + this.connectionAttempt = 0 + this.reconnectAttempt = 0 + this.retryStartedAt = Date.now() + } + if (!this.lastFailure) this.lastFailure = "connection closed" + this.reportRetryState() + void this.checkHealthAndScheduleReconnect() }) } @@ -1028,6 +1172,7 @@ export class NanobotClient { this.closedByClient = true if (this.reconnectTimer) clearTimeout(this.reconnectTimer) this.reconnectTimer = null + this.clearFailureEscalation() const socket = this.socket this.socket = null socket?.close() @@ -1182,20 +1327,80 @@ export class NanobotClient { this.options.onEvent(event) } - private scheduleReconnect(announce = true): void { + private scheduleReconnect(): void { if (this.reconnectTimer || this.closedByClient) return + if (!this.retryStartedAt) this.retryStartedAt = Date.now() const base = this.options.reconnectDelayMs ?? 500 const maxDelay = this.connectedOnce ? 8_000 : this.options.startupRetryMaxDelayMs ?? 8_000 const delay = Math.min(maxDelay, base * 2 ** Math.min(this.reconnectAttempt++, 4)) - if (announce) this.options.onStatus("connecting", `reconnecting in ${delay}ms`) + this.nextRetryAt = Date.now() + delay + this.reportRetryState() this.reconnectTimer = setTimeout(() => { this.reconnectTimer = null void this.open() }, delay) } + private async checkHealthAndScheduleReconnect(): Promise { + if (this.options.checkHealth) { + try { + this.healthStatus = await this.options.checkHealth() + } catch { + this.healthStatus = "unreachable" + } + } + if (!this.closedByClient) this.scheduleReconnect() + } + + private connectionInfo(): ConnectionStatusInfo { + return { + endpoint: this.endpoint, + attempt: Math.max(1, this.connectionAttempt), + elapsedMs: this.retryStartedAt ? Math.max(0, Date.now() - this.retryStartedAt) : 0, + ...(this.nextRetryAt + ? { retryInMs: Math.max(0, this.nextRetryAt - Date.now()) } + : {}), + ...(this.healthStatus ? { health: this.healthStatus } : {}), + } + } + + private reportConnectionProgress(): void { + if (this.connectedOnce) { + this.options.onStatus("reconnecting", this.lastFailure || undefined, this.connectionInfo()) + return + } + const phase = this.options.resolveConnection ? "starting" : "connecting" + this.options.onStatus(phase, undefined, this.connectionInfo()) + } + + private reportRetryState(): void { + const info = this.connectionInfo() + if (this.connectedOnce) { + this.options.onStatus("reconnecting", this.lastFailure, info) + return + } + const failureDelay = this.options.startupFailureDelayMs ?? 3_000 + if (info.elapsedMs >= failureDelay) { + this.clearFailureEscalation() + this.options.onStatus("unavailable", this.lastFailure, info) + return + } + this.reportConnectionProgress() + if (this.failureEscalationTimer) return + this.failureEscalationTimer = setTimeout(() => { + this.failureEscalationTimer = null + if (this.closedByClient || this.connectedOnce || !this.lastFailure) return + this.options.onStatus("unavailable", this.lastFailure, this.connectionInfo()) + }, Math.max(0, failureDelay - info.elapsedMs)) + } + + private clearFailureEscalation(): void { + if (this.failureEscalationTimer) clearTimeout(this.failureEscalationTimer) + this.failureEscalationTimer = null + } + private write(event: OutboundEvent): void { if (!this.socket || this.socket.readyState !== WebSocket.OPEN) { throw new Error("gateway connection is not open")