fix(tui): surface chat connection failures (#5543)

* fix(tui): surface chat connection failures

* fix(tui): keep connection copy implementation-agnostic

* test(tui): avoid millisecond timing boundary

* fix(tui): use gateway health for connection status

* fix(tui): use product-level readiness copy
This commit is contained in:
chengyongru
2026-08-27 11:37:30 +08:00
committed by GitHub
parent 55f85b3c1f
commit 3a62b0b744
7 changed files with 615 additions and 48 deletions
+5
View File
@@ -21,6 +21,7 @@ from nanobot.cli.process_identity import named_executable
from nanobot.cli.runtime_config import _model_display from nanobot.cli.runtime_config import _model_display
from nanobot.cli.webui_support import ( from nanobot.cli.webui_support import (
_gateway_health_ready, _gateway_health_ready,
_gateway_health_url,
_gateway_instance_command, _gateway_instance_command,
_host_for_local_browser, _host_for_local_browser,
_webui_endpoint_reachable, _webui_endpoint_reachable,
@@ -96,6 +97,10 @@ def launch_tui(
env.update( env.update(
{ {
"NANOBOT_TUI_BOOTSTRAP_URL": f"{base_url}/webui/bootstrap", "NANOBOT_TUI_BOOTSTRAP_URL": f"{base_url}/webui/bootstrap",
"NANOBOT_TUI_HEALTH_URL": _gateway_health_url(
config.gateway.host,
config.gateway.port,
),
"NANOBOT_TUI_API_URL": base_url, "NANOBOT_TUI_API_URL": base_url,
"NANOBOT_TUI_MODEL": _model_display(config)[0], "NANOBOT_TUI_MODEL": _model_display(config)[0],
"NANOBOT_TUI_MODEL_PRESET": config.agents.defaults.model_preset or "default", "NANOBOT_TUI_MODEL_PRESET": config.agents.defaults.model_preset or "default",
+1
View File
@@ -142,6 +142,7 @@ def test_launcher_passes_the_canonical_model_preset_to_the_tui(
assert captured["NANOBOT_TUI_BOOTSTRAP_URL"] == ( assert captured["NANOBOT_TUI_BOOTSTRAP_URL"] == (
"http://127.0.0.1:8765/webui/bootstrap" "http://127.0.0.1:8765/webui/bootstrap"
) )
assert captured["NANOBOT_TUI_HEALTH_URL"] == "http://127.0.0.1:18790/health"
assert captured["NANOBOT_TUI_BOOTSTRAP_SECRET"] == "bootstrap-secret" assert captured["NANOBOT_TUI_BOOTSTRAP_SECRET"] == "bootstrap-secret"
assert "NANOBOT_TUI_WS_URL" not in captured assert "NANOBOT_TUI_WS_URL" not in captured
assert "NANOBOT_TUI_API_TOKEN" not in captured assert "NANOBOT_TUI_API_TOKEN" not in captured
+106 -6
View File
@@ -182,7 +182,7 @@ describe("NanobotTui layout", () => {
expect(setup.renderer.height).toBe(height) expect(setup.renderer.height).toBe(height)
expect(occurrences(frame, "Ask nanobot anything")).toBe(1) expect(occurrences(frame, "Ask nanobot anything")).toBe(1)
expect(occurrences(frame, "Ready")).toBe(0) expect(occurrences(frame, "Ready")).toBe(0)
expect(occurrences(frame, "Connecting…")).toBe(1) expect(occurrences(frame, "Getting ready…")).toBe(1)
expect(occurrences(frame, "nanobot · test/model")).toBe(1) expect(occurrences(frame, "nanobot · test/model")).toBe(1)
} }
@@ -2384,6 +2384,85 @@ describe("NanobotTui layout", () => {
expect(state()).toBe(false) expect(state()).toBe(false)
}) })
test("shows actionable connection states without implementation details", async () => {
setup = await createRenderer({ width: 100, height: 20, screenMode: "alternate-screen" })
const app = mount(setup)
const ui = app as unknown as {
status: TextRenderable
handleStatus(
status: "starting" | "connecting" | "connected" | "reconnecting" | "unavailable" | "error",
detail?: string,
info?: {
endpoint: string
attempt: number
elapsedMs: number
health?: "ready" | "degraded" | "unreachable"
},
): void
}
ui.handleStatus("starting", undefined, {
endpoint: "127.0.0.1:8769",
attempt: 1,
elapsedMs: 0,
})
expect(ui.status.plainText).toBe("Getting ready…")
ui.handleStatus("connecting")
expect(ui.status.plainText).toBe("Getting ready…")
ui.handleStatus("connected")
expect(ui.status.plainText).toBe("Getting ready…")
ui.handleStatus("error", "gateway sent an invalid event")
expect(ui.status.plainText).toBe("Getting ready…")
expect(ui.status.plainText).not.toContain("Unable")
ui.handleStatus("reconnecting", "connection closed", {
endpoint: "127.0.0.1:8769",
attempt: 2,
elapsedMs: 800,
})
expect(ui.status.plainText).toBe("Resuming…")
ui.handleStatus("reconnecting", "connection closed", {
endpoint: "127.0.0.1:8769",
attempt: 2,
elapsedMs: 900,
health: "degraded",
})
expect(ui.status.plainText).toBe("Resuming…")
ui.handleStatus("unavailable", "connection refused", {
endpoint: "127.0.0.1:8769",
attempt: 7,
elapsedMs: 3_200,
health: "degraded",
})
expect(ui.status.plainText).toBe("Still getting ready…")
expect(ui.status.plainText).not.toContain("Unable")
ui.handleStatus("unavailable", "connection refused", {
endpoint: "127.0.0.1:8769",
attempt: 8,
elapsedMs: 3_500,
health: "unreachable",
})
expect(ui.status.plainText).toBe("Nanobot is taking longer to respond…")
expect(ui.status.plainText).not.toContain("Unable")
ui.handleStatus("error", "gateway bootstrap failed: HTTP 401", {
endpoint: "127.0.0.1:8769",
attempt: 9,
elapsedMs: 3_800,
})
expect(ui.status.plainText).toBe("Nanobot unavailable · restart nanobot")
expect(ui.status.plainText).not.toContain("gateway")
expect(ui.status.plainText).not.toContain("127.0.0.1")
expect(ui.status.plainText).not.toContain("HTTP")
expect(ui.status.plainText).not.toContain("attempt")
})
test("replays events after asynchronous history hydration", async () => { test("replays events after asynchronous history hydration", async () => {
setup = await createRenderer({ width: 80, height: 22, screenMode: "alternate-screen" }) setup = await createRenderer({ width: 80, height: 22, screenMode: "alternate-screen" })
const original = globalThis.fetch const original = globalThis.fetch
@@ -2444,7 +2523,12 @@ describe("NanobotTui layout", () => {
client(sent), client(sent),
new MockTreeSitterClient({ autoResolveTimeout: 0 }), new MockTreeSitterClient({ autoResolveTimeout: 0 }),
) )
const composer = (app as unknown as { composer: TextareaRenderable }).composer const ui = app as unknown as {
composer: TextareaRenderable
ready: boolean
status: TextRenderable
}
const composer = ui.composer
try { try {
app.accept({ event: "attached", chat_id: "chat" }) app.accept({ event: "attached", chat_id: "chat" })
@@ -2452,16 +2536,18 @@ describe("NanobotTui layout", () => {
app.accept({ event: "attached", chat_id: "chat" }) app.accept({ event: "attached", chat_id: "chat" })
composer.setText("sent during reconnect") composer.setText("sent during reconnect")
composer.submit() composer.submit()
await Bun.sleep(5) await waitUntil(() => ui.status.plainText.includes("Not sent"))
expect(sent).toEqual([]) expect(sent).toEqual([])
expect(composer.plainText).toBe("sent during reconnect") expect(composer.plainText).toBe("sent during reconnect")
expect(ui.status.plainText).toContain("Not sent · press Enter to retry when ready")
resolveReconnect(new Response(JSON.stringify({ resolveReconnect(new Response(JSON.stringify({
messages: [{ role: "assistant", content: "restored history" }], messages: [{ role: "assistant", content: "restored history" }],
page: { has_more_before: false }, page: { has_more_before: false },
}))) })))
await waitUntil(() => (app as unknown as { ready: boolean }).ready) await waitUntil(() => ui.ready)
expect(ui.status.plainText).toBe("Not sent · press Enter to retry")
composer.submit() composer.submit()
await waitUntil(() => sent.length === 1) await waitUntil(() => sent.length === 1)
await setup.flush() await setup.flush()
@@ -2480,24 +2566,38 @@ describe("NanobotTui layout", () => {
const app = mount(setup, sent) const app = mount(setup, sent)
const composer = (app as unknown as { composer: TextareaRenderable }).composer const composer = (app as unknown as { composer: TextareaRenderable }).composer
const connection = app as unknown as { const connection = app as unknown as {
handleStatus(status: "connecting" | "connected", detail?: string): void handleStatus(
status: "reconnecting" | "connected",
detail?: string,
info?: { endpoint: string; attempt: number; elapsedMs: number },
): void
} }
app.accept({ event: "attached", chat_id: "chat" }) app.accept({ event: "attached", chat_id: "chat" })
await Bun.sleep(1) await Bun.sleep(1)
connection.handleStatus("connecting", "reconnecting") connection.handleStatus("reconnecting", "connection closed", {
endpoint: "127.0.0.1:8769",
attempt: 1,
elapsedMs: 0,
})
connection.handleStatus("connected") connection.handleStatus("connected")
composer.setText("draft before attach") composer.setText("draft before attach")
composer.submit() composer.submit()
await Bun.sleep(5) await Bun.sleep(5)
composer.submit()
await Bun.sleep(5)
expect(sent).toEqual([]) expect(sent).toEqual([])
expect(composer.plainText).toBe("draft before attach") expect(composer.plainText).toBe("draft before attach")
app.accept({ event: "attached", chat_id: "chat" })
app.accept({ event: "attached", chat_id: "chat" }) app.accept({ event: "attached", chat_id: "chat" })
await waitUntil(() => (app as unknown as { ready: boolean }).ready) await waitUntil(() => (app as unknown as { ready: boolean }).ready)
expect(sent).toEqual([])
composer.submit() composer.submit()
await waitUntil(() => sent.length === 1) await waitUntil(() => sent.length === 1)
app.accept({ event: "attached", chat_id: "chat" })
await Bun.sleep(5)
expect(sent).toEqual(["draft before attach"]) expect(sent).toEqual(["draft before attach"])
}) })
+82 -23
View File
@@ -20,7 +20,9 @@ import {
import { import {
NanobotClient, NanobotClient,
connectionEndpoint,
fetchAvailableSkills, fetchAvailableSkills,
fetchGatewayHealth,
fetchHistory, fetchHistory,
fetchGatewayConnection, fetchGatewayConnection,
fetchMentionCandidates, fetchMentionCandidates,
@@ -29,6 +31,7 @@ import {
fetchSlashCommands, fetchSlashCommands,
type ApiReauthenticator, type ApiReauthenticator,
type ConnectionStatus, type ConnectionStatus,
type ConnectionStatusInfo,
type FileEditEvent, type FileEditEvent,
type GatewayApiConnection, type GatewayApiConnection,
type HistoryMessage, type HistoryMessage,
@@ -93,6 +96,7 @@ interface AppOptions {
wsUrl?: string wsUrl?: string
bootstrapUrl?: string bootstrapUrl?: string
bootstrapSecret?: string bootstrapSecret?: string
healthUrl?: string
apiUrl: string apiUrl: string
apiToken: string apiToken: string
chatId?: string chatId?: string
@@ -374,6 +378,21 @@ function formatElapsed(milliseconds: number): string {
return `${Math.floor(seconds / 60)}m ${String(seconds % 60).padStart(2, "0")}s` return `${Math.floor(seconds / 60)}m ${String(seconds % 60).padStart(2, "0")}s`
} }
function connectionStatusText(
status: ConnectionStatus,
info?: ConnectionStatusInfo,
): string {
if (["starting", "connecting", "connected"].includes(status)) return "Getting ready…"
if (status === "reconnecting") return "Resuming…"
if (status === "unavailable") {
return info?.health === "degraded"
? "Still getting ready…"
: "Nanobot is taking longer to respond…"
}
if (status === "error") return "Nanobot unavailable · restart nanobot"
return "Session ended"
}
function singleLine(value: string, limit = 120): string { function singleLine(value: string, limit = 120): string {
return value.replace(/\s+/gu, " ").trim().slice(0, limit) return value.replace(/\s+/gu, " ").trim().slice(0, limit)
} }
@@ -449,6 +468,8 @@ export class NanobotTui {
private shimmerTimer: ReturnType<typeof setInterval> | null = null private shimmerTimer: ReturnType<typeof setInterval> | null = null
private submitPending = false private submitPending = false
private submitGeneration = 0 private submitGeneration = 0
private unsentSubmit = false
private connectionMessage = "Getting ready…"
private readonly promptHistory: string[] = [] private readonly promptHistory: string[] = []
private historyCursor = 0 private historyCursor = 0
private historyDraft = "" private historyDraft = ""
@@ -556,11 +577,14 @@ export class NanobotTui {
options.apiUrl, options.apiUrl,
`tui-${process.pid}`, `tui-${process.pid}`,
), ),
...(options.healthUrl
? { checkHealth: () => fetchGatewayHealth(options.healthUrl || "") }
: {}),
onConnection: (connection) => this.useGatewayConnection( onConnection: (connection) => this.useGatewayConnection(
connection.apiUrl, connection.apiUrl,
connection.apiToken, connection.apiToken,
), ),
connectionRetryLabel: "Starting local gateway", targetEndpoint: connectionEndpoint(options.bootstrapUrl),
reconnectDelayMs: 100, reconnectDelayMs: 100,
startupRetryMaxDelayMs: 250, startupRetryMaxDelayMs: 250,
} }
@@ -571,7 +595,7 @@ export class NanobotTui {
access_mode: options.access.toLocaleLowerCase().includes("full") ? "full" : "restricted", access_mode: options.access.toLocaleLowerCase().includes("full") ? "full" : "restricted",
}, },
onEvent: (event) => this.accept(event), onEvent: (event) => this.accept(event),
onStatus: (status, detail) => this.handleStatus(status, detail), onStatus: (status, detail, info) => this.handleStatus(status, detail, info),
}) })
// The terminal owns its canvas. Keeping the default-background intent is // The terminal owns its canvas. Keeping the default-background intent is
@@ -723,6 +747,8 @@ export class NanobotTui {
}, },
onContentChange: () => { onContentChange: () => {
this.draft.prune(this.composer.plainText) this.draft.prune(this.composer.plainText)
const clearedUnsent = this.unsentSubmit && !this.composer.plainText.trim()
if (clearedUnsent) this.unsentSubmit = false
this.runtimeControls.hide() this.runtimeControls.hide()
if (this.contextPanel.visible && this.composer.plainText) this.contextPanel.hide() if (this.contextPanel.visible && this.composer.plainText) this.contextPanel.hide()
this.syncComposerPlaceholder() this.syncComposerPlaceholder()
@@ -730,6 +756,9 @@ export class NanobotTui {
else if (this.branchMenu.visible) this.syncBranchMenu() else if (this.branchMenu.visible) this.syncBranchMenu()
else this.syncComposerMenus() else this.syncComposerMenus()
this.resizeComposer() this.resizeComposer()
if (clearedUnsent && !this.activeTurn) {
this.status.content = this.ready ? this.readyStatus() : this.connectionMessage
}
}, },
// IMEs may commit their final composed glyph after Enter. Matching the // IMEs may commit their final composed glyph after Enter. Matching the
// OpenCode/OpenTUI integration, defer twice before reading plainText. // OpenCode/OpenTUI integration, defer twice before reading plainText.
@@ -738,7 +767,7 @@ export class NanobotTui {
}) })
this.status = new TextRenderable(renderer, { this.status = new TextRenderable(renderer, {
id: "nanobot-tui-status", id: "nanobot-tui-status",
content: "Connecting…", content: "Getting ready…",
fg: this.palette.muted, fg: this.palette.muted,
height: 1, height: 1,
width: "auto", width: "auto",
@@ -824,7 +853,7 @@ export class NanobotTui {
// Network setup and small menu payloads do not depend on terminal colors. // Network setup and small menu payloads do not depend on terminal colors.
// Start them while OSC theme detection is in flight instead of serializing // Start them while OSC theme detection is in flight instead of serializing
// up to one second of otherwise independent startup work. // up to one second of otherwise independent startup work.
this.host.reportState("unknown", "Connecting") this.host.reportState("unknown", "Getting ready")
this.client.connect() this.client.connect()
void this.loadCommands() void this.loadCommands()
void this.loadMentions() void this.loadMentions()
@@ -899,6 +928,10 @@ export class NanobotTui {
return return
} }
if (["/continue", "/dismiss"].includes(visibleContent.toLowerCase())) { if (["/continue", "/dismiss"].includes(visibleContent.toLowerCase())) {
if (!this.ready) {
this.markSubmitUnsent()
return
}
this.clearComposer() this.clearComposer()
this.commandMenu.hide() this.commandMenu.hide()
void this.updateRecovery(visibleContent.toLowerCase() === "/continue" ? "continue" : "dismiss") void this.updateRecovery(visibleContent.toLowerCase() === "/continue" ? "continue" : "dismiss")
@@ -932,7 +965,7 @@ export class NanobotTui {
return return
} }
if (!this.ready) { if (!this.ready) {
this.status.content = "Preparing chat…" this.markSubmitUnsent()
return return
} }
const prompt = { content, options: mentionOptions(content, this.availableMentions()) } const prompt = { content, options: mentionOptions(content, this.availableMentions()) }
@@ -947,10 +980,11 @@ export class NanobotTui {
let turnId: string let turnId: string
try { try {
turnId = this.client.send(prompt.content, prompt.options) turnId = this.client.send(prompt.content, prompt.options)
} catch (error) { } catch {
this.status.content = error instanceof Error ? error.message : String(error) this.markSubmitUnsent(true)
return false return false
} }
this.unsentSubmit = false
this.clearComposer() this.clearComposer()
this.commandMenu.hide() this.commandMenu.hide()
this.mentionMenu.hide() this.mentionMenu.hide()
@@ -1392,36 +1426,58 @@ export class NanobotTui {
} }
} }
private handleStatus(status: ConnectionStatus, detail?: string): void { private handleStatus(
status: ConnectionStatus,
_detail?: string,
info?: ConnectionStatusInfo,
): void {
// Invalid frames do not mean the transport is unavailable. Keep the last
// accurate user-facing state unless the protocol supplied connection diagnostics.
if (status === "error" && !info) return
this.connectionMessage = connectionStatusText(status, info)
if (status === "connected") { if (status === "connected") {
this.ready = false this.ready = false
this.host.reportState("unknown", "Connecting") this.host.reportState("unknown", "Getting ready")
this.status.content = "Connected · preparing chat…" this.renderConnectionMessage()
return return
} }
if (status === "connecting") { if (["starting", "connecting", "reconnecting", "unavailable"].includes(status)) {
this.ready = false this.ready = false
const label = detail === "Starting local gateway" this.host.reportState("unknown", this.connectionMessage)
? detail if (status === "reconnecting" || status === "unavailable") this.setActive(false)
: detail ? "Reconnecting" : "Connecting" this.renderConnectionMessage()
this.host.reportState("unknown", label)
if (detail) this.setActive(false)
this.status.content = `${label}`
return return
} }
if (status === "error") { if (status === "error") {
if (info) this.ready = false
this.setActive(false) this.setActive(false)
this.host.reportState("unknown", detail || "Connection error") this.host.reportState("unknown", this.connectionMessage)
this.status.content = detail || "Connection error" this.renderConnectionMessage()
return return
} }
if (!this.quitting) { if (!this.quitting) {
this.ready = false
this.setActive(false) this.setActive(false)
this.host.reportState("unknown", "Disconnected") this.host.reportState("unknown", "Disconnected")
this.status.content = "Disconnected" this.renderConnectionMessage()
} }
} }
private renderConnectionMessage(): void {
this.status.content = this.unsentSubmit
? `Not sent · press Enter to retry when ready · ${this.connectionMessage}`
: this.connectionMessage
}
private markSubmitUnsent(sendFailed = false): void {
this.unsentSubmit = true
if (sendFailed) {
this.status.content = "Not sent · send failed; press Enter to retry when ready"
return
}
this.renderConnectionMessage()
}
private setActive(active: boolean, startedAt?: number): void { private setActive(active: boolean, startedAt?: number): void {
if (this.activeTurn === active) { if (this.activeTurn === active) {
if (active && startedAt !== undefined) this.activeStartedAt = startedAt if (active && startedAt !== undefined) this.activeStartedAt = startedAt
@@ -1460,6 +1516,7 @@ export class NanobotTui {
} }
private readyStatus(detail = this.readyDetail): string { private readyStatus(detail = this.readyDetail): string {
if (this.unsentSubmit) return "Not sent · press Enter to retry"
if (this.transcriptNavigation.awayFromBottom) { if (this.transcriptNavigation.awayFromBottom) {
return this.transcriptNavigation.unseenOutput return this.transcriptNavigation.unseenOutput
? "New output · Ctrl+End latest" ? "New output · Ctrl+End latest"
@@ -2083,6 +2140,7 @@ export class NanobotTui {
} }
private clearComposer(): void { private clearComposer(): void {
this.unsentSubmit = false
this.draft.clear() this.draft.clear()
this.composer.setText("") this.composer.setText("")
} }
@@ -2383,7 +2441,7 @@ export class NanobotTui {
options: MessageOptions = {}, options: MessageOptions = {},
): void { ): void {
if (!this.ready) { if (!this.ready) {
this.status.content = "Preparing chat…" this.markSubmitUnsent()
return return
} }
if (this.activeTurn && lifecycle === "agent_turn") { if (this.activeTurn && lifecycle === "agent_turn") {
@@ -2393,10 +2451,11 @@ export class NanobotTui {
let turnId: string let turnId: string
try { try {
turnId = this.client.send(content, options) turnId = this.client.send(content, options)
} catch (error) { } catch {
this.status.content = error instanceof Error ? error.message : String(error) this.markSubmitUnsent(true)
return return
} }
this.unsentSubmit = false
this.commandTurns.set(turnId, lifecycle) this.commandTurns.set(turnId, lifecycle)
if (silent) this.silentCommandTurns.add(turnId) if (silent) this.silentCommandTurns.add(turnId)
if (/^\/model(?:\s|$)/iu.test(content)) this.modelCommandTurns.add(turnId) if (/^\/model(?:\s|$)/iu.test(content)) this.modelCommandTurns.add(turnId)
+2
View File
@@ -14,6 +14,7 @@ const workspace = process.env.NANOBOT_TUI_WORKSPACE?.trim() || ""
const hostWorkspace = process.cwd() const hostWorkspace = process.cwd()
const bootstrapUrl = process.env.NANOBOT_TUI_BOOTSTRAP_URL?.trim() || "" const bootstrapUrl = process.env.NANOBOT_TUI_BOOTSTRAP_URL?.trim() || ""
const wsUrl = process.env.NANOBOT_TUI_WS_URL?.trim() || "" const wsUrl = process.env.NANOBOT_TUI_WS_URL?.trim() || ""
const healthUrl = process.env.NANOBOT_TUI_HEALTH_URL?.trim() || ""
const gatewayStopCommand = process.env.NANOBOT_TUI_GATEWAY_STOP_COMMAND?.trim() const gatewayStopCommand = process.env.NANOBOT_TUI_GATEWAY_STOP_COMMAND?.trim()
|| "nanobot gateway stop" || "nanobot gateway stop"
if (!bootstrapUrl && !wsUrl) { if (!bootstrapUrl && !wsUrl) {
@@ -24,6 +25,7 @@ const options: AppOptions = {
? { ? {
bootstrapUrl, bootstrapUrl,
bootstrapSecret: process.env.NANOBOT_TUI_BOOTSTRAP_SECRET?.trim() || "", bootstrapSecret: process.env.NANOBOT_TUI_BOOTSTRAP_SECRET?.trim() || "",
healthUrl: healthUrl || undefined,
} }
: { wsUrl }), : { wsUrl }),
apiUrl: process.env.NANOBOT_TUI_API_URL?.trim() || "", apiUrl: process.env.NANOBOT_TUI_API_URL?.trim() || "",
+196 -1
View File
@@ -3,14 +3,19 @@ import { describe, expect, test } from "bun:test"
import { import {
NanobotClient, NanobotClient,
GatewayConnectionError, GatewayConnectionError,
connectionEndpoint,
fetchAvailableSkills, fetchAvailableSkills,
fetchGatewayConnection, fetchGatewayConnection,
fetchGatewayHealth,
fetchHistory, fetchHistory,
fetchMentionCandidates, fetchMentionCandidates,
fetchRuntimeControls, fetchRuntimeControls,
fetchSessionContext, fetchSessionContext,
fetchSessions, fetchSessions,
fetchSlashCommands, fetchSlashCommands,
sanitizeConnectionFailure,
type ConnectionStatus,
type ConnectionStatusInfo,
type InboundEvent, type InboundEvent,
} from "./protocol" } from "./protocol"
@@ -39,6 +44,12 @@ class FakeSocket {
} }
} }
async function waitUntil(predicate: () => boolean, timeout = 1_000): Promise<void> {
const deadline = Date.now() + timeout
while (!predicate() && Date.now() < deadline) await Bun.sleep(2)
if (!predicate()) throw new Error(`condition was not met within ${timeout}ms`)
}
describe("gateway protocol", () => { describe("gateway protocol", () => {
test("bootstraps fresh websocket and API credentials", async () => { test("bootstraps fresh websocket and API credentials", async () => {
const original = globalThis.fetch const original = globalThis.fetch
@@ -70,6 +81,41 @@ describe("gateway protocol", () => {
} }
}) })
test("classifies gateway health without sending credentials", async () => {
const original = globalThis.fetch
const requests: Array<{ url: string; headers: Headers }> = []
const responses = [
new Response(JSON.stringify({
status: "degraded",
process: "alive",
ready: false,
websocket: "unavailable",
}), { status: 503 }),
new Response(JSON.stringify({
status: "ok",
process: "alive",
ready: true,
websocket: "running",
})),
new Response("not json"),
]
globalThis.fetch = ((input: string | URL | Request, init?: RequestInit) => {
requests.push({ url: String(input), headers: new Headers(init?.headers) })
return Promise.resolve(responses.shift() || new Response("missing", { status: 500 }))
}) as typeof fetch
try {
const healthUrl = "http://127.0.0.1:18790/health"
expect(await fetchGatewayHealth(healthUrl)).toBe("degraded")
expect(await fetchGatewayHealth(healthUrl)).toBe("ready")
expect(await fetchGatewayHealth(healthUrl)).toBe("unreachable")
expect(requests.map(({ url }) => url)).toEqual([healthUrl, healthUrl, healthUrl])
expect(requests.every(({ headers }) => [...headers].length === 0)).toBe(true)
} finally {
globalThis.fetch = original
}
})
test("rejects malformed bootstrap responses without retrying", async () => { test("rejects malformed bootstrap responses without retrying", async () => {
const original = globalThis.fetch const original = globalThis.fetch
globalThis.fetch = (() => Promise.resolve(new Response("not json"))) as unknown as typeof fetch globalThis.fetch = (() => Promise.resolve(new Response("not json"))) as unknown as typeof fetch
@@ -138,8 +184,13 @@ describe("gateway protocol", () => {
try { try {
const connections: string[] = [] const connections: string[] = []
let healthChecks = 0
const client = new NanobotClient({ const client = new NanobotClient({
resolveConnection: () => new Promise((resolve) => { resolveConnection = resolve }), resolveConnection: () => new Promise((resolve) => { resolveConnection = resolve }),
checkHealth: async () => {
healthChecks += 1
return "ready"
},
onConnection: (connection) => connections.push(connection.apiToken), onConnection: (connection) => connections.push(connection.apiToken),
onEvent: () => undefined, onEvent: () => undefined,
onStatus: () => undefined, onStatus: () => undefined,
@@ -155,6 +206,7 @@ describe("gateway protocol", () => {
await Bun.sleep(1) await Bun.sleep(1)
expect(requestedUrl).toBe("ws://nanobot.test/ws?token=fresh") expect(requestedUrl).toBe("ws://nanobot.test/ws?token=fresh")
expect(connections).toEqual(["fresh-api-token"]) expect(connections).toEqual(["fresh-api-token"])
expect(healthChecks).toBe(0)
client.close() client.close()
} finally { } finally {
Object.defineProperty(globalThis, "WebSocket", { configurable: true, value: original }) Object.defineProperty(globalThis, "WebSocket", { configurable: true, value: original })
@@ -200,6 +252,133 @@ describe("gateway protocol", () => {
} }
}) })
test("escalates a refused bootstrap with safe endpoint and retry diagnostics", async () => {
const original = globalThis.fetch
const bootstrapUrl = "http://bootstrap-user:bootstrap-pass@127.0.0.1:8769"
+ "/webui/bootstrap?token=socket-secret"
const bootstrapSecret = "bootstrap-secret"
const statuses: Array<{
status: ConnectionStatus
detail?: string
info?: ConnectionStatusInfo
}> = []
const refused = new TypeError(
`fetch failed for ${bootstrapUrl}&api_token=api-secret`,
{
cause: Object.assign(new Error("connect ECONNREFUSED 127.0.0.1:8769"), {
code: "ECONNREFUSED",
}),
},
)
globalThis.fetch = (() => Promise.reject(refused)) as unknown as typeof fetch
const client = new NanobotClient({
resolveConnection: () => fetchGatewayConnection(
bootstrapUrl,
bootstrapSecret,
"http://127.0.0.1:8769",
"tui-42",
),
targetEndpoint: connectionEndpoint(bootstrapUrl),
checkHealth: async () => "degraded",
startupFailureDelayMs: 8,
reconnectDelayMs: 100,
onEvent: () => undefined,
onStatus: (status, detail, info) => statuses.push({ status, detail, info }),
})
try {
client.connect()
await waitUntil(() => statuses.some(({ status }) => status === "unavailable"))
const failure = [...statuses].reverse().find(({ status }) => status === "unavailable")
expect(statuses[0]?.status).toBe("starting")
expect(failure?.detail).toBe("connection refused")
expect(failure?.info).toMatchObject({
endpoint: "127.0.0.1:8769",
attempt: 1,
elapsedMs: expect.any(Number),
health: "degraded",
})
const visible = JSON.stringify(statuses)
expect(visible).not.toContain("bootstrap-user")
expect(visible).not.toContain("bootstrap-pass")
expect(visible).not.toContain(bootstrapSecret)
expect(visible).not.toContain("socket-secret")
expect(visible).not.toContain("api-secret")
expect(visible).not.toContain("/webui/bootstrap")
} finally {
client.close()
globalThis.fetch = original
}
})
test("recovers after sustained bootstrap failures without hiding the outage", async () => {
const original = globalThis.WebSocket
const sockets: FakeSocket[] = []
let available = false
let attempts = 0
const statuses: ConnectionStatus[] = []
Object.defineProperty(globalThis, "WebSocket", {
configurable: true,
value: class extends FakeSocket {
constructor() {
super()
sockets.push(this)
}
},
})
const client = new NanobotClient({
resolveConnection: async () => {
attempts += 1
if (!available) {
throw Object.assign(new Error("connect ECONNREFUSED"), { code: "ECONNREFUSED" })
}
return {
wsUrl: "ws://127.0.0.1:8769/ws?token=fresh",
apiUrl: "http://127.0.0.1:8769",
apiToken: "fresh-api-token",
}
},
targetEndpoint: "127.0.0.1:8769",
checkHealth: async () => available ? "ready" : "degraded",
startupFailureDelayMs: 8,
reconnectDelayMs: 2,
startupRetryMaxDelayMs: 2,
onEvent: () => undefined,
onStatus: (status) => statuses.push(status),
})
try {
client.connect()
await waitUntil(() => statuses.includes("unavailable"))
available = true
await waitUntil(() => sockets.length === 1)
sockets[0]?.emit("open")
await waitUntil(() => statuses.at(-1) === "connected")
expect(attempts).toBeGreaterThan(1)
expect(statuses.indexOf("starting")).toBeLessThan(statuses.indexOf("unavailable"))
expect(statuses.indexOf("unavailable")).toBeLessThan(statuses.lastIndexOf("connected"))
} finally {
client.close()
Object.defineProperty(globalThis, "WebSocket", { configurable: true, value: original })
}
})
test("sanitizes arbitrary connection errors and authenticated URLs", () => {
const authenticated = "wss://user:password@127.0.0.1:8769/ws"
+ "?token=socket-secret&api_token=api-secret"
const unknown = new Error(`could not reach ${authenticated}`)
const refused = Object.assign(new Error(`ECONNREFUSED ${authenticated}`), {
code: "ECONNREFUSED",
})
expect(connectionEndpoint(authenticated)).toBe("127.0.0.1:8769")
expect(sanitizeConnectionFailure(unknown)).toBe("connection failed")
expect(sanitizeConnectionFailure(refused)).toBe("connection refused")
expect(sanitizeConnectionFailure(unknown)).not.toContain("socket-secret")
})
test("reports a permanent bootstrap rejection without retrying", async () => { test("reports a permanent bootstrap rejection without retrying", async () => {
let attempts = 0 let attempts = 0
const statuses: string[] = [] const statuses: string[] = []
@@ -588,13 +767,19 @@ describe("gateway protocol", () => {
}) })
try { try {
const statuses: Array<{
status: ConnectionStatus
detail?: string
info?: ConnectionStatusInfo
}> = []
const client = new NanobotClient({ const client = new NanobotClient({
url: "ws://nanobot.test/ws", url: "ws://nanobot.test/ws",
reconnectDelayMs: 1, reconnectDelayMs: 1,
onEvent: () => undefined, onEvent: () => undefined,
onStatus: () => undefined, onStatus: (status, detail, info) => statuses.push({ status, detail, info }),
}) })
client.connect() client.connect()
sockets[0]?.emit("open")
sockets[0]?.emit("message", { sockets[0]?.emit("message", {
data: JSON.stringify({ event: "ready", chat_id: "", client_id: "client" }), data: JSON.stringify({ event: "ready", chat_id: "", client_id: "client" }),
}) })
@@ -605,11 +790,21 @@ describe("gateway protocol", () => {
await Bun.sleep(5) await Bun.sleep(5)
expect(sockets).toHaveLength(2) expect(sockets).toHaveLength(2)
const reconnecting = [...statuses].reverse().find(
({ status }) => status === "reconnecting",
)
expect(reconnecting).toMatchObject({
status: "reconnecting",
detail: "connection closed",
info: { endpoint: "nanobot.test", attempt: 1 },
})
sockets[1]?.emit("open")
sockets[1]?.emit("message", { sockets[1]?.emit("message", {
data: JSON.stringify({ event: "ready", chat_id: "", client_id: "client-2" }), data: JSON.stringify({ event: "ready", chat_id: "", client_id: "client-2" }),
}) })
const outbound = sockets[1]?.sent.map((value) => JSON.parse(value)) || [] const outbound = sockets[1]?.sent.map((value) => JSON.parse(value)) || []
expect(outbound).toEqual([{ type: "attach", chat_id: "generated-chat" }]) expect(outbound).toEqual([{ type: "attach", chat_id: "generated-chat" }])
expect(statuses.at(-1)?.status).toBe("connected")
client.close() client.close()
} finally { } finally {
Object.defineProperty(globalThis, "WebSocket", { configurable: true, value: original }) Object.defineProperty(globalThis, "WebSocket", { configurable: true, value: original })
+223 -18
View File
@@ -1,4 +1,21 @@
export type ConnectionStatus = "connecting" | "connected" | "closed" | "error" export type ConnectionStatus =
| "starting"
| "connecting"
| "connected"
| "reconnecting"
| "unavailable"
| "closed"
| "error"
export interface ConnectionStatusInfo {
endpoint: string
attempt: number
elapsedMs: number
retryInMs?: number
health?: GatewayHealthStatus
}
export type GatewayHealthStatus = "ready" | "degraded" | "unreachable"
export interface ToolProgressEvent { export interface ToolProgressEvent {
version?: number version?: number
@@ -172,14 +189,16 @@ type OutboundEvent =
export interface ClientOptions { export interface ClientOptions {
url?: string url?: string
resolveConnection?: () => Promise<GatewayConnection> resolveConnection?: () => Promise<GatewayConnection>
checkHealth?: () => Promise<GatewayHealthStatus>
onConnection?: (connection: GatewayConnection) => void onConnection?: (connection: GatewayConnection) => void
connectionRetryLabel?: string targetEndpoint?: string
startupFailureDelayMs?: number
startupRetryMaxDelayMs?: number startupRetryMaxDelayMs?: number
chatId?: string chatId?: string
initialWorkspaceScope?: WorkspaceScopePayload initialWorkspaceScope?: WorkspaceScopePayload
reconnectDelayMs?: number reconnectDelayMs?: number
onEvent: (event: InboundEvent) => void onEvent: (event: InboundEvent) => void
onStatus: (status: ConnectionStatus, detail?: string) => void onStatus: (status: ConnectionStatus, detail?: string, info?: ConnectionStatusInfo) => void
} }
export interface GatewayApiConnection { export interface GatewayApiConnection {
@@ -940,6 +959,92 @@ export async function fetchGatewayConnection(
} }
} }
/** Read gateway readiness without sending bootstrap or API credentials. */
export async function fetchGatewayHealth(
healthUrl: string,
timeoutMs = 400,
): Promise<GatewayHealthStatus> {
const controller = new AbortController()
const timer = setTimeout(() => controller.abort(), timeoutMs)
try {
const response = await fetch(healthUrl, { signal: controller.signal })
if (response.status !== 200 && response.status !== 503) return "unreachable"
const payload: unknown = await response.json()
if (!isRecord(payload)) return "unreachable"
if (
response.status === 503
&& payload.status === "degraded"
&& payload.ready === false
&& payload.process === "alive"
) return "degraded"
if (response.status === 200 && payload.status === "ok" && payload.ready !== false) {
return "ready"
}
return "unreachable"
} catch {
return "unreachable"
} finally {
clearTimeout(timer)
}
}
/** Return only the authority users can act on, never credentials or an authenticated path. */
export function connectionEndpoint(value: string | undefined): string {
if (!value) return "local gateway"
try {
return new URL(value).host || "local gateway"
} catch {
return "local gateway"
}
}
/** Reduce arbitrary fetch/WebSocket errors to a small set of credential-safe reasons. */
export function sanitizeConnectionFailure(error: unknown): string {
const signals: string[] = []
const seen = new Set<unknown>()
const collect = (value: unknown): void => {
if (value === null || value === undefined || seen.has(value)) return
if (typeof value === "object") seen.add(value)
if (typeof value === "string") {
signals.push(value)
return
}
if (value instanceof Error) {
signals.push(value.name, value.message)
collect(value.cause)
if (value instanceof AggregateError) {
for (const nested of value.errors) collect(nested)
}
return
}
if (!isRecord(value)) return
if (typeof value.code === "string") signals.push(value.code)
collect(value.cause)
if (Array.isArray(value.errors)) {
for (const nested of value.errors) collect(nested)
}
}
collect(error)
const signal = signals.join(" ")
if (/ECONNREFUSED|connection refused/iu.test(signal)) return "connection refused"
if (/ETIMEDOUT|timed? out|timeout/iu.test(signal)) return "connection timed out"
if (/ENOTFOUND|EAI_AGAIN|name not resolved|host not found/iu.test(signal)) {
return "host not found"
}
if (/certificate|TLS|SSL/iu.test(signal)) return "secure connection failed"
const bootstrapStatus = signal.match(/gateway bootstrap failed:\s*HTTP\s*(\d{3})/iu)
if (bootstrapStatus?.[1]) return `gateway bootstrap failed: HTTP ${bootstrapStatus[1]}`
if (/bootstrap response is missing ws_url/iu.test(signal)) {
return "gateway bootstrap response is missing ws_url"
}
if (/bootstrap response (?:has an invalid ws_url|is invalid)/iu.test(signal)) {
return "gateway bootstrap response is invalid"
}
if (/gateway is still starting/iu.test(signal)) return "gateway is still starting"
if (/fetch failed|failed to fetch|network error/iu.test(signal)) return "network request failed"
return "connection failed"
}
export class NanobotClient { export class NanobotClient {
private socket: WebSocket | null = null private socket: WebSocket | null = null
private chatId = "" private chatId = ""
@@ -949,13 +1054,22 @@ export class NanobotClient {
private closedByClient = false private closedByClient = false
private opening = false private opening = false
private connectedOnce = false private connectedOnce = false
private connectionAttempt = 0
private retryStartedAt = 0
private nextRetryAt = 0
private lastFailure = ""
private healthStatus: GatewayHealthStatus | undefined
private failureEscalationTimer: ReturnType<typeof setTimeout> | null = null
private readonly endpoint: string
private readonly pendingMutations = new Map<string, { private readonly pendingMutations = new Map<string, {
resolve: (value: unknown) => void resolve: (value: unknown) => void
reject: (error: Error) => void reject: (error: Error) => void
timer: ReturnType<typeof setTimeout> timer: ReturnType<typeof setTimeout>
}>() }>()
constructor(private readonly options: ClientOptions) {} constructor(private readonly options: ClientOptions) {
this.endpoint = options.targetEndpoint || connectionEndpoint(options.url)
}
get activeChatId(): string { get activeChatId(): string {
return this.chatId return this.chatId
@@ -963,13 +1077,21 @@ export class NanobotClient {
connect(): void { connect(): void {
this.closedByClient = false this.closedByClient = false
this.connectionAttempt = 0
this.reconnectAttempt = 0
this.retryStartedAt = Date.now()
this.nextRetryAt = 0
this.lastFailure = ""
this.healthStatus = undefined
void this.open() void this.open()
} }
private async open(): Promise<void> { private async open(): Promise<void> {
if (this.socket || this.opening || this.closedByClient) return if (this.socket || this.opening || this.closedByClient) return
this.opening = true this.opening = true
this.options.onStatus("connecting") this.nextRetryAt = 0
this.connectionAttempt += 1
this.reportConnectionProgress()
let url = this.options.url let url = this.options.url
try { try {
if (this.options.resolveConnection) { if (this.options.resolveConnection) {
@@ -980,37 +1102,52 @@ export class NanobotClient {
} }
} catch (error) { } catch (error) {
if (!this.closedByClient) { if (!this.closedByClient) {
this.lastFailure = sanitizeConnectionFailure(error)
if (error instanceof GatewayConnectionError && !error.retryable) { if (error instanceof GatewayConnectionError && !error.retryable) {
this.options.onStatus("error", error.message) this.clearFailureEscalation()
this.options.onStatus("error", this.lastFailure, this.connectionInfo())
return return
} }
this.options.onStatus( await this.checkHealthAndScheduleReconnect()
"connecting",
this.options.connectionRetryLabel || "gateway unavailable",
)
this.scheduleReconnect(false)
} }
return return
} finally { } finally {
this.opening = false this.opening = false
} }
if (!url) { if (!url) {
this.options.onStatus("error", "gateway URL is not configured") this.options.onStatus("error", "gateway URL is not configured", this.connectionInfo())
return return
} }
const socket = new WebSocket(url) let socket: WebSocket
try {
socket = new WebSocket(url)
} catch (error) {
this.lastFailure = sanitizeConnectionFailure(error)
await this.checkHealthAndScheduleReconnect()
return
}
let opened = false
this.socket = socket this.socket = socket
socket.addEventListener("open", () => { socket.addEventListener("open", () => {
if (this.socket !== socket) return if (this.socket !== socket) return
opened = true
this.connectedOnce = true this.connectedOnce = true
this.connectionAttempt = 0
this.reconnectAttempt = 0 this.reconnectAttempt = 0
this.options.onStatus("connected") this.retryStartedAt = 0
this.nextRetryAt = 0
this.lastFailure = ""
this.healthStatus = "ready"
this.clearFailureEscalation()
this.options.onStatus("connected", undefined, this.connectionInfo())
}) })
socket.addEventListener("message", (message) => { socket.addEventListener("message", (message) => {
if (this.socket === socket) this.handleMessage(String(message.data)) if (this.socket === socket) this.handleMessage(String(message.data))
}) })
socket.addEventListener("error", () => { socket.addEventListener("error", () => {
if (this.socket === socket) this.options.onStatus("error", "connection failed") if (this.socket !== socket) return
this.lastFailure = "connection failed"
this.reportRetryState()
}) })
socket.addEventListener("close", () => { socket.addEventListener("close", () => {
if (this.socket !== socket) return if (this.socket !== socket) return
@@ -1020,7 +1157,14 @@ export class NanobotClient {
this.options.onStatus("closed") this.options.onStatus("closed")
return return
} }
this.scheduleReconnect() if (opened) {
this.connectionAttempt = 0
this.reconnectAttempt = 0
this.retryStartedAt = Date.now()
}
if (!this.lastFailure) this.lastFailure = "connection closed"
this.reportRetryState()
void this.checkHealthAndScheduleReconnect()
}) })
} }
@@ -1028,6 +1172,7 @@ export class NanobotClient {
this.closedByClient = true this.closedByClient = true
if (this.reconnectTimer) clearTimeout(this.reconnectTimer) if (this.reconnectTimer) clearTimeout(this.reconnectTimer)
this.reconnectTimer = null this.reconnectTimer = null
this.clearFailureEscalation()
const socket = this.socket const socket = this.socket
this.socket = null this.socket = null
socket?.close() socket?.close()
@@ -1182,20 +1327,80 @@ export class NanobotClient {
this.options.onEvent(event) this.options.onEvent(event)
} }
private scheduleReconnect(announce = true): void { private scheduleReconnect(): void {
if (this.reconnectTimer || this.closedByClient) return if (this.reconnectTimer || this.closedByClient) return
if (!this.retryStartedAt) this.retryStartedAt = Date.now()
const base = this.options.reconnectDelayMs ?? 500 const base = this.options.reconnectDelayMs ?? 500
const maxDelay = this.connectedOnce const maxDelay = this.connectedOnce
? 8_000 ? 8_000
: this.options.startupRetryMaxDelayMs ?? 8_000 : this.options.startupRetryMaxDelayMs ?? 8_000
const delay = Math.min(maxDelay, base * 2 ** Math.min(this.reconnectAttempt++, 4)) const delay = Math.min(maxDelay, base * 2 ** Math.min(this.reconnectAttempt++, 4))
if (announce) this.options.onStatus("connecting", `reconnecting in ${delay}ms`) this.nextRetryAt = Date.now() + delay
this.reportRetryState()
this.reconnectTimer = setTimeout(() => { this.reconnectTimer = setTimeout(() => {
this.reconnectTimer = null this.reconnectTimer = null
void this.open() void this.open()
}, delay) }, delay)
} }
private async checkHealthAndScheduleReconnect(): Promise<void> {
if (this.options.checkHealth) {
try {
this.healthStatus = await this.options.checkHealth()
} catch {
this.healthStatus = "unreachable"
}
}
if (!this.closedByClient) this.scheduleReconnect()
}
private connectionInfo(): ConnectionStatusInfo {
return {
endpoint: this.endpoint,
attempt: Math.max(1, this.connectionAttempt),
elapsedMs: this.retryStartedAt ? Math.max(0, Date.now() - this.retryStartedAt) : 0,
...(this.nextRetryAt
? { retryInMs: Math.max(0, this.nextRetryAt - Date.now()) }
: {}),
...(this.healthStatus ? { health: this.healthStatus } : {}),
}
}
private reportConnectionProgress(): void {
if (this.connectedOnce) {
this.options.onStatus("reconnecting", this.lastFailure || undefined, this.connectionInfo())
return
}
const phase = this.options.resolveConnection ? "starting" : "connecting"
this.options.onStatus(phase, undefined, this.connectionInfo())
}
private reportRetryState(): void {
const info = this.connectionInfo()
if (this.connectedOnce) {
this.options.onStatus("reconnecting", this.lastFailure, info)
return
}
const failureDelay = this.options.startupFailureDelayMs ?? 3_000
if (info.elapsedMs >= failureDelay) {
this.clearFailureEscalation()
this.options.onStatus("unavailable", this.lastFailure, info)
return
}
this.reportConnectionProgress()
if (this.failureEscalationTimer) return
this.failureEscalationTimer = setTimeout(() => {
this.failureEscalationTimer = null
if (this.closedByClient || this.connectedOnce || !this.lastFailure) return
this.options.onStatus("unavailable", this.lastFailure, this.connectionInfo())
}, Math.max(0, failureDelay - info.elapsedMs))
}
private clearFailureEscalation(): void {
if (this.failureEscalationTimer) clearTimeout(this.failureEscalationTimer)
this.failureEscalationTimer = null
}
private write(event: OutboundEvent): void { private write(event: OutboundEvent): void {
if (!this.socket || this.socket.readyState !== WebSocket.OPEN) { if (!this.socket || this.socket.readyState !== WebSocket.OPEN) {
throw new Error("gateway connection is not open") throw new Error("gateway connection is not open")