fix(tui): surface chat connection failures (#5543)

* fix(tui): surface chat connection failures

* fix(tui): keep connection copy implementation-agnostic

* test(tui): avoid millisecond timing boundary

* fix(tui): use gateway health for connection status

* fix(tui): use product-level readiness copy
This commit is contained in:
chengyongru
2026-08-27 11:37:30 +08:00
committed by GitHub
parent 55f85b3c1f
commit 3a62b0b744
7 changed files with 615 additions and 48 deletions
+106 -6
View File
@@ -182,7 +182,7 @@ describe("NanobotTui layout", () => {
expect(setup.renderer.height).toBe(height)
expect(occurrences(frame, "Ask nanobot anything")).toBe(1)
expect(occurrences(frame, "Ready")).toBe(0)
expect(occurrences(frame, "Connecting…")).toBe(1)
expect(occurrences(frame, "Getting ready…")).toBe(1)
expect(occurrences(frame, "nanobot · test/model")).toBe(1)
}
@@ -2384,6 +2384,85 @@ describe("NanobotTui layout", () => {
expect(state()).toBe(false)
})
test("shows actionable connection states without implementation details", async () => {
setup = await createRenderer({ width: 100, height: 20, screenMode: "alternate-screen" })
const app = mount(setup)
const ui = app as unknown as {
status: TextRenderable
handleStatus(
status: "starting" | "connecting" | "connected" | "reconnecting" | "unavailable" | "error",
detail?: string,
info?: {
endpoint: string
attempt: number
elapsedMs: number
health?: "ready" | "degraded" | "unreachable"
},
): void
}
ui.handleStatus("starting", undefined, {
endpoint: "127.0.0.1:8769",
attempt: 1,
elapsedMs: 0,
})
expect(ui.status.plainText).toBe("Getting ready…")
ui.handleStatus("connecting")
expect(ui.status.plainText).toBe("Getting ready…")
ui.handleStatus("connected")
expect(ui.status.plainText).toBe("Getting ready…")
ui.handleStatus("error", "gateway sent an invalid event")
expect(ui.status.plainText).toBe("Getting ready…")
expect(ui.status.plainText).not.toContain("Unable")
ui.handleStatus("reconnecting", "connection closed", {
endpoint: "127.0.0.1:8769",
attempt: 2,
elapsedMs: 800,
})
expect(ui.status.plainText).toBe("Resuming…")
ui.handleStatus("reconnecting", "connection closed", {
endpoint: "127.0.0.1:8769",
attempt: 2,
elapsedMs: 900,
health: "degraded",
})
expect(ui.status.plainText).toBe("Resuming…")
ui.handleStatus("unavailable", "connection refused", {
endpoint: "127.0.0.1:8769",
attempt: 7,
elapsedMs: 3_200,
health: "degraded",
})
expect(ui.status.plainText).toBe("Still getting ready…")
expect(ui.status.plainText).not.toContain("Unable")
ui.handleStatus("unavailable", "connection refused", {
endpoint: "127.0.0.1:8769",
attempt: 8,
elapsedMs: 3_500,
health: "unreachable",
})
expect(ui.status.plainText).toBe("Nanobot is taking longer to respond…")
expect(ui.status.plainText).not.toContain("Unable")
ui.handleStatus("error", "gateway bootstrap failed: HTTP 401", {
endpoint: "127.0.0.1:8769",
attempt: 9,
elapsedMs: 3_800,
})
expect(ui.status.plainText).toBe("Nanobot unavailable · restart nanobot")
expect(ui.status.plainText).not.toContain("gateway")
expect(ui.status.plainText).not.toContain("127.0.0.1")
expect(ui.status.plainText).not.toContain("HTTP")
expect(ui.status.plainText).not.toContain("attempt")
})
test("replays events after asynchronous history hydration", async () => {
setup = await createRenderer({ width: 80, height: 22, screenMode: "alternate-screen" })
const original = globalThis.fetch
@@ -2444,7 +2523,12 @@ describe("NanobotTui layout", () => {
client(sent),
new MockTreeSitterClient({ autoResolveTimeout: 0 }),
)
const composer = (app as unknown as { composer: TextareaRenderable }).composer
const ui = app as unknown as {
composer: TextareaRenderable
ready: boolean
status: TextRenderable
}
const composer = ui.composer
try {
app.accept({ event: "attached", chat_id: "chat" })
@@ -2452,16 +2536,18 @@ describe("NanobotTui layout", () => {
app.accept({ event: "attached", chat_id: "chat" })
composer.setText("sent during reconnect")
composer.submit()
await Bun.sleep(5)
await waitUntil(() => ui.status.plainText.includes("Not sent"))
expect(sent).toEqual([])
expect(composer.plainText).toBe("sent during reconnect")
expect(ui.status.plainText).toContain("Not sent · press Enter to retry when ready")
resolveReconnect(new Response(JSON.stringify({
messages: [{ role: "assistant", content: "restored history" }],
page: { has_more_before: false },
})))
await waitUntil(() => (app as unknown as { ready: boolean }).ready)
await waitUntil(() => ui.ready)
expect(ui.status.plainText).toBe("Not sent · press Enter to retry")
composer.submit()
await waitUntil(() => sent.length === 1)
await setup.flush()
@@ -2480,24 +2566,38 @@ describe("NanobotTui layout", () => {
const app = mount(setup, sent)
const composer = (app as unknown as { composer: TextareaRenderable }).composer
const connection = app as unknown as {
handleStatus(status: "connecting" | "connected", detail?: string): void
handleStatus(
status: "reconnecting" | "connected",
detail?: string,
info?: { endpoint: string; attempt: number; elapsedMs: number },
): void
}
app.accept({ event: "attached", chat_id: "chat" })
await Bun.sleep(1)
connection.handleStatus("connecting", "reconnecting")
connection.handleStatus("reconnecting", "connection closed", {
endpoint: "127.0.0.1:8769",
attempt: 1,
elapsedMs: 0,
})
connection.handleStatus("connected")
composer.setText("draft before attach")
composer.submit()
await Bun.sleep(5)
composer.submit()
await Bun.sleep(5)
expect(sent).toEqual([])
expect(composer.plainText).toBe("draft before attach")
app.accept({ event: "attached", chat_id: "chat" })
app.accept({ event: "attached", chat_id: "chat" })
await waitUntil(() => (app as unknown as { ready: boolean }).ready)
expect(sent).toEqual([])
composer.submit()
await waitUntil(() => sent.length === 1)
app.accept({ event: "attached", chat_id: "chat" })
await Bun.sleep(5)
expect(sent).toEqual(["draft before attach"])
})
+82 -23
View File
@@ -20,7 +20,9 @@ import {
import {
NanobotClient,
connectionEndpoint,
fetchAvailableSkills,
fetchGatewayHealth,
fetchHistory,
fetchGatewayConnection,
fetchMentionCandidates,
@@ -29,6 +31,7 @@ import {
fetchSlashCommands,
type ApiReauthenticator,
type ConnectionStatus,
type ConnectionStatusInfo,
type FileEditEvent,
type GatewayApiConnection,
type HistoryMessage,
@@ -93,6 +96,7 @@ interface AppOptions {
wsUrl?: string
bootstrapUrl?: string
bootstrapSecret?: string
healthUrl?: string
apiUrl: string
apiToken: string
chatId?: string
@@ -374,6 +378,21 @@ function formatElapsed(milliseconds: number): string {
return `${Math.floor(seconds / 60)}m ${String(seconds % 60).padStart(2, "0")}s`
}
function connectionStatusText(
status: ConnectionStatus,
info?: ConnectionStatusInfo,
): string {
if (["starting", "connecting", "connected"].includes(status)) return "Getting ready…"
if (status === "reconnecting") return "Resuming…"
if (status === "unavailable") {
return info?.health === "degraded"
? "Still getting ready…"
: "Nanobot is taking longer to respond…"
}
if (status === "error") return "Nanobot unavailable · restart nanobot"
return "Session ended"
}
function singleLine(value: string, limit = 120): string {
return value.replace(/\s+/gu, " ").trim().slice(0, limit)
}
@@ -449,6 +468,8 @@ export class NanobotTui {
private shimmerTimer: ReturnType<typeof setInterval> | null = null
private submitPending = false
private submitGeneration = 0
private unsentSubmit = false
private connectionMessage = "Getting ready…"
private readonly promptHistory: string[] = []
private historyCursor = 0
private historyDraft = ""
@@ -556,11 +577,14 @@ export class NanobotTui {
options.apiUrl,
`tui-${process.pid}`,
),
...(options.healthUrl
? { checkHealth: () => fetchGatewayHealth(options.healthUrl || "") }
: {}),
onConnection: (connection) => this.useGatewayConnection(
connection.apiUrl,
connection.apiToken,
),
connectionRetryLabel: "Starting local gateway",
targetEndpoint: connectionEndpoint(options.bootstrapUrl),
reconnectDelayMs: 100,
startupRetryMaxDelayMs: 250,
}
@@ -571,7 +595,7 @@ export class NanobotTui {
access_mode: options.access.toLocaleLowerCase().includes("full") ? "full" : "restricted",
},
onEvent: (event) => this.accept(event),
onStatus: (status, detail) => this.handleStatus(status, detail),
onStatus: (status, detail, info) => this.handleStatus(status, detail, info),
})
// The terminal owns its canvas. Keeping the default-background intent is
@@ -723,6 +747,8 @@ export class NanobotTui {
},
onContentChange: () => {
this.draft.prune(this.composer.plainText)
const clearedUnsent = this.unsentSubmit && !this.composer.plainText.trim()
if (clearedUnsent) this.unsentSubmit = false
this.runtimeControls.hide()
if (this.contextPanel.visible && this.composer.plainText) this.contextPanel.hide()
this.syncComposerPlaceholder()
@@ -730,6 +756,9 @@ export class NanobotTui {
else if (this.branchMenu.visible) this.syncBranchMenu()
else this.syncComposerMenus()
this.resizeComposer()
if (clearedUnsent && !this.activeTurn) {
this.status.content = this.ready ? this.readyStatus() : this.connectionMessage
}
},
// IMEs may commit their final composed glyph after Enter. Matching the
// OpenCode/OpenTUI integration, defer twice before reading plainText.
@@ -738,7 +767,7 @@ export class NanobotTui {
})
this.status = new TextRenderable(renderer, {
id: "nanobot-tui-status",
content: "Connecting…",
content: "Getting ready…",
fg: this.palette.muted,
height: 1,
width: "auto",
@@ -824,7 +853,7 @@ export class NanobotTui {
// Network setup and small menu payloads do not depend on terminal colors.
// Start them while OSC theme detection is in flight instead of serializing
// up to one second of otherwise independent startup work.
this.host.reportState("unknown", "Connecting")
this.host.reportState("unknown", "Getting ready")
this.client.connect()
void this.loadCommands()
void this.loadMentions()
@@ -899,6 +928,10 @@ export class NanobotTui {
return
}
if (["/continue", "/dismiss"].includes(visibleContent.toLowerCase())) {
if (!this.ready) {
this.markSubmitUnsent()
return
}
this.clearComposer()
this.commandMenu.hide()
void this.updateRecovery(visibleContent.toLowerCase() === "/continue" ? "continue" : "dismiss")
@@ -932,7 +965,7 @@ export class NanobotTui {
return
}
if (!this.ready) {
this.status.content = "Preparing chat…"
this.markSubmitUnsent()
return
}
const prompt = { content, options: mentionOptions(content, this.availableMentions()) }
@@ -947,10 +980,11 @@ export class NanobotTui {
let turnId: string
try {
turnId = this.client.send(prompt.content, prompt.options)
} catch (error) {
this.status.content = error instanceof Error ? error.message : String(error)
} catch {
this.markSubmitUnsent(true)
return false
}
this.unsentSubmit = false
this.clearComposer()
this.commandMenu.hide()
this.mentionMenu.hide()
@@ -1392,36 +1426,58 @@ export class NanobotTui {
}
}
private handleStatus(status: ConnectionStatus, detail?: string): void {
private handleStatus(
status: ConnectionStatus,
_detail?: string,
info?: ConnectionStatusInfo,
): void {
// Invalid frames do not mean the transport is unavailable. Keep the last
// accurate user-facing state unless the protocol supplied connection diagnostics.
if (status === "error" && !info) return
this.connectionMessage = connectionStatusText(status, info)
if (status === "connected") {
this.ready = false
this.host.reportState("unknown", "Connecting")
this.status.content = "Connected · preparing chat…"
this.host.reportState("unknown", "Getting ready")
this.renderConnectionMessage()
return
}
if (status === "connecting") {
if (["starting", "connecting", "reconnecting", "unavailable"].includes(status)) {
this.ready = false
const label = detail === "Starting local gateway"
? detail
: detail ? "Reconnecting" : "Connecting"
this.host.reportState("unknown", label)
if (detail) this.setActive(false)
this.status.content = `${label}`
this.host.reportState("unknown", this.connectionMessage)
if (status === "reconnecting" || status === "unavailable") this.setActive(false)
this.renderConnectionMessage()
return
}
if (status === "error") {
if (info) this.ready = false
this.setActive(false)
this.host.reportState("unknown", detail || "Connection error")
this.status.content = detail || "Connection error"
this.host.reportState("unknown", this.connectionMessage)
this.renderConnectionMessage()
return
}
if (!this.quitting) {
this.ready = false
this.setActive(false)
this.host.reportState("unknown", "Disconnected")
this.status.content = "Disconnected"
this.renderConnectionMessage()
}
}
private renderConnectionMessage(): void {
this.status.content = this.unsentSubmit
? `Not sent · press Enter to retry when ready · ${this.connectionMessage}`
: this.connectionMessage
}
private markSubmitUnsent(sendFailed = false): void {
this.unsentSubmit = true
if (sendFailed) {
this.status.content = "Not sent · send failed; press Enter to retry when ready"
return
}
this.renderConnectionMessage()
}
private setActive(active: boolean, startedAt?: number): void {
if (this.activeTurn === active) {
if (active && startedAt !== undefined) this.activeStartedAt = startedAt
@@ -1460,6 +1516,7 @@ export class NanobotTui {
}
private readyStatus(detail = this.readyDetail): string {
if (this.unsentSubmit) return "Not sent · press Enter to retry"
if (this.transcriptNavigation.awayFromBottom) {
return this.transcriptNavigation.unseenOutput
? "New output · Ctrl+End latest"
@@ -2083,6 +2140,7 @@ export class NanobotTui {
}
private clearComposer(): void {
this.unsentSubmit = false
this.draft.clear()
this.composer.setText("")
}
@@ -2383,7 +2441,7 @@ export class NanobotTui {
options: MessageOptions = {},
): void {
if (!this.ready) {
this.status.content = "Preparing chat…"
this.markSubmitUnsent()
return
}
if (this.activeTurn && lifecycle === "agent_turn") {
@@ -2393,10 +2451,11 @@ export class NanobotTui {
let turnId: string
try {
turnId = this.client.send(content, options)
} catch (error) {
this.status.content = error instanceof Error ? error.message : String(error)
} catch {
this.markSubmitUnsent(true)
return
}
this.unsentSubmit = false
this.commandTurns.set(turnId, lifecycle)
if (silent) this.silentCommandTurns.add(turnId)
if (/^\/model(?:\s|$)/iu.test(content)) this.modelCommandTurns.add(turnId)
+2
View File
@@ -14,6 +14,7 @@ const workspace = process.env.NANOBOT_TUI_WORKSPACE?.trim() || ""
const hostWorkspace = process.cwd()
const bootstrapUrl = process.env.NANOBOT_TUI_BOOTSTRAP_URL?.trim() || ""
const wsUrl = process.env.NANOBOT_TUI_WS_URL?.trim() || ""
const healthUrl = process.env.NANOBOT_TUI_HEALTH_URL?.trim() || ""
const gatewayStopCommand = process.env.NANOBOT_TUI_GATEWAY_STOP_COMMAND?.trim()
|| "nanobot gateway stop"
if (!bootstrapUrl && !wsUrl) {
@@ -24,6 +25,7 @@ const options: AppOptions = {
? {
bootstrapUrl,
bootstrapSecret: process.env.NANOBOT_TUI_BOOTSTRAP_SECRET?.trim() || "",
healthUrl: healthUrl || undefined,
}
: { wsUrl }),
apiUrl: process.env.NANOBOT_TUI_API_URL?.trim() || "",
+196 -1
View File
@@ -3,14 +3,19 @@ import { describe, expect, test } from "bun:test"
import {
NanobotClient,
GatewayConnectionError,
connectionEndpoint,
fetchAvailableSkills,
fetchGatewayConnection,
fetchGatewayHealth,
fetchHistory,
fetchMentionCandidates,
fetchRuntimeControls,
fetchSessionContext,
fetchSessions,
fetchSlashCommands,
sanitizeConnectionFailure,
type ConnectionStatus,
type ConnectionStatusInfo,
type InboundEvent,
} from "./protocol"
@@ -39,6 +44,12 @@ class FakeSocket {
}
}
async function waitUntil(predicate: () => boolean, timeout = 1_000): Promise<void> {
const deadline = Date.now() + timeout
while (!predicate() && Date.now() < deadline) await Bun.sleep(2)
if (!predicate()) throw new Error(`condition was not met within ${timeout}ms`)
}
describe("gateway protocol", () => {
test("bootstraps fresh websocket and API credentials", async () => {
const original = globalThis.fetch
@@ -70,6 +81,41 @@ describe("gateway protocol", () => {
}
})
test("classifies gateway health without sending credentials", async () => {
const original = globalThis.fetch
const requests: Array<{ url: string; headers: Headers }> = []
const responses = [
new Response(JSON.stringify({
status: "degraded",
process: "alive",
ready: false,
websocket: "unavailable",
}), { status: 503 }),
new Response(JSON.stringify({
status: "ok",
process: "alive",
ready: true,
websocket: "running",
})),
new Response("not json"),
]
globalThis.fetch = ((input: string | URL | Request, init?: RequestInit) => {
requests.push({ url: String(input), headers: new Headers(init?.headers) })
return Promise.resolve(responses.shift() || new Response("missing", { status: 500 }))
}) as typeof fetch
try {
const healthUrl = "http://127.0.0.1:18790/health"
expect(await fetchGatewayHealth(healthUrl)).toBe("degraded")
expect(await fetchGatewayHealth(healthUrl)).toBe("ready")
expect(await fetchGatewayHealth(healthUrl)).toBe("unreachable")
expect(requests.map(({ url }) => url)).toEqual([healthUrl, healthUrl, healthUrl])
expect(requests.every(({ headers }) => [...headers].length === 0)).toBe(true)
} finally {
globalThis.fetch = original
}
})
test("rejects malformed bootstrap responses without retrying", async () => {
const original = globalThis.fetch
globalThis.fetch = (() => Promise.resolve(new Response("not json"))) as unknown as typeof fetch
@@ -138,8 +184,13 @@ describe("gateway protocol", () => {
try {
const connections: string[] = []
let healthChecks = 0
const client = new NanobotClient({
resolveConnection: () => new Promise((resolve) => { resolveConnection = resolve }),
checkHealth: async () => {
healthChecks += 1
return "ready"
},
onConnection: (connection) => connections.push(connection.apiToken),
onEvent: () => undefined,
onStatus: () => undefined,
@@ -155,6 +206,7 @@ describe("gateway protocol", () => {
await Bun.sleep(1)
expect(requestedUrl).toBe("ws://nanobot.test/ws?token=fresh")
expect(connections).toEqual(["fresh-api-token"])
expect(healthChecks).toBe(0)
client.close()
} finally {
Object.defineProperty(globalThis, "WebSocket", { configurable: true, value: original })
@@ -200,6 +252,133 @@ describe("gateway protocol", () => {
}
})
test("escalates a refused bootstrap with safe endpoint and retry diagnostics", async () => {
const original = globalThis.fetch
const bootstrapUrl = "http://bootstrap-user:bootstrap-pass@127.0.0.1:8769"
+ "/webui/bootstrap?token=socket-secret"
const bootstrapSecret = "bootstrap-secret"
const statuses: Array<{
status: ConnectionStatus
detail?: string
info?: ConnectionStatusInfo
}> = []
const refused = new TypeError(
`fetch failed for ${bootstrapUrl}&api_token=api-secret`,
{
cause: Object.assign(new Error("connect ECONNREFUSED 127.0.0.1:8769"), {
code: "ECONNREFUSED",
}),
},
)
globalThis.fetch = (() => Promise.reject(refused)) as unknown as typeof fetch
const client = new NanobotClient({
resolveConnection: () => fetchGatewayConnection(
bootstrapUrl,
bootstrapSecret,
"http://127.0.0.1:8769",
"tui-42",
),
targetEndpoint: connectionEndpoint(bootstrapUrl),
checkHealth: async () => "degraded",
startupFailureDelayMs: 8,
reconnectDelayMs: 100,
onEvent: () => undefined,
onStatus: (status, detail, info) => statuses.push({ status, detail, info }),
})
try {
client.connect()
await waitUntil(() => statuses.some(({ status }) => status === "unavailable"))
const failure = [...statuses].reverse().find(({ status }) => status === "unavailable")
expect(statuses[0]?.status).toBe("starting")
expect(failure?.detail).toBe("connection refused")
expect(failure?.info).toMatchObject({
endpoint: "127.0.0.1:8769",
attempt: 1,
elapsedMs: expect.any(Number),
health: "degraded",
})
const visible = JSON.stringify(statuses)
expect(visible).not.toContain("bootstrap-user")
expect(visible).not.toContain("bootstrap-pass")
expect(visible).not.toContain(bootstrapSecret)
expect(visible).not.toContain("socket-secret")
expect(visible).not.toContain("api-secret")
expect(visible).not.toContain("/webui/bootstrap")
} finally {
client.close()
globalThis.fetch = original
}
})
test("recovers after sustained bootstrap failures without hiding the outage", async () => {
const original = globalThis.WebSocket
const sockets: FakeSocket[] = []
let available = false
let attempts = 0
const statuses: ConnectionStatus[] = []
Object.defineProperty(globalThis, "WebSocket", {
configurable: true,
value: class extends FakeSocket {
constructor() {
super()
sockets.push(this)
}
},
})
const client = new NanobotClient({
resolveConnection: async () => {
attempts += 1
if (!available) {
throw Object.assign(new Error("connect ECONNREFUSED"), { code: "ECONNREFUSED" })
}
return {
wsUrl: "ws://127.0.0.1:8769/ws?token=fresh",
apiUrl: "http://127.0.0.1:8769",
apiToken: "fresh-api-token",
}
},
targetEndpoint: "127.0.0.1:8769",
checkHealth: async () => available ? "ready" : "degraded",
startupFailureDelayMs: 8,
reconnectDelayMs: 2,
startupRetryMaxDelayMs: 2,
onEvent: () => undefined,
onStatus: (status) => statuses.push(status),
})
try {
client.connect()
await waitUntil(() => statuses.includes("unavailable"))
available = true
await waitUntil(() => sockets.length === 1)
sockets[0]?.emit("open")
await waitUntil(() => statuses.at(-1) === "connected")
expect(attempts).toBeGreaterThan(1)
expect(statuses.indexOf("starting")).toBeLessThan(statuses.indexOf("unavailable"))
expect(statuses.indexOf("unavailable")).toBeLessThan(statuses.lastIndexOf("connected"))
} finally {
client.close()
Object.defineProperty(globalThis, "WebSocket", { configurable: true, value: original })
}
})
test("sanitizes arbitrary connection errors and authenticated URLs", () => {
const authenticated = "wss://user:password@127.0.0.1:8769/ws"
+ "?token=socket-secret&api_token=api-secret"
const unknown = new Error(`could not reach ${authenticated}`)
const refused = Object.assign(new Error(`ECONNREFUSED ${authenticated}`), {
code: "ECONNREFUSED",
})
expect(connectionEndpoint(authenticated)).toBe("127.0.0.1:8769")
expect(sanitizeConnectionFailure(unknown)).toBe("connection failed")
expect(sanitizeConnectionFailure(refused)).toBe("connection refused")
expect(sanitizeConnectionFailure(unknown)).not.toContain("socket-secret")
})
test("reports a permanent bootstrap rejection without retrying", async () => {
let attempts = 0
const statuses: string[] = []
@@ -588,13 +767,19 @@ describe("gateway protocol", () => {
})
try {
const statuses: Array<{
status: ConnectionStatus
detail?: string
info?: ConnectionStatusInfo
}> = []
const client = new NanobotClient({
url: "ws://nanobot.test/ws",
reconnectDelayMs: 1,
onEvent: () => undefined,
onStatus: () => undefined,
onStatus: (status, detail, info) => statuses.push({ status, detail, info }),
})
client.connect()
sockets[0]?.emit("open")
sockets[0]?.emit("message", {
data: JSON.stringify({ event: "ready", chat_id: "", client_id: "client" }),
})
@@ -605,11 +790,21 @@ describe("gateway protocol", () => {
await Bun.sleep(5)
expect(sockets).toHaveLength(2)
const reconnecting = [...statuses].reverse().find(
({ status }) => status === "reconnecting",
)
expect(reconnecting).toMatchObject({
status: "reconnecting",
detail: "connection closed",
info: { endpoint: "nanobot.test", attempt: 1 },
})
sockets[1]?.emit("open")
sockets[1]?.emit("message", {
data: JSON.stringify({ event: "ready", chat_id: "", client_id: "client-2" }),
})
const outbound = sockets[1]?.sent.map((value) => JSON.parse(value)) || []
expect(outbound).toEqual([{ type: "attach", chat_id: "generated-chat" }])
expect(statuses.at(-1)?.status).toBe("connected")
client.close()
} finally {
Object.defineProperty(globalThis, "WebSocket", { configurable: true, value: original })
+223 -18
View File
@@ -1,4 +1,21 @@
export type ConnectionStatus = "connecting" | "connected" | "closed" | "error"
export type ConnectionStatus =
| "starting"
| "connecting"
| "connected"
| "reconnecting"
| "unavailable"
| "closed"
| "error"
export interface ConnectionStatusInfo {
endpoint: string
attempt: number
elapsedMs: number
retryInMs?: number
health?: GatewayHealthStatus
}
export type GatewayHealthStatus = "ready" | "degraded" | "unreachable"
export interface ToolProgressEvent {
version?: number
@@ -172,14 +189,16 @@ type OutboundEvent =
export interface ClientOptions {
url?: string
resolveConnection?: () => Promise<GatewayConnection>
checkHealth?: () => Promise<GatewayHealthStatus>
onConnection?: (connection: GatewayConnection) => void
connectionRetryLabel?: string
targetEndpoint?: string
startupFailureDelayMs?: number
startupRetryMaxDelayMs?: number
chatId?: string
initialWorkspaceScope?: WorkspaceScopePayload
reconnectDelayMs?: number
onEvent: (event: InboundEvent) => void
onStatus: (status: ConnectionStatus, detail?: string) => void
onStatus: (status: ConnectionStatus, detail?: string, info?: ConnectionStatusInfo) => void
}
export interface GatewayApiConnection {
@@ -940,6 +959,92 @@ export async function fetchGatewayConnection(
}
}
/** Read gateway readiness without sending bootstrap or API credentials. */
export async function fetchGatewayHealth(
healthUrl: string,
timeoutMs = 400,
): Promise<GatewayHealthStatus> {
const controller = new AbortController()
const timer = setTimeout(() => controller.abort(), timeoutMs)
try {
const response = await fetch(healthUrl, { signal: controller.signal })
if (response.status !== 200 && response.status !== 503) return "unreachable"
const payload: unknown = await response.json()
if (!isRecord(payload)) return "unreachable"
if (
response.status === 503
&& payload.status === "degraded"
&& payload.ready === false
&& payload.process === "alive"
) return "degraded"
if (response.status === 200 && payload.status === "ok" && payload.ready !== false) {
return "ready"
}
return "unreachable"
} catch {
return "unreachable"
} finally {
clearTimeout(timer)
}
}
/** Return only the authority users can act on, never credentials or an authenticated path. */
export function connectionEndpoint(value: string | undefined): string {
if (!value) return "local gateway"
try {
return new URL(value).host || "local gateway"
} catch {
return "local gateway"
}
}
/** Reduce arbitrary fetch/WebSocket errors to a small set of credential-safe reasons. */
export function sanitizeConnectionFailure(error: unknown): string {
const signals: string[] = []
const seen = new Set<unknown>()
const collect = (value: unknown): void => {
if (value === null || value === undefined || seen.has(value)) return
if (typeof value === "object") seen.add(value)
if (typeof value === "string") {
signals.push(value)
return
}
if (value instanceof Error) {
signals.push(value.name, value.message)
collect(value.cause)
if (value instanceof AggregateError) {
for (const nested of value.errors) collect(nested)
}
return
}
if (!isRecord(value)) return
if (typeof value.code === "string") signals.push(value.code)
collect(value.cause)
if (Array.isArray(value.errors)) {
for (const nested of value.errors) collect(nested)
}
}
collect(error)
const signal = signals.join(" ")
if (/ECONNREFUSED|connection refused/iu.test(signal)) return "connection refused"
if (/ETIMEDOUT|timed? out|timeout/iu.test(signal)) return "connection timed out"
if (/ENOTFOUND|EAI_AGAIN|name not resolved|host not found/iu.test(signal)) {
return "host not found"
}
if (/certificate|TLS|SSL/iu.test(signal)) return "secure connection failed"
const bootstrapStatus = signal.match(/gateway bootstrap failed:\s*HTTP\s*(\d{3})/iu)
if (bootstrapStatus?.[1]) return `gateway bootstrap failed: HTTP ${bootstrapStatus[1]}`
if (/bootstrap response is missing ws_url/iu.test(signal)) {
return "gateway bootstrap response is missing ws_url"
}
if (/bootstrap response (?:has an invalid ws_url|is invalid)/iu.test(signal)) {
return "gateway bootstrap response is invalid"
}
if (/gateway is still starting/iu.test(signal)) return "gateway is still starting"
if (/fetch failed|failed to fetch|network error/iu.test(signal)) return "network request failed"
return "connection failed"
}
export class NanobotClient {
private socket: WebSocket | null = null
private chatId = ""
@@ -949,13 +1054,22 @@ export class NanobotClient {
private closedByClient = false
private opening = false
private connectedOnce = false
private connectionAttempt = 0
private retryStartedAt = 0
private nextRetryAt = 0
private lastFailure = ""
private healthStatus: GatewayHealthStatus | undefined
private failureEscalationTimer: ReturnType<typeof setTimeout> | null = null
private readonly endpoint: string
private readonly pendingMutations = new Map<string, {
resolve: (value: unknown) => void
reject: (error: Error) => void
timer: ReturnType<typeof setTimeout>
}>()
constructor(private readonly options: ClientOptions) {}
constructor(private readonly options: ClientOptions) {
this.endpoint = options.targetEndpoint || connectionEndpoint(options.url)
}
get activeChatId(): string {
return this.chatId
@@ -963,13 +1077,21 @@ export class NanobotClient {
connect(): void {
this.closedByClient = false
this.connectionAttempt = 0
this.reconnectAttempt = 0
this.retryStartedAt = Date.now()
this.nextRetryAt = 0
this.lastFailure = ""
this.healthStatus = undefined
void this.open()
}
private async open(): Promise<void> {
if (this.socket || this.opening || this.closedByClient) return
this.opening = true
this.options.onStatus("connecting")
this.nextRetryAt = 0
this.connectionAttempt += 1
this.reportConnectionProgress()
let url = this.options.url
try {
if (this.options.resolveConnection) {
@@ -980,37 +1102,52 @@ export class NanobotClient {
}
} catch (error) {
if (!this.closedByClient) {
this.lastFailure = sanitizeConnectionFailure(error)
if (error instanceof GatewayConnectionError && !error.retryable) {
this.options.onStatus("error", error.message)
this.clearFailureEscalation()
this.options.onStatus("error", this.lastFailure, this.connectionInfo())
return
}
this.options.onStatus(
"connecting",
this.options.connectionRetryLabel || "gateway unavailable",
)
this.scheduleReconnect(false)
await this.checkHealthAndScheduleReconnect()
}
return
} finally {
this.opening = false
}
if (!url) {
this.options.onStatus("error", "gateway URL is not configured")
this.options.onStatus("error", "gateway URL is not configured", this.connectionInfo())
return
}
const socket = new WebSocket(url)
let socket: WebSocket
try {
socket = new WebSocket(url)
} catch (error) {
this.lastFailure = sanitizeConnectionFailure(error)
await this.checkHealthAndScheduleReconnect()
return
}
let opened = false
this.socket = socket
socket.addEventListener("open", () => {
if (this.socket !== socket) return
opened = true
this.connectedOnce = true
this.connectionAttempt = 0
this.reconnectAttempt = 0
this.options.onStatus("connected")
this.retryStartedAt = 0
this.nextRetryAt = 0
this.lastFailure = ""
this.healthStatus = "ready"
this.clearFailureEscalation()
this.options.onStatus("connected", undefined, this.connectionInfo())
})
socket.addEventListener("message", (message) => {
if (this.socket === socket) this.handleMessage(String(message.data))
})
socket.addEventListener("error", () => {
if (this.socket === socket) this.options.onStatus("error", "connection failed")
if (this.socket !== socket) return
this.lastFailure = "connection failed"
this.reportRetryState()
})
socket.addEventListener("close", () => {
if (this.socket !== socket) return
@@ -1020,7 +1157,14 @@ export class NanobotClient {
this.options.onStatus("closed")
return
}
this.scheduleReconnect()
if (opened) {
this.connectionAttempt = 0
this.reconnectAttempt = 0
this.retryStartedAt = Date.now()
}
if (!this.lastFailure) this.lastFailure = "connection closed"
this.reportRetryState()
void this.checkHealthAndScheduleReconnect()
})
}
@@ -1028,6 +1172,7 @@ export class NanobotClient {
this.closedByClient = true
if (this.reconnectTimer) clearTimeout(this.reconnectTimer)
this.reconnectTimer = null
this.clearFailureEscalation()
const socket = this.socket
this.socket = null
socket?.close()
@@ -1182,20 +1327,80 @@ export class NanobotClient {
this.options.onEvent(event)
}
private scheduleReconnect(announce = true): void {
private scheduleReconnect(): void {
if (this.reconnectTimer || this.closedByClient) return
if (!this.retryStartedAt) this.retryStartedAt = Date.now()
const base = this.options.reconnectDelayMs ?? 500
const maxDelay = this.connectedOnce
? 8_000
: this.options.startupRetryMaxDelayMs ?? 8_000
const delay = Math.min(maxDelay, base * 2 ** Math.min(this.reconnectAttempt++, 4))
if (announce) this.options.onStatus("connecting", `reconnecting in ${delay}ms`)
this.nextRetryAt = Date.now() + delay
this.reportRetryState()
this.reconnectTimer = setTimeout(() => {
this.reconnectTimer = null
void this.open()
}, delay)
}
private async checkHealthAndScheduleReconnect(): Promise<void> {
if (this.options.checkHealth) {
try {
this.healthStatus = await this.options.checkHealth()
} catch {
this.healthStatus = "unreachable"
}
}
if (!this.closedByClient) this.scheduleReconnect()
}
private connectionInfo(): ConnectionStatusInfo {
return {
endpoint: this.endpoint,
attempt: Math.max(1, this.connectionAttempt),
elapsedMs: this.retryStartedAt ? Math.max(0, Date.now() - this.retryStartedAt) : 0,
...(this.nextRetryAt
? { retryInMs: Math.max(0, this.nextRetryAt - Date.now()) }
: {}),
...(this.healthStatus ? { health: this.healthStatus } : {}),
}
}
private reportConnectionProgress(): void {
if (this.connectedOnce) {
this.options.onStatus("reconnecting", this.lastFailure || undefined, this.connectionInfo())
return
}
const phase = this.options.resolveConnection ? "starting" : "connecting"
this.options.onStatus(phase, undefined, this.connectionInfo())
}
private reportRetryState(): void {
const info = this.connectionInfo()
if (this.connectedOnce) {
this.options.onStatus("reconnecting", this.lastFailure, info)
return
}
const failureDelay = this.options.startupFailureDelayMs ?? 3_000
if (info.elapsedMs >= failureDelay) {
this.clearFailureEscalation()
this.options.onStatus("unavailable", this.lastFailure, info)
return
}
this.reportConnectionProgress()
if (this.failureEscalationTimer) return
this.failureEscalationTimer = setTimeout(() => {
this.failureEscalationTimer = null
if (this.closedByClient || this.connectedOnce || !this.lastFailure) return
this.options.onStatus("unavailable", this.lastFailure, this.connectionInfo())
}, Math.max(0, failureDelay - info.elapsedMs))
}
private clearFailureEscalation(): void {
if (this.failureEscalationTimer) clearTimeout(this.failureEscalationTimer)
this.failureEscalationTimer = null
}
private write(event: OutboundEvent): void {
if (!this.socket || this.socket.readyState !== WebSocket.OPEN) {
throw new Error("gateway connection is not open")