diff --git a/tui/README.md b/tui/README.md index 9d0d58aac..2228b725a 100644 --- a/tui/README.md +++ b/tui/README.md @@ -29,7 +29,10 @@ Changes reuse the gateway's normal model command and workspace policy checks. When you scroll away from the latest output, the scrollbar and `Ctrl+End` hint appear only until you return to the bottom. Large pastes are represented by a short editable placeholder in the -composer; nanobot sends the original text unchanged. +composer; nanobot sends the original text unchanged. Press `Ctrl+V` or `Alt+V` while the composer +is focused to attach an image from the system clipboard. Image bytes stay behind removable +`[Image #n]` placeholders until the message is sent; each placeholder behaves as one unit, and +deleting it removes its image. While nanobot is working, the composer prompt becomes `Enter send now · Tab send next`; narrow terminals shorten it to `Enter now · Tab next`. diff --git a/tui/src/app.test.ts b/tui/src/app.test.ts index b170efb9b..9e649f8bb 100644 --- a/tui/src/app.test.ts +++ b/tui/src/app.test.ts @@ -1,5 +1,12 @@ import { afterEach, describe, expect, test } from "bun:test" -import { BoxRenderable, CliRenderEvents, TextareaRenderable, TextRenderable } from "@opentui/core" +import { + BoxRenderable, + CliRenderEvents, + StyledText, + TextareaRenderable, + TextAttributes, + TextRenderable, +} from "@opentui/core" import { MockTreeSitterClient, createTestRenderer, @@ -15,7 +22,8 @@ import type { WorkspaceScopePayload, } from "./protocol" import type { HostAgentState, HostMetadata, TuiHost } from "./host" -import type { Transcript } from "./transcript" +import type { ClipboardImageReader } from "./clipboard-image" +import { userMessageText, type Transcript } from "./transcript" const options: AppOptions = { wsUrl: "ws://localhost.invalid/ws", @@ -51,6 +59,20 @@ test("formats a reusable session ID after exit", () => { ) }) +test("projects image media as stable placeholders without exposing filenames", () => { + expect(userMessageText("What is this?", [ + { name: "clipboard-image-2.png" }, + { kind: "image", name: "screenshot.png" }, + { kind: "file", name: "report.pdf" }, + ])).toBe([ + "What is this? [Image #2] [Image #1]", + "Attachments: report.pdf", + ].join("\n")) + expect(userMessageText("What is this?", [ + { name: "clipboard-image-1.png" }, + ], "What is this? [Image #1]")).toBe("What is this? [Image #1]") +}) + function contrastRatio(foreground: string, background: string): number { const luminance = (color: string) => { const channel = (offset: number) => { @@ -306,6 +328,278 @@ describe("NanobotTui layout", () => { expect(ui.composer.plainText).toBe("") }) + test("pastes clipboard images into removable placeholders and sends their data", async () => { + const sent: string[] = [] + const sentOptions: MessageOptions[] = [] + let disposed = false + const clipboard: ClipboardImageReader = { + read: async () => ({ + mimeType: "image/png", + dataUrl: "data:image/png;base64,AAEC/w==", + }), + dispose: async () => { disposed = true }, + } + setup = await createRenderer({ width: 72, height: 20, screenMode: "alternate-screen" }) + const transport = client(sent, [], [], sentOptions) + const recordSend = transport.send + transport.send = (content, messageOptions) => { + recordSend(content, messageOptions) + return `image-turn-${sent.length}` + } + const app = NanobotTui.mount( + setup.renderer, + options, + transport, + new MockTreeSitterClient({ autoResolveTimeout: 0 }), + undefined, + clipboard, + ) + app.accept({ event: "attached", chat_id: "chat" }) + await waitUntil(() => (app as unknown as { ready: boolean }).ready) + const ui = app as unknown as { + composer: TextareaRenderable + draft: { imageCount: number } + promptHistory: string[] + status: { plainText: string } + transcript: { + userMessages: Set<{ renderable: TextRenderable }> + } + } + + setup.mockInput.pressKey("v", { ctrl: true }) + await waitUntil(() => ui.composer.plainText === "[Image #1] ") + expect(ui.status.plainText).toContain("Pasted Image #1") + const placeholderStyle = ui.composer.syntaxStyle?.getStyle("image.placeholder") + expect(placeholderStyle?.bold).toBeTrue() + expect(placeholderStyle?.fg?.toInts().slice(0, 3)).toEqual([239, 142, 48]) + const placeholderStyleId = ui.composer.syntaxStyle?.getStyleId("image.placeholder") + if (placeholderStyleId === null || placeholderStyleId === undefined) { + throw new Error("image placeholder style was not registered") + } + expect(ui.composer.getLineHighlights(0)).toEqual([{ + start: 0, + end: 10, + styleId: placeholderStyleId, + priority: 100, + hlRef: 0, + }]) + ui.composer.setText("") + await waitUntil(() => ui.draft.imageCount === 0) + expect(ui.composer.getLineHighlights(0)).toEqual([]) + + setup.mockInput.pressKey("v", { ctrl: true }) + await waitUntil(() => ui.composer.plainText === "[Image #1] ") + await setup.mockInput.typeText("[Image #1]") + ui.composer.submit() + await waitUntil(() => ui.status.plainText.includes("Duplicate image placeholder")) + expect(sent).toEqual([]) + ui.composer.setText("[Image #1]") + ui.composer.submit() + await waitUntil(() => sent.length === 1) + expect(sent).toEqual([""]) + expect(ui.promptHistory).toEqual([]) + expect(sentOptions[0]?.media).toEqual([{ + data_url: "data:image/png;base64,AAEC/w==", + name: "clipboard-image-1.png", + }]) + expect(sentOptions[0]).not.toHaveProperty("displayContent") + await setup.flush() + const frame = setup.captureCharFrame() + expect(frame).toContain("[Image #1]") + expect(frame).not.toContain("clipboard-image-1.png") + const userContent = [...ui.transcript.userMessages].at(-1)?.renderable.content + expect(userContent).toBeInstanceOf(StyledText) + const imageChunk = (userContent as StyledText).chunks.find(({ text }) => text === "[Image #1]") + expect(imageChunk?.attributes).toBe(TextAttributes.BOLD) + expect(imageChunk?.fg?.toInts().slice(0, 3)).toEqual([239, 142, 48]) + + await setup.mockInput.typeText("这是什么? ") + setup.mockInput.pressKey("v", { ctrl: true }) + await waitUntil(() => ui.status.plainText.includes("Pasted Image #1"), 3_000) + expect(ui.composer.plainText).toBe("这是什么? [Image #1] ") + setup.mockInput.pressTab() + expect(ui.status.plainText).toContain("Images cannot be queued") + expect(ui.composer.plainText).toBe("这是什么? [Image #1] ") + ui.composer.submit() + await waitUntil(() => sent.length === 2) + expect(sent[1]).toBe("这是什么?") + expect(sentOptions[1]?.media).toHaveLength(1) + expect(sentOptions[1]).not.toHaveProperty("displayContent") + await setup.flush() + expect(setup.captureCharFrame()).toContain("这是什么? [Image #1]") + + setup.renderer.destroy() + expect(disposed).toBeTrue() + }) + + test("keeps image placeholders atomic for cursor movement and deletion", async () => { + const clipboard: ClipboardImageReader = { + read: async () => ({ + mimeType: "image/png", + dataUrl: "data:image/png;base64,AAEC/w==", + }), + dispose: async () => undefined, + } + setup = await createRenderer({ width: 72, height: 20, screenMode: "alternate-screen" }) + const app = NanobotTui.mount( + setup.renderer, + options, + client(), + new MockTreeSitterClient({ autoResolveTimeout: 0 }), + undefined, + clipboard, + ) + app.accept({ event: "attached", chat_id: "chat" }) + await waitUntil(() => (app as unknown as { ready: boolean }).ready) + const ui = app as unknown as { + composer: TextareaRenderable + draft: { imageCount: number } + status: { plainText: string } + } + + setup.mockInput.pressKey("v", { ctrl: true }) + await waitUntil(() => ui.composer.plainText === "[Image #1] ") + await setup.flush() + await setup.mockMouse.click(ui.composer.x + 5, ui.composer.y) + expect(ui.composer.cursorOffset > 0 && ui.composer.cursorOffset < 10).toBeFalse() + ui.composer.cursorOffset = 0 + setup.mockInput.pressArrow("right") + await waitUntil(() => ui.composer.cursorOffset === 10) + setup.mockInput.pressArrow("left") + await waitUntil(() => ui.composer.cursorOffset === 0) + + setup.mockInput.pressArrow("right", { shift: true }) + await waitUntil(() => ui.composer.cursorOffset === 10) + await setup.mockInput.typeText("replacement") + await waitUntil(() => ui.draft.imageCount === 0) + expect(ui.composer.plainText).toContain("replacement") + expect(ui.composer.plainText).not.toContain("Image #1") + + ui.composer.setText("") + setup.mockInput.pressKey("v", { ctrl: true }) + await waitUntil(() => ui.composer.plainText === "[Image #1] ") + ui.composer.cursorOffset = 0 + setup.mockInput.pressKey("DELETE") + await waitUntil(() => ui.draft.imageCount === 0) + expect(ui.composer.plainText.trim()).toBe("") + expect(ui.status.plainText).toContain("Removed Image #1") + + ui.composer.setText("") + setup.mockInput.pressKey("v", { ctrl: true }) + await waitUntil(() => ui.composer.plainText === "[Image #1] ") + ui.composer.cursorOffset = 10 + setup.mockInput.pressBackspace() + await waitUntil(() => ui.draft.imageCount === 0) + expect(ui.composer.plainText.trim()).toBe("") + + ui.composer.setText("") + setup.mockInput.pressKey("v", { ctrl: true }) + await waitUntil(() => ui.composer.plainText === "[Image #1] ") + ui.composer.setText("Image #1] ") + await waitUntil(() => ui.draft.imageCount === 0) + expect(ui.composer.plainText.trim()).toBe("") + }) + + test("keeps clipboard failures visible while an agent turn is active", async () => { + const sent: string[] = [] + const clipboard: ClipboardImageReader = { + read: async () => { throw new Error("No image in clipboard") }, + dispose: async () => undefined, + } + setup = await createRenderer({ width: 72, height: 20, screenMode: "alternate-screen" }) + const app = NanobotTui.mount( + setup.renderer, + options, + client(sent), + new MockTreeSitterClient({ autoResolveTimeout: 0 }), + undefined, + clipboard, + ) + app.accept({ event: "attached", chat_id: "chat" }) + await waitUntil(() => (app as unknown as { ready: boolean }).ready) + const composer = (app as unknown as { composer: TextareaRenderable }).composer + composer.setText("start") + composer.submit() + await waitUntil(() => sent.length === 1) + + setup.mockInput.pressKey("v", { ctrl: true }) + await waitUntil(() => setup?.captureCharFrame().includes("No image in clipboard") === true) + }) + + test("keeps image placeholders out of command arguments", async () => { + const sent: string[] = [] + const clipboard: ClipboardImageReader = { + read: async () => ({ + mimeType: "image/png", + dataUrl: "data:image/png;base64,AAEC/w==", + }), + dispose: async () => undefined, + } + setup = await createRenderer({ width: 72, height: 20, screenMode: "alternate-screen" }) + const app = NanobotTui.mount( + setup.renderer, + options, + client(sent), + new MockTreeSitterClient({ autoResolveTimeout: 0 }), + undefined, + clipboard, + ) + app.accept({ event: "attached", chat_id: "chat" }) + await waitUntil(() => (app as unknown as { ready: boolean }).ready) + const ui = app as unknown as { + composer: TextareaRenderable + status: { plainText: string } + commandMenu: { setCommands(commands: SlashCommand[]): void } + } + ui.commandMenu.setCommands([{ + command: "/model", + title: "Model", + description: "Show or switch model presets", + argHint: "[preset]", + lifecycle: "side_channel", + acceptsArgs: true, + }]) + + await setup.mockInput.typeText("/model ") + setup.mockInput.pressKey("v", { ctrl: true }) + await waitUntil(() => ui.composer.plainText === "/model [Image #1] ") + ui.composer.submit() + await waitUntil(() => ui.status.plainText.includes("Images cannot be used with commands")) + + expect(sent).toEqual([]) + expect(ui.composer.plainText).toBe("/model [Image #1] ") + }) + + test("ignores a clipboard result that finishes after the renderer is destroyed", async () => { + let resolveRead: ((image: { + mimeType: "image/png" + dataUrl: string + }) => void) | undefined + let disposed = false + const clipboard: ClipboardImageReader = { + read: () => new Promise((resolve) => { resolveRead = resolve }), + dispose: async () => { disposed = true }, + } + setup = await createRenderer({ width: 72, height: 20, screenMode: "alternate-screen" }) + const app = NanobotTui.mount( + setup.renderer, + options, + client(), + new MockTreeSitterClient({ autoResolveTimeout: 0 }), + undefined, + clipboard, + ) + app.accept({ event: "attached", chat_id: "chat" }) + await waitUntil(() => (app as unknown as { ready: boolean }).ready) + + setup.mockInput.pressKey("v", { ctrl: true }) + await waitUntil(() => resolveRead !== undefined) + setup.renderer.destroy() + resolveRead?.({ mimeType: "image/png", dataUrl: "data:image/png;base64,AAAA" }) + await Bun.sleep(10) + expect(disposed).toBeTrue() + }) + test("steers with Enter, queues with Tab, and restores queued text with Alt+Up", async () => { const sent: string[] = [] const sentOptions: MessageOptions[] = [] @@ -412,6 +706,11 @@ describe("NanobotTui layout", () => { turn_id: "remote-steer", active_turn_id: "remote-turn", starts_turn: false, + media_urls: [{ + kind: "image", + url: "/api/media/sig/image", + name: "clipboard-image-2.png", + }], }) await setup.flush() @@ -420,6 +719,9 @@ describe("NanobotTui layout", () => { expect(occurrences(frame, "hello from terminal A")).toBe(1) expect(occurrences(frame, "Attachments: report.pdf")).toBe(1) expect(occurrences(frame, "one more remote detail")).toBe(1) + expect(occurrences(frame, "[Image #2]")).toBe(1) + expect(frame).toContain("one more remote detail [Image #2]") + expect(frame).not.toContain("clipboard-image-2.png") expect(state.activeTurn).toBeTrue() expect(state.activeTurnId).toBe("remote-turn") @@ -1790,19 +2092,26 @@ describe("NanobotTui layout", () => { composer: { backgroundColor: { intent: string; toInts(): number[] } textColor: { toInts(): number[] } + syntaxStyle: { getStyle(name: string): { fg?: { toInts(): number[] } } | undefined } | null } transcript: { markdown: Set<{ syntaxStyle: object }> frames: Set<{ borderColor: { toInts(): number[] } }> userRows: Set<{ backgroundColor: { intent: string; toInts(): number[] } }> - user(content: string): void + userMessages: Set<{ renderable: TextRenderable }> + user(content: string, turnId?: string, media?: Array<{ kind: "image"; name: string }>): void } } - internals.transcript.user("Existing question") + internals.transcript.user("Existing question", undefined, [{ + kind: "image", + name: "clipboard-image-1.png", + }]) const userRow = [...internals.transcript.userRows][0] + const userMessage = [...internals.transcript.userMessages][0] const markdown = [...internals.transcript.markdown][0] const sessionFrame = [...internals.transcript.frames][0] const darkSyntax = markdown?.syntaxStyle + const darkComposerSyntax = internals.composer.syntaxStyle expect(userRow?.backgroundColor.intent).toBe("default") @@ -1821,6 +2130,12 @@ describe("NanobotTui layout", () => { expect(sessionFrame?.borderColor.toInts().slice(0, 3)).toEqual([212, 212, 216]) expect(userRow?.backgroundColor.toInts().slice(0, 3)).toEqual([240, 240, 240]) expect(markdown?.syntaxStyle).not.toBe(darkSyntax) + expect(internals.composer.syntaxStyle).not.toBe(darkComposerSyntax) + expect(internals.composer.syntaxStyle?.getStyle("image.placeholder")?.fg?.toInts().slice(0, 3)) + .toEqual([185, 77, 11]) + const recolored = userMessage?.renderable.content as StyledText + expect(recolored.chunks.find(({ text }) => text === "[Image #1]")?.fg?.toInts().slice(0, 3)) + .toEqual([185, 77, 11]) }) test("distinguishes the composer with a quiet focus edge", async () => { diff --git a/tui/src/app.ts b/tui/src/app.ts index 20689a14e..11f4097d8 100644 --- a/tui/src/app.ts +++ b/tui/src/app.ts @@ -65,7 +65,11 @@ import { type TranscriptNavigation, type TranscriptTheme, } from "./transcript" -import { ComposerDraft } from "./composer-draft" +import { ComposerDraft, MAX_DRAFT_IMAGES } from "./composer-draft" +import { + createClipboardImageReader, + type ClipboardImageReader, +} from "./clipboard-image" import { BranchMenu, branchPoints } from "./branch-menu" import { MentionMenu, @@ -184,6 +188,7 @@ const LIGHT: Palette = { const COMPOSER_PLACEHOLDER = "Ask nanobot anything" const ACTIVE_COMPOSER_PLACEHOLDER = "Enter send now · Tab send next" const COMPACT_ACTIVE_COMPOSER_PLACEHOLDER = "Enter now · Tab next" +const IMAGE_PLACEHOLDER_STYLE = "image.placeholder" const SHIMMER_PAUSE = 16 const SHIMMER_BAND = 4 const SHIMMER_INTERVAL_MS = 80 @@ -259,6 +264,12 @@ function syntaxStyle(palette: Palette): SyntaxStyle { }) } +function composerSyntaxStyle(palette: Palette): SyntaxStyle { + return SyntaxStyle.fromStyles({ + [IMAGE_PLACEHOLDER_STYLE]: { fg: RGBA.fromHex(palette.accent), bold: true }, + }) +} + function transcriptTheme(palette: Palette, backgroundKnown: boolean): TranscriptTheme { return { text: palette.text, @@ -440,6 +451,7 @@ export class NanobotTui { private readonly titleText: TextRenderable private readonly composerFrame: BoxRenderable private readonly composer: TextareaRenderable + private composerSyntax: SyntaxStyle private readonly status: TextRenderable private readonly meta: TextRenderable private readonly host: TuiHost @@ -510,8 +522,14 @@ export class NanobotTui { private hostWorkspace: string private hostBranch: string private readonly apiReauthenticator: ApiReauthenticator | undefined + private readonly clipboardImageReader: ClipboardImageReader private apiRefreshPromise: Promise | null = null private skillLoadId = 0 + private clipboardImagePending = false + private clipboardPasteGeneration = 0 + private composerValue = "" + private composerCursor = 0 + private reconcilingComposer = false private constructor( renderer: CliRenderer, @@ -519,8 +537,10 @@ export class NanobotTui { client?: ChatClient, treeSitterClient = getTreeSitterClient(), host: TuiHost = createTuiHost({}), + clipboardImageReader: ClipboardImageReader = createClipboardImageReader(), ) { this.renderer = renderer + this.clipboardImageReader = clipboardImageReader this.defaultModelName = options.model this.defaultModelPreset = options.modelPreset this.modelName = options.model @@ -534,6 +554,7 @@ export class NanobotTui { this.backgroundKnown = options.theme !== "auto" || renderer.themeMode !== null this.activeThemeMode = this.resolveThemeMode(renderer.themeMode) this.palette = this.activeThemeMode === "light" ? LIGHT : DARK + this.composerSyntax = composerSyntaxStyle(this.palette) this.host = host this.transcript = new Transcript( renderer, @@ -730,6 +751,7 @@ export class NanobotTui { backgroundColor: composerSurface, focusedBackgroundColor: composerSurface, cursorColor: this.palette.accent, + syntaxStyle: this.composerSyntax, // A steady line cursor avoids the block-cell trails produced by some // terminals when a retained full-screen UI redraws around the composer. cursorStyle: { style: "line", blinking: false }, @@ -743,23 +765,14 @@ export class NanobotTui { { name: "return", action: "submit" }, ], onCursorChange: () => { + this.keepComposerCursorOutsideImages() if (!this.sessionMenu.visible && !this.branchMenu.visible) this.syncComposerMenus() }, - onContentChange: () => { - this.draft.prune(this.composer.plainText) - const clearedUnsent = this.unsentSubmit && !this.composer.plainText.trim() - if (clearedUnsent) this.unsentSubmit = false - this.runtimeControls.hide() - if (this.contextPanel.visible && this.composer.plainText) this.contextPanel.hide() - this.syncComposerPlaceholder() - if (this.sessionMenu.visible) this.syncSessionMenu() - else if (this.branchMenu.visible) this.syncBranchMenu() - else this.syncComposerMenus() - this.resizeComposer() - if (clearedUnsent && !this.activeTurn) { - this.status.content = this.ready ? this.readyStatus() : this.connectionMessage - } - }, + onContentChange: () => this.handleComposerContentChange(), + onMouseDown: () => queueMicrotask(() => this.keepComposerCursorOutsideImages()), + onMouseUp: () => queueMicrotask(() => this.keepComposerCursorOutsideImages()), + onMouseDrag: () => queueMicrotask(() => this.keepComposerCursorOutsideImages()), + onMouseDragEnd: () => queueMicrotask(() => this.keepComposerCursorOutsideImages()), // IMEs may commit their final composed glyph after Enter. Matching the // OpenCode/OpenTUI integration, defer twice before reading plainText. onSubmit: () => this.deferSubmit(), @@ -845,8 +858,16 @@ export class NanobotTui { client?: ChatClient, treeSitterClient?: TreeSitterClient, host?: TuiHost, + clipboardImageReader?: ClipboardImageReader, ): NanobotTui { - return new NanobotTui(renderer, options, client, treeSitterClient, host) + return new NanobotTui( + renderer, + options, + client, + treeSitterClient, + host, + clipboardImageReader, + ) } async start(): Promise { @@ -889,7 +910,6 @@ export class NanobotTui { private submit(): void { if (this.quitting || this.composer.isDestroyed) return const visibleContent = this.composer.plainText.trim() - const content = this.draft.expand(visibleContent).trim() if (this.sessionLoading) { this.status.content = "Loading sessions…" return @@ -945,6 +965,10 @@ export class NanobotTui { return } const command = this.commandMenu.resolve(visibleContent) + if ((command || visibleContent.startsWith("!")) && this.draft.media(visibleContent).length) { + this.status.content = "Images cannot be used with commands · remove the image first" + return + } if (command?.source === "tui") { if (command.command.action === "sessions") void this.openSessions() else if (command.command.action === "context") void this.openContext() @@ -968,7 +992,8 @@ export class NanobotTui { this.markSubmitUnsent() return } - const prompt = { content, options: mentionOptions(content, this.availableMentions()) } + const prompt = this.composerPrompt() + if (!this.canSendPrompt(prompt)) return if (this.activeTurn) { this.sendPrompt(prompt, true) return @@ -990,7 +1015,12 @@ export class NanobotTui { this.mentionMenu.hide() this.skillMenu.hide() this.recordPrompt(prompt.content) - this.transcript.user(prompt.content, turnId) + this.transcript.user( + prompt.content, + turnId, + prompt.options.media, + prompt.displayContent, + ) this.hostBlocked = false this.setCurrentTask(prompt.content) if (steering) { @@ -1080,12 +1110,13 @@ export class NanobotTui { this.reconcileTurnOwnership(event) return case "user_message": { - const attachments = event.media_urls?.map((media) => media.name).filter(Boolean) || [] - const content = [ + if (this.transcript.user( event.text, - attachments.length ? `Attachments: ${attachments.join(", ")}` : "", - ].filter(Boolean).join("\n") - if (this.transcript.user(content, event.turn_id)) this.recordPrompt(event.text) + event.turn_id, + event.media_urls, + )) { + this.recordPrompt(event.text) + } this.hostBlocked = false this.setCurrentTask(event.text) this.reconcileTurnOwnership(event) @@ -1544,6 +1575,36 @@ export class NanobotTui { return queue } + private composerPrompt(): QueuedPrompt { + const visible = this.composer.plainText.trim() + const content = this.draft.expand(visible).trim() + const media = this.draft.media(visible) + const displayContent = this.draft.display(visible).trim() + return { + content, + ...(media.length ? { displayContent } : {}), + options: { + ...mentionOptions(content, this.availableMentions()), + ...(media.length ? { media } : {}), + }, + } + } + + private hasPrompt(prompt: QueuedPrompt): boolean { + return Boolean(prompt.content || prompt.options.media?.length) + } + + private canSendPrompt(prompt: QueuedPrompt): boolean { + if (this.draft.hasImageLabelConflict(this.composer.plainText)) { + this.status.content = "Duplicate image placeholder text · rename or remove it before sending" + return false + } + if (!this.hasPrompt(prompt)) return false + if ((prompt.options.media?.length || 0) <= MAX_DRAFT_IMAGES) return true + this.status.content = `Remove images until ${MAX_DRAFT_IMAGES} or fewer remain` + return false + } + private restoreQueuedPrompts(): void { const queued = this.promptQueue.restore() if (!queued.length) return @@ -1555,6 +1616,10 @@ export class NanobotTui { private queueFollowUp(): void { if (!this.activeTurn || !this.ready) return const visibleContent = this.composer.plainText.trim() + if (this.draft.media(visibleContent).length) { + this.status.content = "Images cannot be queued · press Enter to send now" + return + } const content = this.draft.expand(visibleContent).trim() if (!content) return this.promptQueue.enqueue({ @@ -1719,6 +1784,18 @@ export class NanobotTui { return } } + if ( + (key.ctrl || key.meta) + && key.name.toLocaleLowerCase() === "v" + && !this.sessionLoading + && !this.sessionMenu.visible + && !this.branchMenu.visible + && !this.contextPanel.visible + ) { + key.preventDefault() + void this.pasteClipboardImage() + return + } if (this.activeTurn && !key.ctrl && !key.meta && key.name === "tab") { this.queueFollowUp() key.preventDefault() @@ -1735,6 +1812,20 @@ export class NanobotTui { key.preventDefault() return } + if (!key.ctrl && !key.meta && !key.shift && (key.name === "left" || key.name === "right")) { + const direction = key.name === "left" ? -1 : 1 + const target = this.draft.moveImageCursor( + this.composer.plainText, + this.composerStringCursor(), + direction, + ) + if (target !== null) { + this.composerCursor = target + this.setComposerStringCursor(this.composer.plainText, target) + key.preventDefault() + return + } + } if (!key.ctrl && !key.meta && (key.name === "up" || key.name === "down")) { const direction = key.name === "up" ? -1 : 1 const boundary = direction < 0 ? 0 : this.composer.plainText.length @@ -1795,7 +1886,7 @@ export class NanobotTui { } private navigateHistory(direction: -1 | 1): boolean { - if (this.promptHistory.length === 0) return false + if (this.promptHistory.length === 0 || this.draft.imageCount) return false if (direction < 0) { if (this.historyCursor === this.promptHistory.length) this.historyDraft = this.composer.plainText if (this.historyCursor === 0) return false @@ -1846,6 +1937,11 @@ export class NanobotTui { this.composer.textColor = this.palette.text this.composer.focusedTextColor = this.palette.text this.composer.cursorColor = this.palette.accent + const previousComposerSyntax = this.composerSyntax + this.composerSyntax = composerSyntaxStyle(this.palette) + this.composer.syntaxStyle = this.composerSyntax + this.syncComposerImageHighlights(this.composer.plainText) + void this.renderer.idle().catch(() => {}).finally(() => previousComposerSyntax.destroy()) this.renderTitleColor() this.status.fg = this.palette.muted this.meta.fg = this.palette.faint @@ -2116,7 +2212,58 @@ export class NanobotTui { return this.composer.editBuffer.getTextRange(0, this.composer.cursorOffset).length } + private keepComposerCursorOutsideImages(): void { + if (this.reconcilingComposer) return + const value = this.composer.plainText + const cursor = this.composerStringCursor() + const target = this.draft.snapImageCursor(value, cursor, this.composerCursor) + this.composerCursor = target + if (target !== cursor) this.setComposerStringCursor(value, target) + } + + private handleComposerContentChange(): void { + if (this.reconcilingComposer) return + let value = this.composer.plainText + let cursor = this.composerStringCursor() + const edit = this.draft.reconcileImageEdit(this.composerValue, value, cursor) + if (edit.value !== value) { + this.reconcilingComposer = true + try { + this.composer.replaceText(edit.value) + this.composer.clearSelection() + this.setComposerStringCursor(edit.value, edit.cursor) + } finally { + this.reconcilingComposer = false + } + value = edit.value + cursor = edit.cursor + } + this.composerValue = value + this.composerCursor = cursor + this.draft.prune(value) + this.syncComposerImageHighlights(value) + const clearedUnsent = this.unsentSubmit && !value.trim() + if (clearedUnsent) this.unsentSubmit = false + this.runtimeControls.hide() + if (this.contextPanel.visible && value) this.contextPanel.hide() + this.syncComposerPlaceholder() + if (this.sessionMenu.visible) this.syncSessionMenu() + else if (this.branchMenu.visible) this.syncBranchMenu() + else this.syncComposerMenus() + this.resizeComposer() + if (clearedUnsent && !this.activeTurn) { + this.status.content = this.ready ? this.readyStatus() : this.connectionMessage + } + if (edit.removedImages.length) { + this.status.content = `Removed ${edit.removedImages.join(", ")}` + } + } + private setComposerStringCursor(value: string, cursor: number): void { + this.composer.cursorOffset = this.composerOffsetForStringIndex(value, cursor) + } + + private composerOffsetForStringIndex(value: string, cursor: number): number { const target = Math.min(Math.max(cursor, 0), value.length) const before = value.slice(0, target) const row = before.split("\n").length - 1 @@ -2130,10 +2277,25 @@ export class NanobotTui { if (candidateLength > target) break if (candidateLength === target) offset = candidate } - this.composer.cursorOffset = offset + return offset + } + + private syncComposerImageHighlights(value: string): void { + this.composer.clearAllHighlights() + const styleId = this.composerSyntax.getStyleId(IMAGE_PLACEHOLDER_STYLE) + if (styleId === null) return + for (const range of this.draft.imagePlaceholderRanges(value)) { + this.composer.addHighlightByCharRange({ + start: this.composerOffsetForStringIndex(value, range.start), + end: this.composerOffsetForStringIndex(value, range.end), + styleId, + priority: 100, + }) + } } private setComposer(content: string): void { + this.clipboardPasteGeneration += 1 this.draft.clear() this.composer.setText(content) this.composer.cursorOffset = content.length @@ -2141,8 +2303,42 @@ export class NanobotTui { private clearComposer(): void { this.unsentSubmit = false - this.draft.clear() - this.composer.setText("") + this.setComposer("") + } + + private async pasteClipboardImage(): Promise { + if (this.clipboardImagePending) return + if (this.draft.imageCount >= MAX_DRAFT_IMAGES) { + this.status.content = `A message can include up to ${MAX_DRAFT_IMAGES} images` + return + } + const generation = this.clipboardPasteGeneration + this.clipboardImagePending = true + this.status.content = "Reading clipboard image…" + try { + const image = await this.clipboardImageReader.read() + if (this.quitting || generation !== this.clipboardPasteGeneration) return + const insertion = this.draft.image(image, this.composer.plainText) + if (!insertion) { + this.status.content = `A message can include up to ${MAX_DRAFT_IMAGES} images` + return + } + this.composer.insertText(insertion.text) + this.status.content = `Pasted ${insertion.description} · review before sending` + } catch (error) { + if ( + this.quitting + || this.composer.isDestroyed + || generation !== this.clipboardPasteGeneration + ) return + const message = error instanceof Error + ? error.message + : "Clipboard image paste is unavailable" + this.status.content = message + this.transcript.notice(message, true) + } finally { + this.clipboardImagePending = false + } } private handlePaste(event: PasteEvent): void { @@ -2496,6 +2692,7 @@ export class NanobotTui { } private recordPrompt(content: string): void { + if (!content) return if (this.promptHistory.at(-1) !== content) this.promptHistory.push(content) if (this.promptHistory.length > 50) this.promptHistory.shift() this.historyCursor = this.promptHistory.length @@ -2723,10 +2920,14 @@ export class NanobotTui { } private handleDestroy = (): void => { + this.quitting = true + this.clipboardPasteGeneration += 1 if (this.shimmerTimer) clearInterval(this.shimmerTimer) this.stopSessionRefresh() + this.composerSyntax.destroy() this.transcript.destroy() this.diffViewer.destroy() + void this.clipboardImageReader.dispose().catch(() => {}) this.host.release() this.client.close() } diff --git a/tui/src/clipboard-image.test.ts b/tui/src/clipboard-image.test.ts new file mode 100644 index 000000000..2496cbab9 --- /dev/null +++ b/tui/src/clipboard-image.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, test } from "bun:test" +import type { ClipboardReadResult, HostClipboardService } from "@opentui/core" + +import { createClipboardImageReader } from "./clipboard-image" + +function clipboard(result: ClipboardReadResult) { + let disposed = false + const service = { + maxWriteBytes: 1, + read: async () => result, + writeText: async () => ({ status: "unsupported" as const }), + clear: async () => ({ status: "unsupported" as const }), + dispose: async () => { disposed = true }, + } satisfies HostClipboardService + return { service, disposed: () => disposed } +} + +describe("clipboard image reader", () => { + test("encodes supported native clipboard bytes as a data URL", async () => { + const fake = clipboard({ + status: "read", + representation: { mimeType: "image/png", bytes: Uint8Array.from([0, 1, 2, 255]) }, + }) + const reader = createClipboardImageReader(() => fake.service) + + expect(await reader.read()).toEqual({ + mimeType: "image/png", + dataUrl: "data:image/png;base64,AAEC/w==", + }) + await reader.dispose() + expect(fake.disposed()).toBeTrue() + }) + + test.each([ + ["empty", "No image in clipboard"], + ["limit-exceeded", "Clipboard image is larger than 6 MB"], + ["timed-out", "Clipboard image read timed out"], + ["unsupported", "Clipboard image paste is unavailable"], + ] as const)("reports %s without exposing native details", async (status, message) => { + const fake = clipboard({ status }) + const reader = createClipboardImageReader(() => fake.service) + + await expect(reader.read()).rejects.toThrow(message) + await reader.dispose() + }) +}) diff --git a/tui/src/clipboard-image.ts b/tui/src/clipboard-image.ts new file mode 100644 index 000000000..ca0aaf90f --- /dev/null +++ b/tui/src/clipboard-image.ts @@ -0,0 +1,64 @@ +import { + createHostClipboard, + type HostClipboardService, +} from "@opentui/core" + +const IMAGE_MIME_TYPES = [ + "image/png", + "image/jpeg", + "image/webp", + "image/gif", +] as const +const MAX_IMAGE_BYTES = 6 * 1024 * 1024 + +export interface ClipboardImage { + dataUrl: string + mimeType: typeof IMAGE_MIME_TYPES[number] +} + +export interface ClipboardImageReader { + read(): Promise + dispose(): Promise +} + +type ClipboardFactory = () => HostClipboardService + +function readFailure(status: string): Error { + if (status === "empty") return new Error("No image in clipboard") + if (status === "limit-exceeded") return new Error("Clipboard image is larger than 6 MB") + if (status === "timed-out") return new Error("Clipboard image read timed out") + return new Error("Clipboard image paste is unavailable") +} + +/** Lazily owns OpenTUI's native host clipboard so ordinary TUI startup does no clipboard work. */ +export function createClipboardImageReader( + createClipboard: ClipboardFactory = () => createHostClipboard({ maxReadBytes: MAX_IMAGE_BYTES }), +): ClipboardImageReader { + let clipboard: HostClipboardService | null = null + let disposed = false + + return { + async read(): Promise { + if (disposed) throw new Error("Clipboard image paste is unavailable") + clipboard ||= createClipboard() + const result = await clipboard.read({ preferredTypes: IMAGE_MIME_TYPES }) + if (result.status !== "read") throw readFailure(result.status) + const normalizedMime = result.representation.mimeType.toLowerCase() + const mimeType = IMAGE_MIME_TYPES.find((candidate) => candidate === normalizedMime) + if (!mimeType) throw new Error("Clipboard does not contain a supported image") + const bytes = result.representation.bytes + if (!bytes.length) throw new Error("Clipboard image is empty") + if (bytes.length > MAX_IMAGE_BYTES) throw new Error("Clipboard image is larger than 6 MB") + return { + mimeType, + dataUrl: `data:${mimeType};base64,${Buffer.from(bytes).toString("base64")}`, + } + }, + async dispose(): Promise { + if (disposed) return + disposed = true + await clipboard?.dispose() + clipboard = null + }, + } +} diff --git a/tui/src/composer-draft.test.ts b/tui/src/composer-draft.test.ts index 0b05e5abc..53de366f9 100644 --- a/tui/src/composer-draft.test.ts +++ b/tui/src/composer-draft.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test" -import { ComposerDraft } from "./composer-draft" +import { ComposerDraft, MAX_DRAFT_IMAGES } from "./composer-draft" describe("ComposerDraft", () => { test("keeps ordinary pastes editable as ordinary text", () => { @@ -25,4 +25,127 @@ describe("ComposerDraft", () => { expect(draft.expand(first.text.trim())).toBe(first.text.trim()) expect(draft.expand(second.text.trim())).toBe(content) }) + + test("keeps image bytes outside the editor and drops attachments with deleted placeholders", () => { + const draft = new ComposerDraft() + const first = draft.image({ mimeType: "image/png", dataUrl: "data:image/png;base64,AAAA" }) + const second = draft.image({ mimeType: "image/jpeg", dataUrl: "data:image/jpeg;base64,BBBB" }) + + expect(first?.text).toBe("[Image #1] ") + expect(second?.text).toBe("[Image #2] ") + const visible = `compare ${second?.text}${first?.text}` + expect(draft.expand(visible)).toBe("compare ") + expect(draft.display(visible)).toBe(visible) + expect(draft.media(visible)).toEqual([ + { data_url: "data:image/jpeg;base64,BBBB", name: "clipboard-image-2.jpg" }, + { data_url: "data:image/png;base64,AAAA", name: "clipboard-image-1.png" }, + ]) + + draft.prune(first?.text || "") + expect(draft.imageCount).toBe(1) + expect(draft.media(second?.text || "")).toEqual([]) + }) + + test("removes a partially edited image placeholder as one atomic unit", () => { + const draft = new ComposerDraft() + const image = draft.image({ mimeType: "image/png", dataUrl: "data:image/png;base64,AAAA" }) + const previous = `before ${image?.text}after` + const value = previous.replace("[Image #1]", "Image #1]") + + expect(draft.reconcileImageEdit(previous, value, 7)).toEqual({ + value: "before after", + cursor: 7, + removedImages: ["Image #1"], + }) + expect(draft.imageCount).toBe(0) + expect(draft.media(value)).toEqual([]) + }) + + test("removes an edited duplicate occurrence without leaving a placeholder fragment", () => { + const draft = new ComposerDraft() + const image = draft.image({ mimeType: "image/png", dataUrl: "data:image/png;base64,AAAA" }) + const label = image?.text.trim() || "" + const previous = `${label} ${label}` + + expect(draft.reconcileImageEdit(previous, previous.slice(1), 0)).toEqual({ + value: ` ${label}`, + cursor: 0, + removedImages: [], + }) + expect(draft.imageCount).toBe(1) + }) + + test("snaps cursor movement across complete image placeholders", () => { + const draft = new ComposerDraft() + const image = draft.image({ mimeType: "image/png", dataUrl: "data:image/png;base64,AAAA" }) + const visible = `a ${image?.text}b` + + expect(draft.snapImageCursor(visible, 3, 2)).toBe(12) + expect(draft.snapImageCursor(visible, 11, 12)).toBe(2) + expect(draft.snapImageCursor(visible, 2, 0)).toBe(2) + expect(draft.snapImageCursor(visible, 12, 13)).toBe(12) + expect(draft.moveImageCursor(visible, 2, 1)).toBe(12) + expect(draft.moveImageCursor(visible, 12, -1)).toBe(2) + }) + + test("allocates image labels around literal composer text", () => { + const draft = new ComposerDraft() + const content = "Explain [Image #1]" + const insertion = draft.image( + { mimeType: "image/png", dataUrl: "data:image/png;base64,AAAA" }, + content, + ) + + expect(insertion?.text).toBe("[Image #2] ") + expect(draft.expand(`${content} ${insertion?.text}`.trim())).toBe(`${content} `) + }) + + test("detects image labels duplicated after insertion without deleting text", () => { + const draft = new ComposerDraft() + const image = draft.image({ mimeType: "image/png", dataUrl: "data:image/png;base64,AAAA" }) + const visible = `${image?.text}Explain [Image #1]` + + expect(draft.hasImageLabelConflict(visible)).toBeTrue() + expect(draft.expand(visible)).toBe(visible) + }) + + test("detects image labels inside compacted paste text added afterward", () => { + const draft = new ComposerDraft() + const image = draft.image({ mimeType: "image/png", dataUrl: "data:image/png;base64,AAAA" }) + const content = ["Explain [Image #1]", ...Array.from({ length: 11 }, () => "detail")].join("\n") + const paste = draft.paste(content) + const visible = `${image?.text}${paste.text}` + + expect(draft.hasImageLabelConflict(visible)).toBeTrue() + expect(draft.expand(visible)).toContain("Explain [Image #1]") + }) + + test("allocates image labels around hidden compacted paste text", () => { + const draft = new ComposerDraft() + const content = ["Explain [Image #1]", ...Array.from({ length: 11 }, () => "detail")].join("\n") + const paste = draft.paste(content) + const image = draft.image( + { mimeType: "image/png", dataUrl: "data:image/png;base64,AAAA" }, + paste.text, + ) + + expect(image?.text).toBe("[Image #2] ") + expect(draft.expand(`${paste.text}${image?.text}`)).toContain("Explain [Image #1]") + }) + + test("matches the gateway image count before accepting another placeholder", () => { + const draft = new ComposerDraft() + for (let index = 0; index < MAX_DRAFT_IMAGES; index += 1) { + expect(draft.image({ + mimeType: "image/png", + dataUrl: `data:image/png;base64,${index}`, + })).not.toBeNull() + } + + expect(draft.image({ + mimeType: "image/png", + dataUrl: "data:image/png;base64,overflow", + })).toBeNull() + expect(draft.imageCount).toBe(MAX_DRAFT_IMAGES) + }) }) diff --git a/tui/src/composer-draft.ts b/tui/src/composer-draft.ts index 9af6666a5..04b1d391a 100644 --- a/tui/src/composer-draft.ts +++ b/tui/src/composer-draft.ts @@ -1,5 +1,8 @@ +import type { OutboundMedia } from "./protocol" + const LARGE_PASTE_CHARS = 1_000 const LARGE_PASTE_LINES = 10 +export const MAX_DRAFT_IMAGES = 4 export interface PasteInsertion { text: string @@ -7,9 +10,32 @@ export interface PasteInsertion { description: string } -/** Keeps large pasted text out of the editor without changing what is sent. */ +export interface DraftEditReconciliation { + value: string + cursor: number + removedImages: string[] +} + +const IMAGE_EXTENSIONS = { + "image/png": "png", + "image/jpeg": "jpg", + "image/webp": "webp", + "image/gif": "gif", +} as const + +interface DraftImage { + dataUrl: string + mimeType: keyof typeof IMAGE_EXTENSIONS +} + +/** Keeps large pasted text and image payloads out of the editable composer surface. */ export class ComposerDraft { private readonly pastes = new Map() + private readonly images = new Map() + + get imageCount(): number { + return this.images.size + } paste(value: string): PasteInsertion { const text = value.replace(/\r\n/gu, "\n").replace(/\r/gu, "\n") @@ -26,19 +52,148 @@ export class ComposerDraft { return { text: `${label} `, compacted: true, description } } - expand(visible: string): string { + private imageLabelInUse(label: string, visible: string): boolean { + if (this.images.has(label) || visible.includes(label)) return true + for (const content of this.pastes.values()) { + if (content.includes(label)) return true + } + return false + } + + private nextImageIndex(visible: string): number { + let index = 1 + while (this.imageLabelInUse(`[Image #${index}]`, visible)) index += 1 + return index + } + + image(image: DraftImage, visible = ""): PasteInsertion | null { + if (this.images.size >= MAX_DRAFT_IMAGES) return null + const index = this.nextImageIndex(visible) + const label = `[Image #${index}]` + this.images.set(label, { + data_url: image.dataUrl, + name: `clipboard-image-${index}.${IMAGE_EXTENSIONS[image.mimeType]}`, + }) + return { text: `${label} `, compacted: true, description: label.slice(1, -1) } + } + + private expandPastes(visible: string): string { let expanded = visible for (const [label, content] of this.pastes) expanded = expanded.split(label).join(content) return expanded } + private labelOccurrences(content: string, label: string): number { + return content.split(label).length - 1 + } + + imagePlaceholderRanges(visible: string): Array<{ start: number; end: number }> { + const ranges: Array<{ start: number; end: number }> = [] + for (const label of this.images.keys()) { + let start = visible.indexOf(label) + while (start >= 0) { + ranges.push({ start, end: start + label.length }) + start = visible.indexOf(label, start + label.length) + } + } + return ranges.sort((left, right) => left.start - right.start) + } + + snapImageCursor(visible: string, cursor: number, previousCursor: number): number { + const range = this.imagePlaceholderRanges(visible) + .find(({ start, end }) => cursor > start && cursor < end) + if (!range) return cursor + if (previousCursor <= range.start) return range.end + if (previousCursor >= range.end) return range.start + return cursor - range.start < range.end - cursor ? range.start : range.end + } + + moveImageCursor(visible: string, cursor: number, direction: -1 | 1): number | null { + const range = this.imagePlaceholderRanges(visible).find(({ start, end }) => ( + direction < 0 + ? cursor > start && cursor <= end + : cursor >= start && cursor < end + )) + if (!range) return null + return direction < 0 ? range.start : range.end + } + + reconcileImageEdit( + previous: string, + value: string, + cursor: number, + ): DraftEditReconciliation { + let oldStart = 0 + const sharedLength = Math.min(previous.length, value.length) + while (oldStart < sharedLength && previous[oldStart] === value[oldStart]) oldStart += 1 + + let oldEnd = previous.length + let newEnd = value.length + while ( + oldEnd > oldStart + && newEnd > oldStart + && previous[oldEnd - 1] === value[newEnd - 1] + ) { + oldEnd -= 1 + newEnd -= 1 + } + + const ranges = this.imagePlaceholderRanges(previous).filter(({ start, end }) => ( + oldStart === oldEnd + ? oldStart > start && oldStart < end + : oldStart < end && oldEnd > start + )) + if (!ranges.length) return { value, cursor, removedImages: [] } + + const replaceStart = Math.min(oldStart, ...ranges.map((range) => range.start)) + const replaceEnd = Math.max(oldEnd, ...ranges.map((range) => range.end)) + const inserted = value.slice(oldStart, newEnd) + const reconciled = previous.slice(0, replaceStart) + inserted + previous.slice(replaceEnd) + const missing = [...this.images.keys()].filter((label) => !reconciled.includes(label)) + for (const label of missing) this.images.delete(label) + return { + value: reconciled, + cursor: replaceStart + inserted.length, + removedImages: missing.map((label) => label.slice(1, -1)), + } + } + + hasImageLabelConflict(visible: string): boolean { + const expanded = this.expandPastes(visible) + return [...this.images.keys()] + .some((label) => this.labelOccurrences(expanded, label) !== 1) + } + + expand(visible: string): string { + let expanded = this.expandPastes(visible) + for (const label of this.images.keys()) { + if (this.labelOccurrences(expanded, label) === 1) expanded = expanded.replace(label, "") + } + return expanded + } + + display(visible: string): string { + return this.expandPastes(visible) + } + + media(visible: string): OutboundMedia[] { + return [...this.images] + .filter(([label]) => visible.includes(label)) + .sort(([left], [right]) => visible.indexOf(left) - visible.indexOf(right)) + .map(([, media]) => media) + } + prune(visible: string): void { for (const label of this.pastes.keys()) { if (!visible.includes(label)) this.pastes.delete(label) } + for (const label of this.images.keys()) { + if (!visible.includes(label)) this.images.delete(label) + } } clear(): void { this.pastes.clear() + this.images.clear() } } diff --git a/tui/src/prompt-queue.ts b/tui/src/prompt-queue.ts index bc5d1c05c..419f0cdfc 100644 --- a/tui/src/prompt-queue.ts +++ b/tui/src/prompt-queue.ts @@ -2,6 +2,7 @@ import type { MessageOptions } from "./protocol" export interface QueuedPrompt { content: string + displayContent?: string options: MessageOptions } diff --git a/tui/src/protocol.test.ts b/tui/src/protocol.test.ts index bb3d8412e..571ff0501 100644 --- a/tui/src/protocol.test.ts +++ b/tui/src/protocol.test.ts @@ -459,6 +459,7 @@ describe("gateway protocol", () => { }), }) client.send("hello", { + media: [{ data_url: "data:image/png;base64,AAAA", name: "clipboard-image-1.png" }], cliApps: [{ name: "github" }], sessionMentions: [{ name: "plan", session_key: "websocket:plan" }], userShell: true, @@ -477,6 +478,9 @@ describe("gateway protocol", () => { expect(outbound[1]?.chat_id).toBe("terminal") expect(outbound[1]?.content).toBe("hello") expect(outbound[1]?.user_shell).toBe(true) + expect(outbound[1]?.media).toEqual([ + { data_url: "data:image/png;base64,AAAA", name: "clipboard-image-1.png" }, + ]) expect(outbound[1]?.cli_apps).toEqual([{ name: "github" }]) expect(outbound[1]?.session_mentions).toEqual([ { name: "plan", session_key: "websocket:plan" }, @@ -875,6 +879,12 @@ describe("gateway protocol", () => { return Promise.resolve(new Response(JSON.stringify({ messages: [ { role: "user", content: "hello", turnId: "turn-1" }, + { + role: "user", + content: "", + turnId: "turn-image", + media: [{ kind: "image", url: "/api/media/sig/image", name: "shot.png" }], + }, { role: "tool", kind: "trace", @@ -883,7 +893,7 @@ describe("gateway protocol", () => { toolEvents: [{ phase: "end", call_id: "read-1", name: "read_file" }], }, { role: "assistant", kind: "reasoning", content: "private thought" }, - { role: "assistant", content: "hi", forkIndex: 1 }, + { role: "assistant", content: "hi", forkIndex: 2 }, ], page: { has_more_before: true, before_cursor: "older-1" }, }))) @@ -894,12 +904,18 @@ describe("gateway protocol", () => { expect(history).toEqual({ messages: [ { role: "user", content: "hello", turnId: "turn-1" }, + { + role: "user", + content: "", + turnId: "turn-image", + media: [{ kind: "image", url: "/api/media/sig/image", name: "shot.png" }], + }, { role: "activity", content: "read_file", toolEvents: [{ phase: "end", call_id: "read-1", name: "read_file" }], }, - { role: "assistant", content: "hi", forkIndex: 1 }, + { role: "assistant", content: "hi", forkIndex: 2 }, ], hasMoreBefore: true, beforeCursor: "older-1", diff --git a/tui/src/protocol.ts b/tui/src/protocol.ts index f895e9515..a9d867104 100644 --- a/tui/src/protocol.ts +++ b/tui/src/protocol.ts @@ -53,12 +53,17 @@ interface FileDiff { text?: string } -interface MediaAttachment { +export interface MediaAttachment { kind: "image" | "video" | "file" url: string name?: string } +export interface OutboundMedia { + data_url: string + name?: string +} + export interface WorkspaceScopePayload { project_path: string project_name?: string @@ -181,6 +186,7 @@ type OutboundEvent = turn_id: string webui: true workspace_scope?: WorkspaceScopePayload + media?: OutboundMedia[] cli_apps?: Array<{ name: string }> mcp_presets?: Array<{ name: string }> session_mentions?: SessionMention[] @@ -225,6 +231,7 @@ export interface HistoryMessage { role: "user" | "assistant" | "activity" content: string turnId?: string + media?: MediaAttachment[] toolEvents?: ToolProgressEvent[] fileEdits?: FileEditEvent[] forkIndex?: number @@ -288,6 +295,7 @@ export interface SkillCandidate { } export interface MessageOptions { + media?: OutboundMedia[] cliApps?: Array<{ name: string }> mcpPresets?: Array<{ name: string }> sessionMentions?: SessionMention[] @@ -623,18 +631,20 @@ export async function fetchHistory( (role !== "user" && role !== "assistant") || message.kind === "reasoning" || typeof content !== "string" - || !content.trim() ) { continue } + const media = Array.isArray(message.media) ? message.media.filter(isMediaAttachment) : [] if (role === "user") { + if (!content.trim() && !media.length) continue userIndex += 1 messages.push({ role: "user", content, + ...(media.length ? { media } : {}), ...(typeof message.turnId === "string" ? { turnId: message.turnId } : {}), }) - } else { + } else if (content.trim()) { messages.push({ role: "assistant", content, forkIndex: userIndex }) } } @@ -1190,6 +1200,7 @@ export class NanobotClient { webui: true, ...(this.workspaceScope ? { workspace_scope: this.workspaceScope } : {}), ...(options.userShell ? { user_shell: true } : {}), + ...(options.media?.length ? { media: options.media } : {}), ...(options.cliApps?.length ? { cli_apps: options.cliApps } : {}), ...(options.mcpPresets?.length ? { mcp_presets: options.mcpPresets } : {}), ...(options.sessionMentions?.length diff --git a/tui/src/transcript.ts b/tui/src/transcript.ts index d9e528bbd..c705c43f4 100644 --- a/tui/src/transcript.ts +++ b/tui/src/transcript.ts @@ -3,14 +3,21 @@ import { MarkdownRenderable, RGBA, ScrollBoxRenderable, + StyledText, SyntaxStyle, TextAttributes, TextRenderable, type CliRenderer, + type TextChunk, type TreeSitterClient, } from "@opentui/core" -import type { FileEditEvent, HistoryMessage, ToolProgressEvent } from "./protocol" +import type { + FileEditEvent, + HistoryMessage, + MediaAttachment, + ToolProgressEvent, +} from "./protocol" import { renderLatexAsUnicode } from "./latex" import { hideScrollbars } from "./scrollbox" import { mergeToolEvent, renderToolEvent } from "./tool-renderers" @@ -58,6 +65,54 @@ const ACTIVITY_PREVIEW_LINES = 4 // subsequent deltas to the renderer cadence. const STREAM_FLUSH_MS = 32 +export interface UserMessageMedia { + kind?: MediaAttachment["kind"] + name?: string +} + +interface UserMessageProjection { + imageLabels: string[] + attachmentNames: string[] +} + +function projectUserMessage(media: readonly UserMessageMedia[]): UserMessageProjection { + const imageNames: Array = [] + const attachmentNames: string[] = [] + for (const item of media) { + // Outbound TUI media has no explicit kind because this path currently only + // sends clipboard images. Gateway and history media carry the kind. + if (item.kind === undefined || item.kind === "image") imageNames.push(item.name) + else if (item.name) attachmentNames.push(item.name) + } + + const used = new Set() + let next = 1 + const imageLabels = imageNames.map((name) => { + const match = name?.match(/^clipboard-image-(\d+)\.[^.]+$/iu) + const preferred = match ? Number(match[1]) : 0 + let index = Number.isSafeInteger(preferred) && preferred > 0 && !used.has(preferred) + ? preferred + : next + while (used.has(index)) index += 1 + used.add(index) + while (used.has(next)) next += 1 + return `[Image #${index}]` + }) + return { imageLabels, attachmentNames } +} + +export function userMessageText( + content: string, + media: readonly UserMessageMedia[] = [], + displayContent?: string, +): string { + const { imageLabels, attachmentNames } = projectUserMessage(media) + return [ + displayContent ?? [content, imageLabels.join(" ")].filter(Boolean).join(" "), + attachmentNames.length ? `Attachments: ${attachmentNames.join(", ")}` : "", + ].filter(Boolean).join("\n") +} + /** Projects gateway events into retained, reflowable conversation cells. */ export class Transcript { readonly root: ScrollBoxRenderable @@ -71,6 +126,12 @@ export class Transcript { private readonly activities = new Set() private readonly frames = new Set() private readonly userRows = new Set() + private readonly userMessages = new Set<{ + renderable: TextRenderable + content: string + media: UserMessageMedia[] + displayContent?: string + }>() private readonly userTurnIds = new Set() private wrote = false private nextId = 0 @@ -116,6 +177,13 @@ export class Transcript { const previousSyntax = this.theme.syntax this.theme = theme for (const { renderable, tone } of this.styledText) renderable.fg = theme[tone] + for (const message of this.userMessages) { + message.renderable.content = this.userMessageContent( + message.content, + message.media, + message.displayContent, + ) + } for (const renderable of this.markdown) renderable.syntaxStyle = theme.syntax for (const frame of this.frames) frame.borderColor = theme.border for (const row of this.userRows) { @@ -171,6 +239,7 @@ export class Transcript { this.activities.clear() this.frames.clear() this.userRows.clear() + this.userMessages.clear() this.userTurnIds.clear() this.wrote = false this.nextId = 0 @@ -182,7 +251,9 @@ export class Transcript { history(messages: HistoryMessage[]): void { for (const message of messages) { - if (message.role === "user") this.user(message.content, message.turnId) + if (message.role === "user") { + this.user(message.content, message.turnId, message.media) + } else if (message.role === "assistant") this.assistant(message.content) else if (message.fileEdits?.length) this.fileEdits(message.fileEdits) else this.progress(message.content, message.toolEvents) @@ -198,7 +269,7 @@ export class Transcript { for (const message of messages) { if (message.role === "user") { if (message.turnId && this.userTurnIds.has(message.turnId)) continue - this.writeRole("›", message.content, "user", index++) + this.writeUser(message.content, message.media, index++) if (message.turnId) this.userTurnIds.add(message.turnId) } else if (message.role === "assistant") { this.writeMarkdown(message.content, false, index++) @@ -224,11 +295,16 @@ export class Transcript { return this.root.scrollTop <= 0 } - user(content: string, turnId?: string): boolean { + user( + content: string, + turnId?: string, + media: readonly UserMessageMedia[] = [], + displayContent?: string, + ): boolean { if (turnId && this.userTurnIds.has(turnId)) return false this.noteOutput() this.finishActivity() - this.writeRole("›", content, "user") + this.writeUser(content, media, undefined, displayContent) if (turnId) this.userTurnIds.add(turnId) return true } @@ -336,6 +412,7 @@ export class Transcript { this.activity = null this.frames.clear() this.userRows.clear() + this.userMessages.clear() this.theme.syntax.destroy() } @@ -468,7 +545,7 @@ export class Transcript { } private createText( - content: string, + content: string | StyledText, tone: "text" | "muted" | "error" | "user", bold = false, id = "text", @@ -487,10 +564,10 @@ export class Transcript { private writeRole( marker: string, - content: string, + content: string | StyledText, tone: "muted" | "error" | "user", index?: number, - ): void { + ): TextRenderable { const row = this.createRow(tone === "user" ? "user" : "notice", "row") if (tone === "user") { row.backgroundColor = this.theme.userBackground @@ -509,6 +586,76 @@ export class Transcript { row.add(text) this.root.add(row, index) this.wrote = true + return text + } + + private writeUser( + content: string, + media: readonly UserMessageMedia[] = [], + index?: number, + displayContent?: string, + ): void { + const retainedMedia = [...media] + const renderable = this.writeRole( + "›", + this.userMessageContent(content, retainedMedia, displayContent), + "user", + index, + ) + this.userMessages.add({ renderable, content, media: retainedMedia, displayContent }) + } + + private userMessageContent( + content: string, + media: readonly UserMessageMedia[], + displayContent?: string, + ): StyledText { + const { imageLabels, attachmentNames } = projectUserMessage(media) + const chunks: TextChunk[] = [] + const append = (text: string) => { + if (text) chunks.push({ __isChunk: true, text }) + } + const nextLine = () => { + if (chunks.length) append("\n") + } + + if (displayContent !== undefined) { + const ranges = imageLabels + .map((label) => ({ label, start: displayContent.indexOf(label) })) + .filter(({ start }) => start >= 0) + .sort((left, right) => left.start - right.start) + let cursor = 0 + for (const { label, start } of ranges) { + append(displayContent.slice(cursor, start)) + chunks.push({ + __isChunk: true, + text: label, + fg: RGBA.fromHex(this.theme.user), + attributes: TextAttributes.BOLD, + }) + cursor = start + label.length + } + append(displayContent.slice(cursor)) + } else { + append(content) + } + if (displayContent === undefined && imageLabels.length) { + if (chunks.length) append(" ") + for (const [index, label] of imageLabels.entries()) { + if (index > 0) append(" ") + chunks.push({ + __isChunk: true, + text: label, + fg: RGBA.fromHex(this.theme.user), + attributes: TextAttributes.BOLD, + }) + } + } + if (attachmentNames.length) { + nextLine() + append(`Attachments: ${attachmentNames.join(", ")}`) + } + return new StyledText(chunks) } private createMarkdown(content: string, streaming: boolean, id = "markdown"): MarkdownRenderable {