Grayscale/nanobot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(tui): support pasting clipboard images (#5563)

Browse Source 
* feat(tui): support pasting clipboard images

* fix(tui): keep image placeholders atomic

* fix(tui): reconcile duplicate image placeholders

* fix(tui): retain highlighted image placeholders

* fix(tui): preserve image placeholder layout

* fix(tui): keep image display state local

* fix(tui): reject images in commands
This commit is contained in:
chengyongru
2026-08-27 20:37:43 +08:00
committed by GitHub
parent b9e7c7f6fe
commit 4d204ba077
11 changed files with 1132 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tui/src/transcript.ts
+155 -8
View File
@@ -3,14 +3,21 @@ import {
MarkdownRenderable,
RGBA,
ScrollBoxRenderable,
StyledText,
SyntaxStyle,
TextAttributes,
TextRenderable,
type CliRenderer,
type TextChunk,
type TreeSitterClient,
} from "@opentui/core"
import type { FileEditEvent, HistoryMessage, ToolProgressEvent } from "./protocol"
import type {
FileEditEvent,
HistoryMessage,
MediaAttachment,
ToolProgressEvent,
} from "./protocol"
import { renderLatexAsUnicode } from "./latex"
import { hideScrollbars } from "./scrollbox"
import { mergeToolEvent, renderToolEvent } from "./tool-renderers"
@@ -58,6 +65,54 @@ const ACTIVITY_PREVIEW_LINES = 4
// subsequent deltas to the renderer cadence.
const STREAM_FLUSH_MS = 32
export interface UserMessageMedia {
kind?: MediaAttachment["kind"]
name?: string
}
interface UserMessageProjection {
imageLabels: string[]
attachmentNames: string[]
}
function projectUserMessage(media: readonly UserMessageMedia[]): UserMessageProjection {
const imageNames: Array<string | undefined> = []
const attachmentNames: string[] = []
for (const item of media) {
// Outbound TUI media has no explicit kind because this path currently only
// sends clipboard images. Gateway and history media carry the kind.
if (item.kind === undefined || item.kind === "image") imageNames.push(item.name)
else if (item.name) attachmentNames.push(item.name)
}
const used = new Set<number>()
let next = 1
const imageLabels = imageNames.map((name) => {
const match = name?.match(/^clipboard-image-(\d+)\.[^.]+$/iu)
const preferred = match ? Number(match[1]) : 0
let index = Number.isSafeInteger(preferred) && preferred > 0 && !used.has(preferred)
? preferred
: next
while (used.has(index)) index += 1
used.add(index)
while (used.has(next)) next += 1
return `[Image #${index}]`
})
return { imageLabels, attachmentNames }
}
export function userMessageText(
content: string,
media: readonly UserMessageMedia[] = [],
displayContent?: string,
): string {
const { imageLabels, attachmentNames } = projectUserMessage(media)
return [
displayContent ?? [content, imageLabels.join(" ")].filter(Boolean).join(" "),
attachmentNames.length ? `Attachments: ${attachmentNames.join(", ")}` : "",
].filter(Boolean).join("\n")
}
/** Projects gateway events into retained, reflowable conversation cells. */
export class Transcript {
readonly root: ScrollBoxRenderable
@@ -71,6 +126,12 @@ export class Transcript {
private readonly activities = new Set<Activity>()
private readonly frames = new Set<BoxRenderable>()
private readonly userRows = new Set<BoxRenderable>()
private readonly userMessages = new Set<{
renderable: TextRenderable
content: string
media: UserMessageMedia[]
displayContent?: string
}>()
private readonly userTurnIds = new Set<string>()
private wrote = false
private nextId = 0
@@ -116,6 +177,13 @@ export class Transcript {
const previousSyntax = this.theme.syntax
this.theme = theme
for (const { renderable, tone } of this.styledText) renderable.fg = theme[tone]
for (const message of this.userMessages) {
message.renderable.content = this.userMessageContent(
message.content,
message.media,
message.displayContent,
)
}
for (const renderable of this.markdown) renderable.syntaxStyle = theme.syntax
for (const frame of this.frames) frame.borderColor = theme.border
for (const row of this.userRows) {
@@ -171,6 +239,7 @@ export class Transcript {
this.activities.clear()
this.frames.clear()
this.userRows.clear()
this.userMessages.clear()
this.userTurnIds.clear()
this.wrote = false
this.nextId = 0
@@ -182,7 +251,9 @@ export class Transcript {
history(messages: HistoryMessage[]): void {
for (const message of messages) {
if (message.role === "user") this.user(message.content, message.turnId)
if (message.role === "user") {
this.user(message.content, message.turnId, message.media)
}
else if (message.role === "assistant") this.assistant(message.content)
else if (message.fileEdits?.length) this.fileEdits(message.fileEdits)
else this.progress(message.content, message.toolEvents)
@@ -198,7 +269,7 @@ export class Transcript {
for (const message of messages) {
if (message.role === "user") {
if (message.turnId && this.userTurnIds.has(message.turnId)) continue
this.writeRole("›", message.content, "user", index++)
this.writeUser(message.content, message.media, index++)
if (message.turnId) this.userTurnIds.add(message.turnId)
} else if (message.role === "assistant") {
this.writeMarkdown(message.content, false, index++)
@@ -224,11 +295,16 @@ export class Transcript {
return this.root.scrollTop <= 0
}
user(content: string, turnId?: string): boolean {
user(
content: string,
turnId?: string,
media: readonly UserMessageMedia[] = [],
displayContent?: string,
): boolean {
if (turnId && this.userTurnIds.has(turnId)) return false
this.noteOutput()
this.finishActivity()
this.writeRole("›", content, "user")
this.writeUser(content, media, undefined, displayContent)
if (turnId) this.userTurnIds.add(turnId)
return true
}
@@ -336,6 +412,7 @@ export class Transcript {
this.activity = null
this.frames.clear()
this.userRows.clear()
this.userMessages.clear()
this.theme.syntax.destroy()
}
@@ -468,7 +545,7 @@ export class Transcript {
}
private createText(
content: string,
content: string | StyledText,
tone: "text" | "muted" | "error" | "user",
bold = false,
id = "text",
@@ -487,10 +564,10 @@ export class Transcript {
private writeRole(
marker: string,
content: string,
content: string | StyledText,
tone: "muted" | "error" | "user",
index?: number,
): void {
): TextRenderable {
const row = this.createRow(tone === "user" ? "user" : "notice", "row")
if (tone === "user") {
row.backgroundColor = this.theme.userBackground
@@ -509,6 +586,76 @@ export class Transcript {
row.add(text)
this.root.add(row, index)
this.wrote = true
return text
}
private writeUser(
content: string,
media: readonly UserMessageMedia[] = [],
index?: number,
displayContent?: string,
): void {
const retainedMedia = [...media]
const renderable = this.writeRole(
"›",
this.userMessageContent(content, retainedMedia, displayContent),
"user",
index,
)
this.userMessages.add({ renderable, content, media: retainedMedia, displayContent })
}
private userMessageContent(
content: string,
media: readonly UserMessageMedia[],
displayContent?: string,
): StyledText {
const { imageLabels, attachmentNames } = projectUserMessage(media)
const chunks: TextChunk[] = []
const append = (text: string) => {
if (text) chunks.push({ __isChunk: true, text })
}
const nextLine = () => {
if (chunks.length) append("\n")
}
if (displayContent !== undefined) {
const ranges = imageLabels
.map((label) => ({ label, start: displayContent.indexOf(label) }))
.filter(({ start }) => start >= 0)
.sort((left, right) => left.start - right.start)
let cursor = 0
for (const { label, start } of ranges) {
append(displayContent.slice(cursor, start))
chunks.push({
__isChunk: true,
text: label,
fg: RGBA.fromHex(this.theme.user),
attributes: TextAttributes.BOLD,
})
cursor = start + label.length
}
append(displayContent.slice(cursor))
} else {
append(content)
}
if (displayContent === undefined && imageLabels.length) {
if (chunks.length) append(" ")
for (const [index, label] of imageLabels.entries()) {
if (index > 0) append(" ")
chunks.push({
__isChunk: true,
text: label,
fg: RGBA.fromHex(this.theme.user),
attributes: TextAttributes.BOLD,
})
}
}
if (attachmentNames.length) {
nextLine()
append(`Attachments: ${attachmentNames.join(", ")}`)
}
return new StyledText(chunks)
}
private createMarkdown(content: string, streaming: boolean, id = "markdown"): MarkdownRenderable {