Grayscale/nanobot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(transcription): add shared voice input support (#4232)

Browse Source 
* feat(webui): add voice transcription input

* feat(webui): render ANSI output in code blocks

* refactor(webui): isolate voice recorder logic

* refactor(transcription): keep websocket ingress thin

* refactor(transcription): resolve channel audio settings on demand

* style(webui): neutralize voice waveform color

* feat(webui): add voice input tooltip

* feat(webui): add voice input keyboard shortcut

* fix(webui): distinguish voice shortcut platforms

* fix(webui): place voice button after model selector

* refactor(webui): share voice hold recording helpers

* fix(desktop): allow microphone voice input

* fix(webui): stabilize token usage month labels

* feat(webui): show voice input on settings overview

* fix(webui): label voice capability as recognition

* fix(webui): align capability overview status

* refactor(webui): isolate transcription socket handling

* fix(webui): soften silent voice waveform

* refactor(audio): clarify transcription service location

* docs(transcription): clarify audio and provider boundaries

* fix(exec): reduce session output polling flake
This commit is contained in:
Xubin Ren
2026-06-09 01:08:49 +08:00
committed by GitHub
parent 06d454a225
commit 9c81280300
49 changed files with 3071 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files
webui/src/components/settings/SettingsView.tsx
+258 -2
View File
@@ -31,6 +31,7 @@ import {
Layers,
Loader2,
LogOut,
Mic,
Moon,
PlayCircle,
Plus,
@@ -92,6 +93,7 @@ import {
updateNetworkSafetySettings,
updateProviderSettings,
updateSettings,
updateTranscriptionSettings,
updateWebSearchSettings,
} from "@/lib/api";
import { notifyCliAppsChanged } from "@/lib/cli-app-events";
@@ -115,6 +117,7 @@ import type {
ProviderModelsPayload,
SettingsPayload,
SkillSummary,
TranscriptionSettingsUpdate,
WebSearchSettingsUpdate,
WebuiDefaultAccessMode,
} from "@/lib/types";
@@ -124,6 +127,7 @@ export type SettingsSectionKey =
| "appearance"
| "models"
| "image"
| "voice"
| "browser"
| "apps"
| "skills"
@@ -367,6 +371,26 @@ const DEFAULT_IMAGE_GENERATION_FORM: ImageGenerationSettingsUpdate = {
maxImagesPerTurn: 4,
};
const DEFAULT_TRANSCRIPTION_FORM: TranscriptionSettingsUpdate = {
enabled: true,
provider: "groq",
model: "",
language: "",
maxDurationSec: 120,
maxUploadMb: 25,
};
const DEFAULT_TRANSCRIPTION_SETTINGS: NonNullable<SettingsPayload["transcription"]> = {
enabled: true,
provider: "groq",
provider_configured: false,
model: "whisper-large-v3",
language: null,
max_duration_sec: 120,
max_upload_mb: 25,
providers: [],
};
const DEFAULT_NETWORK_SAFETY_FORM: NetworkSafetySettingsUpdate = {
webuiAllowLocalServiceAccess: true,
webuiDefaultAccessMode: "default",
@@ -419,6 +443,18 @@ function imageGenerationFormFromPayload(payload: SettingsPayload): ImageGenerati
};
}
function transcriptionFormFromPayload(payload: SettingsPayload): TranscriptionSettingsUpdate {
const transcription = payload.transcription ?? DEFAULT_TRANSCRIPTION_SETTINGS;
return {
enabled: transcription.enabled,
provider: transcription.provider,
model: transcription.model,
language: transcription.language ?? "",
maxDurationSec: transcription.max_duration_sec,
maxUploadMb: transcription.max_upload_mb,
};
}
function networkSafetyFormFromPayload(payload: SettingsPayload): NetworkSafetySettingsUpdate {
return {
webuiAllowLocalServiceAccess:
@@ -479,6 +515,7 @@ export function SettingsView({
const [providerSaving, setProviderSaving] = useState<string | null>(null);
const [webSearchSaving, setWebSearchSaving] = useState(false);
const [imageGenerationSaving, setImageGenerationSaving] = useState(false);
const [transcriptionSaving, setTranscriptionSaving] = useState(false);
const [networkSafetySaving, setNetworkSafetySaving] = useState(false);
const [hostEngineApplying, setHostEngineApplying] = useState(false);
const [error, setError] = useState<string | null>(null);
@@ -511,6 +548,9 @@ export function SettingsView({
? imageGenerationFormFromPayload(initialSettings)
: DEFAULT_IMAGE_GENERATION_FORM,
);
const [transcriptionForm, setTranscriptionForm] = useState<TranscriptionSettingsUpdate>(
() => initialSettings ? transcriptionFormFromPayload(initialSettings) : DEFAULT_TRANSCRIPTION_FORM,
);
const [networkSafetyForm, setNetworkSafetyForm] = useState<NetworkSafetySettingsUpdate>(() =>
initialSettings ? networkSafetyFormFromPayload(initialSettings) : DEFAULT_NETWORK_SAFETY_FORM,
);
@@ -543,6 +583,7 @@ export function SettingsView({
setForm(agentDraftFromPayload(payload));
setWebSearchForm((prev) => webSearchFormFromPayload(payload, prev));
setImageGenerationForm(imageGenerationFormFromPayload(payload));
setTranscriptionForm(transcriptionFormFromPayload(payload));
setNetworkSafetyForm(networkSafetyFormFromPayload(payload));
if (payload.restart_required_sections) {
setPendingRestartSections(pendingRestartSectionsFromPayload(payload));
@@ -711,6 +752,19 @@ export function SettingsView({
);
}, [imageGenerationForm, settings]);
const transcriptionDirty = useMemo(() => {
if (!settings) return false;
const transcription = settings.transcription ?? DEFAULT_TRANSCRIPTION_SETTINGS;
return (
transcriptionForm.enabled !== transcription.enabled ||
transcriptionForm.provider !== transcription.provider ||
transcriptionForm.model !== transcription.model ||
transcriptionForm.language !== (transcription.language ?? "") ||
transcriptionForm.maxDurationSec !== transcription.max_duration_sec ||
transcriptionForm.maxUploadMb !== transcription.max_upload_mb
);
}, [settings, transcriptionForm]);
const networkSafetyDirty = useMemo(() => {
if (!settings) return false;
const currentLocalServiceAccess =
@@ -913,6 +967,24 @@ export function SettingsView({
}
};
const saveTranscriptionSettings = async () => {
if (!settings || !transcriptionDirty || transcriptionSaving) return;
setTranscriptionSaving(true);
try {
const payload = await updateTranscriptionSettings(token, transcriptionForm);
applyPayload(payload);
if (payload.requires_restart) {
setPendingRestartSections((prev) => ({ ...prev, browser: true }));
}
await maybeRestartHostEngine(payload);
setError(null);
} catch (err) {
setError((err as Error).message);
} finally {
setTranscriptionSaving(false);
}
};
const saveNetworkSafetySettings = async () => {
if (!settings || !networkSafetyDirty || networkSafetySaving) return;
setNetworkSafetySaving(true);
@@ -1333,6 +1405,22 @@ export function SettingsView({
requiresRestartPending={pendingRestartSections.image}
/>
);
case "voice":
return (
<TranscriptionSettings
settings={settings}
form={transcriptionForm}
dirty={transcriptionDirty}
saving={transcriptionSaving}
onChangeForm={setTranscriptionForm}
onSave={saveTranscriptionSettings}
onOpenProviders={() => selectSection("models")}
showBrandLogos={localPrefs.brandLogos}
onRestart={restartViaSettingsSurface}
isRestarting={isRestarting || hostEngineApplying}
requiresRestartPending={pendingRestartSections.browser}
/>
);
case "browser":
return (
<WebSettings
@@ -1523,6 +1611,7 @@ const SETTINGS_NAV_ITEMS: Array<{ key: SettingsSectionKey; icon: LucideIcon; fal
{ key: "appearance", icon: Palette, fallback: "Appearance" },
{ key: "models", icon: SlidersHorizontal, fallback: "Models" },
{ key: "image", icon: ImageIcon, fallback: "Image" },
{ key: "voice", icon: Mic, fallback: "Voice" },
{ key: "browser", icon: Globe2, fallback: "Web" },
{ key: "runtime", icon: Server, fallback: "System" },
{ key: "advanced", icon: ShieldCheck, fallback: "Security" },
@@ -1642,6 +1731,24 @@ function OverviewSettings({
const webStatus = settings.web.enable
? tx("settings.values.enabled", "Enabled")
: tx("settings.values.disabled", "Disabled");
const webSearchProvider =
settings.web_search.providers.find((provider) => provider.name === settings.web_search.provider) ??
settings.web_search.providers[0];
const webSearchProviderLabel = providerDisplayLabel(
settings.web_search.providers,
settings.web_search.provider,
);
const webSearchCredentialStatus =
webSearchProvider?.credential === "none"
? tx("settings.byok.webSearch.noCredentialRequired", "No key required")
: webSearchProvider?.credential === "base_url"
? settings.web_search.base_url
? tx("settings.values.configured", "Configured")
: tx("settings.values.notConfigured", "Not configured")
: settings.web_search.api_key_hint
? tx("settings.values.configured", "Configured")
: tx("settings.values.notConfigured", "Not configured");
const webCaption = `${webSearchProviderLabel} · ${webSearchCredentialStatus}`;
const imageStatus = settings.image_generation.enabled
? tx("settings.values.enabled", "Enabled")
: tx("settings.values.disabled", "Disabled");
@@ -1650,6 +1757,15 @@ function OverviewSettings({
? tx("settings.values.configured", "Configured")
: tx("settings.values.notConfigured", "Not configured")
}`;
const transcription = settings.transcription ?? DEFAULT_TRANSCRIPTION_SETTINGS;
const voiceStatus = transcription.enabled
? tx("settings.values.enabled", "Enabled")
: tx("settings.values.disabled", "Disabled");
const voiceCaption = `${providerDisplayLabel(transcription.providers, transcription.provider)} · ${
transcription.provider_configured
? tx("settings.values.configured", "Configured")
: tx("settings.values.notConfigured", "Not configured")
}`;
const isNativeHost = (settings.surface ?? settings.runtime_surface) === "native";
const workspaceCaption = shortWorkspacePath(settings.runtime.workspace_path);
const runtimeTitle = isNativeHost
@@ -1691,8 +1807,8 @@ function OverviewSettings({
icon={Globe2}
valueLogoProvider={settings.web_search.provider}
title={tx("settings.overview.webSearch", "Web search")}
value={providerDisplayLabel(settings.web_search.providers, settings.web_search.provider)}
caption={webStatus}
value={webStatus}
caption={webCaption}
showBrandLogos={showBrandLogos}
onClick={() => onSelectSection("browser")}
/>
@@ -1705,6 +1821,15 @@ function OverviewSettings({
showBrandLogos={showBrandLogos}
onClick={() => onSelectSection("image")}
/>
<OverviewListRow
icon={Mic}
valueLogoProvider={transcription.provider}
title={tx("settings.overview.voiceInput", "Voice input")}
value={voiceStatus}
caption={voiceCaption}
showBrandLogos={showBrandLogos}
onClick={() => onSelectSection("voice")}
/>
</SettingsGroup>
</section>
@@ -2654,6 +2779,137 @@ function ImageGenerationSettings({
);
}
function TranscriptionSettings({
settings,
form,
dirty,
saving,
onChangeForm,
onSave,
onOpenProviders,
showBrandLogos,
onRestart,
isRestarting,
requiresRestartPending,
}: {
settings: SettingsPayload;
form: TranscriptionSettingsUpdate;
dirty: boolean;
saving: boolean;
onChangeForm: Dispatch<SetStateAction<TranscriptionSettingsUpdate>>;
onSave: () => void;
onOpenProviders: () => void;
showBrandLogos: boolean;
onRestart?: () => void;
isRestarting?: boolean;
requiresRestartPending: boolean;
}) {
const { t } = useTranslation();
const tx = (key: string, fallback: string) => t(key, { defaultValue: fallback });
const transcription = settings.transcription ?? DEFAULT_TRANSCRIPTION_SETTINGS;
const selectedProvider =
transcription.providers.find((provider) => provider.name === form.provider) ??
transcription.providers[0];
const providerConfigured = !!selectedProvider?.configured;
return (
<section>
<SettingsSectionTitle>{tx("settings.sections.voiceInput", "Voice input")}</SettingsSectionTitle>
<SettingsGroup>
<SettingsRow
title={tx("settings.rows.transcription", "Transcription")}
description={tx("settings.help.transcription", "Transcribe microphone input before sending it. Chat channel voice messages use the same settings.")}
>
<ToggleButton
checked={form.enabled}
onChange={(enabled) => onChangeForm((prev) => ({ ...prev, enabled }))}
ariaLabel={tx("settings.rows.transcription", "Transcription")}
label={form.enabled ? tx("settings.values.on", "On") : tx("settings.values.off", "Off")}
/>
</SettingsRow>
<SettingsRow
title={tx("settings.rows.transcriptionProvider", "Provider")}
description={tx("settings.help.transcriptionProvider", "Uses the matching provider credentials from Providers.")}
>
<ProviderPicker
providers={transcription.providers}
value={form.provider}
emptyLabel={tx("settings.voice.selectProvider", "Select provider")}
showProviderLogos={showBrandLogos}
onChange={(provider) => onChangeForm((prev) => ({ ...prev, provider }))}
/>
</SettingsRow>
<SettingsRow
title={tx("settings.rows.transcriptionProviderStatus", "Provider status")}
description={tx("settings.help.transcriptionProviderStatus", "API keys stay under providers, not in transcription settings.")}
>
<div className="flex flex-wrap items-center justify-end gap-2">
<StatusPill tone={providerConfigured ? "success" : "neutral"}>
{providerConfigured
? tx("settings.values.configured", "Configured")
: tx("settings.values.notConfigured", "Not configured")}
</StatusPill>
{!providerConfigured ? (
<Button size="sm" variant="outline" onClick={onOpenProviders} className="rounded-full">
{tx("settings.voice.configureProvider", "Configure provider")}
</Button>
) : null}
</div>
</SettingsRow>
<SettingsRow
title={tx("settings.rows.transcriptionModel", "Model")}
description={tx("settings.help.transcriptionModel", "Leave as the resolved default unless your provider needs a custom model id.")}
>
<Input
value={form.model}
onChange={(event) => onChangeForm((prev) => ({ ...prev, model: event.target.value }))}
className="h-8 w-[min(300px,70vw)] rounded-full text-[13px]"
/>
</SettingsRow>
<SettingsRow
title={tx("settings.rows.transcriptionLanguage", "Language")}
description={tx("settings.help.transcriptionLanguage", "Optional ISO-639 hint such as en, zh, ja, or ko.")}
>
<Input
value={form.language}
onChange={(event) => onChangeForm((prev) => ({ ...prev, language: event.target.value }))}
placeholder={tx("settings.voice.languageAuto", "Auto")}
className="h-8 w-[min(180px,60vw)] rounded-full text-[13px]"
/>
</SettingsRow>
<SettingsRow title={tx("settings.rows.voiceLimits", "Limits")}>
<div className="flex flex-wrap justify-end gap-2">
<NumberInput
value={form.maxDurationSec}
min={1}
max={600}
suffix="s"
onChange={(maxDurationSec) => onChangeForm((prev) => ({ ...prev, maxDurationSec }))}
/>
<NumberInput
value={form.maxUploadMb}
min={1}
max={100}
suffix="MB"
onChange={(maxUploadMb) => onChangeForm((prev) => ({ ...prev, maxUploadMb }))}
/>
</div>
</SettingsRow>
<RestartSettingsFooter
dirty={dirty}
saving={saving}
pendingRestart={requiresRestartPending}
dirtyMessage={tx("settings.status.restartAfterSaving", "Save changes, then restart when ready.")}
pendingMessage={tx("settings.status.savedRestartApply", "Saved. Restart when ready.")}
onSave={onSave}
onRestart={onRestart}
isRestarting={isRestarting}
/>
</SettingsGroup>
</section>
);
}
function WebSettings({
settings,
form,
webui/src/components/settings/TokenUsageHeatmap.tsx
+4 -11
View File
@@ -78,16 +78,13 @@ function buildTokenUsageCalendar(
const today = utcDateFromIsoDay(isoDayInTimeZone(new Date(), timeZone));
const end = addUtcDays(today, 6 - today.getUTCDay());
const start = addUtcDays(end, -(TOKEN_HEATMAP_CELLS - 1));
const seenMonths = new Set<string>();
const monthLabels: TokenUsageMonthLabel[] = [];
const cells = Array.from({ length: TOKEN_HEATMAP_CELLS }, (_, index) => {
const date = addUtcDays(start, index);
const key = isoDay(date);
const row = byDate.get(key);
const monthKey = key.slice(0, 7);
if (!seenMonths.has(monthKey)) {
seenMonths.add(monthKey);
if (date.getUTCDate() === 1) {
monthLabels.push({
label: monthFormatter.format(date),
column: Math.floor(index / 7) + 1,
@@ -186,16 +183,12 @@ export function TokenUsageHeatmap({
{tx("settings.usage.shortTitle", "Token Usage")}
</span>
</div>
<div
className="mb-2 grid min-h-4 gap-1.5 text-[10px] font-normal leading-4 text-muted-foreground/62"
style={{ gridTemplateColumns: `repeat(${TOKEN_HEATMAP_COLUMNS}, minmax(0, 1fr))` }}
aria-hidden
>
<div className="relative mb-2 h-4 text-[10px] font-normal leading-4 text-muted-foreground/62" aria-hidden>
{monthLabels.map((month) => (
<span
key={`${month.label}-${month.column}`}
className="whitespace-nowrap"
style={{ gridColumnStart: month.column, gridColumnEnd: "span 4" }}
className="absolute top-0 whitespace-nowrap"
style={{ left: `${((month.column - 1) / TOKEN_HEATMAP_COLUMNS) * 100}%` }}
>
{month.label}
</span>