feat(transcription): add shared voice input support (#4232)
* feat(webui): add voice transcription input * feat(webui): render ANSI output in code blocks * refactor(webui): isolate voice recorder logic * refactor(transcription): keep websocket ingress thin * refactor(transcription): resolve channel audio settings on demand * style(webui): neutralize voice waveform color * feat(webui): add voice input tooltip * feat(webui): add voice input keyboard shortcut * fix(webui): distinguish voice shortcut platforms * fix(webui): place voice button after model selector * refactor(webui): share voice hold recording helpers * fix(desktop): allow microphone voice input * fix(webui): stabilize token usage month labels * feat(webui): show voice input on settings overview * fix(webui): label voice capability as recognition * fix(webui): align capability overview status * refactor(webui): isolate transcription socket handling * fix(webui): soften silent voice waveform * refactor(audio): clarify transcription service location * docs(transcription): clarify audio and provider boundaries * fix(exec): reduce session output polling flake
This commit is contained in:
@@ -24,6 +24,7 @@ DEFAULT_WAIT_FOR_MS = 10_000
|
||||
MAX_WAIT_FOR_MS = 120_000
|
||||
DEFAULT_MAX_OUTPUT_CHARS = 10_000
|
||||
MAX_OUTPUT_CHARS = 50_000
|
||||
OUTPUT_DRAIN_GRACE_S = 0.1
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
@@ -139,6 +140,8 @@ class _ExecSession:
|
||||
asyncio.gather(self._stdout_task, self._stderr_task),
|
||||
timeout=2.0,
|
||||
)
|
||||
elif yield_time_ms > 0:
|
||||
await self._wait_for_buffered_output()
|
||||
|
||||
async with self._lock:
|
||||
output = "".join(self._chunks)
|
||||
@@ -163,6 +166,14 @@ class _ExecSession:
|
||||
with suppress(asyncio.TimeoutError):
|
||||
await asyncio.wait_for(self.process.wait(), timeout=5.0)
|
||||
|
||||
async def _wait_for_buffered_output(self) -> None:
|
||||
deadline = time.monotonic() + OUTPUT_DRAIN_GRACE_S
|
||||
while time.monotonic() < deadline:
|
||||
async with self._lock:
|
||||
if self._chunks:
|
||||
return
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
|
||||
class ExecSessionManager:
|
||||
def __init__(self, *, max_sessions: int = 8, idle_timeout: int = 1800) -> None:
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
"""Shared audio service helpers."""
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Application-level audio transcription service.
|
||||
|
||||
This module owns nanobot's transcription behavior: config resolution,
|
||||
legacy channel fallback, upload validation, temporary-file handling, and
|
||||
dispatch to provider adapters. It deliberately does not know provider-specific
|
||||
HTTP details; those live in ``nanobot.providers.transcription``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from nanobot.config.paths import get_media_dir
|
||||
from nanobot.utils.media_decode import FileSizeExceeded, save_base64_data_url
|
||||
|
||||
TranscriptionProviderName = Literal["groq", "openai"]
|
||||
|
||||
_DEFAULT_PROVIDER: TranscriptionProviderName = "groq"
|
||||
_DEFAULT_MODELS: dict[TranscriptionProviderName, str] = {
|
||||
"groq": "whisper-large-v3",
|
||||
"openai": "whisper-1",
|
||||
}
|
||||
_MAX_AUDIO_BYTES_FALLBACK = 25 * 1024 * 1024
|
||||
_AUDIO_MIME_ALLOWED: frozenset[str] = frozenset({
|
||||
"audio/aac",
|
||||
"audio/flac",
|
||||
"audio/m4a",
|
||||
"audio/mp4",
|
||||
"audio/mpeg",
|
||||
"audio/ogg",
|
||||
"audio/wav",
|
||||
"audio/webm",
|
||||
"audio/x-m4a",
|
||||
"audio/x-wav",
|
||||
})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EffectiveTranscriptionConfig:
|
||||
enabled: bool
|
||||
provider: TranscriptionProviderName
|
||||
model: str
|
||||
language: str | None
|
||||
api_key: str = field(repr=False)
|
||||
api_base: str
|
||||
max_duration_sec: int
|
||||
max_upload_mb: int
|
||||
|
||||
@property
|
||||
def configured(self) -> bool:
|
||||
return bool(self.api_key)
|
||||
|
||||
|
||||
class TranscriptionIngressError(Exception):
|
||||
"""Stable transcription upload error surfaced to WebUI clients."""
|
||||
|
||||
def __init__(self, detail: str, **extra: Any):
|
||||
super().__init__(detail)
|
||||
self.detail = detail
|
||||
self.extra = extra
|
||||
|
||||
|
||||
def _as_provider(value: Any) -> TranscriptionProviderName | None:
|
||||
if isinstance(value, str):
|
||||
name = value.strip().lower()
|
||||
if name in _DEFAULT_MODELS:
|
||||
return name # type: ignore[return-value]
|
||||
return None
|
||||
|
||||
|
||||
def _provider_config(config: Any, provider: str) -> Any:
|
||||
return getattr(getattr(config, "providers", None), provider, None)
|
||||
|
||||
|
||||
def _extract_data_url_mime(url: str) -> str | None:
|
||||
header, _, _ = url.partition(",")
|
||||
if not header.startswith("data:") or ";base64" not in header:
|
||||
return None
|
||||
return header[5:].split(";", 1)[0].strip().lower() or None
|
||||
|
||||
|
||||
def resolve_transcription_config(config: Any) -> EffectiveTranscriptionConfig:
|
||||
"""Resolve top-level transcription settings with legacy channel fallback."""
|
||||
top = getattr(config, "transcription", None)
|
||||
channels = getattr(config, "channels", None)
|
||||
provider = (
|
||||
_as_provider(getattr(top, "provider", None))
|
||||
or _as_provider(getattr(channels, "transcription_provider", None))
|
||||
or _DEFAULT_PROVIDER
|
||||
)
|
||||
provider_cfg = _provider_config(config, provider)
|
||||
return EffectiveTranscriptionConfig(
|
||||
enabled=bool(getattr(top, "enabled", True)),
|
||||
provider=provider,
|
||||
model=(getattr(top, "model", None) or _DEFAULT_MODELS[provider]).strip(),
|
||||
language=getattr(top, "language", None) or getattr(channels, "transcription_language", None),
|
||||
api_key=getattr(provider_cfg, "api_key", None) or "",
|
||||
api_base=getattr(provider_cfg, "api_base", None) or "",
|
||||
max_duration_sec=int(getattr(top, "max_duration_sec", 120)),
|
||||
max_upload_mb=int(getattr(top, "max_upload_mb", 25)),
|
||||
)
|
||||
|
||||
|
||||
async def transcribe_audio_data_url(
|
||||
data_url: Any,
|
||||
config: EffectiveTranscriptionConfig,
|
||||
*,
|
||||
duration_ms: Any = None,
|
||||
) -> str:
|
||||
"""Validate, persist, transcribe, and remove a WebUI audio data URL."""
|
||||
if not isinstance(data_url, str) or not data_url:
|
||||
raise TranscriptionIngressError("missing_audio")
|
||||
if not config.enabled:
|
||||
raise TranscriptionIngressError("disabled")
|
||||
if not config.configured:
|
||||
raise TranscriptionIngressError("not_configured", provider=config.provider)
|
||||
if (
|
||||
isinstance(duration_ms, (int, float))
|
||||
and duration_ms > (config.max_duration_sec * 1000 + 1000)
|
||||
):
|
||||
raise TranscriptionIngressError("duration")
|
||||
if _extract_data_url_mime(data_url) not in _AUDIO_MIME_ALLOWED:
|
||||
raise TranscriptionIngressError("mime")
|
||||
|
||||
audio_path: str | None = None
|
||||
max_bytes = max(
|
||||
1,
|
||||
config.max_upload_mb * 1024 * 1024 if config.max_upload_mb else _MAX_AUDIO_BYTES_FALLBACK,
|
||||
)
|
||||
try:
|
||||
audio_path = save_base64_data_url(
|
||||
data_url,
|
||||
get_media_dir("webui-transcription"),
|
||||
max_bytes=max_bytes,
|
||||
)
|
||||
except FileSizeExceeded as exc:
|
||||
raise TranscriptionIngressError("size") from exc
|
||||
except Exception as exc:
|
||||
logger.warning("transcription audio decode failed: {}", exc)
|
||||
if not audio_path:
|
||||
raise TranscriptionIngressError("decode")
|
||||
|
||||
try:
|
||||
text = await transcribe_audio_file(audio_path, config)
|
||||
finally:
|
||||
with suppress(OSError):
|
||||
Path(audio_path).unlink(missing_ok=True)
|
||||
if not text:
|
||||
raise TranscriptionIngressError("empty")
|
||||
return text
|
||||
|
||||
|
||||
async def transcribe_audio_file(
|
||||
file_path: str | Path,
|
||||
config: EffectiveTranscriptionConfig,
|
||||
) -> str:
|
||||
"""Transcribe *file_path* using the already-resolved transcription config."""
|
||||
if not config.enabled or not config.configured:
|
||||
return ""
|
||||
if config.provider == "openai":
|
||||
from nanobot.providers.transcription import OpenAITranscriptionProvider
|
||||
|
||||
provider = OpenAITranscriptionProvider(
|
||||
api_key=config.api_key,
|
||||
api_base=config.api_base or None,
|
||||
language=config.language,
|
||||
model=config.model,
|
||||
)
|
||||
else:
|
||||
from nanobot.providers.transcription import GroqTranscriptionProvider
|
||||
|
||||
provider = GroqTranscriptionProvider(
|
||||
api_key=config.api_key,
|
||||
api_base=config.api_base or None,
|
||||
language=config.language,
|
||||
model=config.model,
|
||||
)
|
||||
return await provider.transcribe(file_path)
|
||||
@@ -28,10 +28,6 @@ class BaseChannel(ABC):
|
||||
|
||||
name: str = "base"
|
||||
display_name: str = "Base"
|
||||
transcription_provider: str = "groq"
|
||||
transcription_api_key: str = ""
|
||||
transcription_api_base: str = ""
|
||||
transcription_language: str | None = None
|
||||
send_progress: bool = True
|
||||
send_tool_hints: bool = False
|
||||
show_reasoning: bool = True
|
||||
@@ -51,24 +47,14 @@ class BaseChannel(ABC):
|
||||
|
||||
async def transcribe_audio(self, file_path: str | Path) -> str:
|
||||
"""Transcribe an audio file via Whisper (OpenAI or Groq). Returns empty string on failure."""
|
||||
if not self.transcription_api_key:
|
||||
return ""
|
||||
try:
|
||||
if self.transcription_provider == "openai":
|
||||
from nanobot.providers.transcription import OpenAITranscriptionProvider
|
||||
provider = OpenAITranscriptionProvider(
|
||||
api_key=self.transcription_api_key,
|
||||
api_base=self.transcription_api_base or None,
|
||||
language=self.transcription_language or None,
|
||||
)
|
||||
else:
|
||||
from nanobot.providers.transcription import GroqTranscriptionProvider
|
||||
provider = GroqTranscriptionProvider(
|
||||
api_key=self.transcription_api_key,
|
||||
api_base=self.transcription_api_base or None,
|
||||
language=self.transcription_language or None,
|
||||
)
|
||||
return await provider.transcribe(file_path)
|
||||
from nanobot.audio.transcription import (
|
||||
resolve_transcription_config,
|
||||
transcribe_audio_file,
|
||||
)
|
||||
from nanobot.config.loader import load_config
|
||||
|
||||
return await transcribe_audio_file(file_path, resolve_transcription_config(load_config()))
|
||||
except Exception:
|
||||
self.logger.exception("Audio transcription failed")
|
||||
return ""
|
||||
|
||||
@@ -80,11 +80,6 @@ class ChannelManager:
|
||||
"""Initialize channels discovered via pkgutil scan + entry_points plugins."""
|
||||
from nanobot.channels.registry import discover_channel_names, discover_enabled
|
||||
|
||||
transcription_provider = self.config.channels.transcription_provider
|
||||
transcription_key = self._resolve_transcription_key(transcription_provider)
|
||||
transcription_base = self._resolve_transcription_base(transcription_provider)
|
||||
transcription_language = self.config.channels.transcription_language
|
||||
|
||||
# Collect enabled module names first, then only import those.
|
||||
# Channel configs live in ChannelsConfig's extra fields (via
|
||||
# extra="allow"), so we enumerate candidates from pkgutil scan
|
||||
@@ -135,10 +130,6 @@ class ChannelManager:
|
||||
)
|
||||
kwargs["gateway"] = gateway
|
||||
channel = cls(section, self.bus, **kwargs)
|
||||
channel.transcription_provider = transcription_provider
|
||||
channel.transcription_api_key = transcription_key
|
||||
channel.transcription_api_base = transcription_base
|
||||
channel.transcription_language = transcription_language
|
||||
channel.send_progress = self._resolve_bool_override(
|
||||
section, "send_progress", self.config.channels.send_progress,
|
||||
)
|
||||
@@ -155,24 +146,6 @@ class ChannelManager:
|
||||
|
||||
self._validate_allow_from()
|
||||
|
||||
def _resolve_transcription_key(self, provider: str) -> str:
|
||||
"""Pick the API key for the configured transcription provider."""
|
||||
try:
|
||||
if provider == "openai":
|
||||
return self.config.providers.openai.api_key
|
||||
return self.config.providers.groq.api_key
|
||||
except AttributeError:
|
||||
return ""
|
||||
|
||||
def _resolve_transcription_base(self, provider: str) -> str:
|
||||
"""Pick the API base URL for the configured transcription provider."""
|
||||
try:
|
||||
if provider == "openai":
|
||||
return self.config.providers.openai.api_base or ""
|
||||
return self.config.providers.groq.api_base or ""
|
||||
except AttributeError:
|
||||
return ""
|
||||
|
||||
def _validate_allow_from(self) -> None:
|
||||
for name, ch in self.channels.items():
|
||||
cfg = ch.config
|
||||
|
||||
@@ -45,6 +45,7 @@ from nanobot.webui.http_utils import (
|
||||
query_first as _query_first,
|
||||
)
|
||||
from nanobot.webui.mcp_presets_api import normalize_mcp_preset_mentions
|
||||
from nanobot.webui.transcription_ws import webui_transcription_event
|
||||
from nanobot.webui.websocket_logging import websockets_server_logger
|
||||
|
||||
|
||||
@@ -235,7 +236,7 @@ _VIDEO_MIME_ALLOWED: frozenset[str] = frozenset({
|
||||
|
||||
_UPLOAD_MIME_ALLOWED: frozenset[str] = _IMAGE_MIME_ALLOWED | _VIDEO_MIME_ALLOWED
|
||||
|
||||
_DATA_URL_MIME_RE = re.compile(r"^data:([^;]+);base64,", re.DOTALL)
|
||||
_DATA_URL_MIME_RE = re.compile(r"^data:([^;,]+)(?:;[^,]*)*;base64,", re.DOTALL)
|
||||
|
||||
|
||||
def _extract_data_url_mime(url: str) -> str | None:
|
||||
@@ -419,7 +420,6 @@ class WebSocketChannel(BaseChannel):
|
||||
return None
|
||||
|
||||
# -- Server lifecycle and connection ingress ---------------------------
|
||||
# -- Server lifecycle and connection ingress ---------------------------
|
||||
|
||||
async def start(self) -> None:
|
||||
from nanobot.utils.logging_bridge import redirect_lib_logging
|
||||
@@ -703,6 +703,10 @@ class WebSocketChannel(BaseChannel):
|
||||
workspace_scope=scope.payload(),
|
||||
)
|
||||
return
|
||||
if t == "transcribe_audio":
|
||||
event, payload = await webui_transcription_event(envelope)
|
||||
await self._send_event(connection, event, **payload)
|
||||
return
|
||||
if t == "message":
|
||||
cid = envelope.get("chat_id")
|
||||
content = envelope.get("content")
|
||||
|
||||
@@ -39,8 +39,19 @@ class ChannelsConfig(Base):
|
||||
show_reasoning: bool = True # surface model reasoning when channel implements it
|
||||
extract_document_text: bool = True # extract text from document attachments before sending to the model
|
||||
send_max_retries: int = Field(default=3, ge=0, le=10) # Max delivery attempts (initial send included)
|
||||
transcription_provider: str = "groq" # Voice transcription backend: "groq" or "openai"
|
||||
transcription_language: str | None = Field(default=None, pattern=r"^[a-z]{2,3}$") # Optional ISO-639-1 hint for audio transcription
|
||||
transcription_provider: str = "groq" # Deprecated: use top-level transcription.provider
|
||||
transcription_language: str | None = Field(default=None, pattern=r"^[a-z]{2,3}$") # Deprecated: use top-level transcription.language
|
||||
|
||||
|
||||
class TranscriptionConfig(Base):
|
||||
"""Cross-channel audio transcription configuration."""
|
||||
|
||||
enabled: bool = True
|
||||
provider: Literal["groq", "openai"] | None = None
|
||||
model: str | None = None
|
||||
language: str | None = Field(default=None, pattern=r"^[a-z]{2,3}$")
|
||||
max_duration_sec: int = Field(default=120, ge=1, le=600)
|
||||
max_upload_mb: int = Field(default=25, ge=1, le=100)
|
||||
|
||||
|
||||
class DreamConfig(Base):
|
||||
@@ -167,7 +178,7 @@ class AgentsConfig(Base):
|
||||
class ProviderConfig(Base):
|
||||
"""LLM provider configuration."""
|
||||
|
||||
api_key: str | None = None
|
||||
api_key: str | None = Field(default=None, repr=False)
|
||||
api_base: str | None = None
|
||||
api_type: Literal["auto", "chat_completions", "responses"] = "auto" # Request API surface
|
||||
extra_headers: dict[str, str] | None = None # Custom headers (e.g. APP-Code for AiHubMix)
|
||||
@@ -312,6 +323,7 @@ class Config(BaseSettings):
|
||||
|
||||
agents: AgentsConfig = Field(default_factory=AgentsConfig)
|
||||
channels: ChannelsConfig = Field(default_factory=ChannelsConfig)
|
||||
transcription: TranscriptionConfig = Field(default_factory=TranscriptionConfig)
|
||||
providers: ProvidersConfig = Field(default_factory=ProvidersConfig)
|
||||
api: ApiConfig = Field(default_factory=ApiConfig)
|
||||
gateway: GatewayConfig = Field(default_factory=GatewayConfig)
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
"""Voice transcription providers (Groq and OpenAI Whisper)."""
|
||||
"""Provider-specific voice transcription adapters.
|
||||
|
||||
This module only knows how to call external transcription APIs such as Groq
|
||||
and OpenAI Whisper. Product-level config fallback, WebUI upload validation,
|
||||
and channel integration live in ``nanobot.audio.transcription``.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import mimetypes
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
@@ -8,6 +14,15 @@ import httpx
|
||||
from loguru import logger
|
||||
|
||||
_TRANSCRIPTIONS_PATH = "audio/transcriptions"
|
||||
_AUDIO_MIME_OVERRIDES = {
|
||||
".m4a": "audio/mp4",
|
||||
".mpga": "audio/mpeg",
|
||||
".ogg": "audio/ogg",
|
||||
".opus": "audio/ogg",
|
||||
".wav": "audio/wav",
|
||||
".weba": "audio/webm",
|
||||
".webm": "audio/webm",
|
||||
}
|
||||
|
||||
|
||||
def _resolve_transcription_url(api_base: str | None, default_url: str) -> str:
|
||||
@@ -26,6 +41,14 @@ def _resolve_transcription_url(api_base: str | None, default_url: str) -> str:
|
||||
return f"{base}/{_TRANSCRIPTIONS_PATH}"
|
||||
|
||||
|
||||
def _audio_mime_type(path: Path) -> str:
|
||||
return (
|
||||
_AUDIO_MIME_OVERRIDES.get(path.suffix.lower())
|
||||
or mimetypes.guess_type(path.name)[0]
|
||||
or "application/octet-stream"
|
||||
)
|
||||
|
||||
|
||||
# Up to 3 retries (4 attempts total) with exponential backoff on transient
|
||||
# failures. Whisper endpoints occasionally return 502/503 under load, and
|
||||
# mobile-network transcription callers hit sporadic connect/read errors.
|
||||
@@ -71,7 +94,7 @@ async def _post_transcription_with_retry(
|
||||
async with httpx.AsyncClient() as client:
|
||||
for attempt in range(_MAX_RETRIES + 1):
|
||||
files = {
|
||||
"file": (path.name, data),
|
||||
"file": (path.name, data, _audio_mime_type(path)),
|
||||
"model": (None, model),
|
||||
}
|
||||
if language:
|
||||
@@ -113,6 +136,16 @@ async def _post_transcription_with_retry(
|
||||
|
||||
try:
|
||||
response.raise_for_status()
|
||||
except httpx.HTTPStatusError:
|
||||
body = response.text.strip().replace("\n", " ")[:500]
|
||||
logger.error(
|
||||
"{} transcription HTTP {}{}{}",
|
||||
provider_label,
|
||||
response.status_code,
|
||||
f" {response.reason_phrase}" if response.reason_phrase else "",
|
||||
f": {body}" if body else "",
|
||||
)
|
||||
return ""
|
||||
except Exception as e:
|
||||
logger.exception("{} transcription error: {}", provider_label, e)
|
||||
return ""
|
||||
@@ -144,6 +177,7 @@ class OpenAITranscriptionProvider:
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
language: str | None = None,
|
||||
model: str | None = None,
|
||||
):
|
||||
self.api_key = api_key or os.environ.get("OPENAI_API_KEY")
|
||||
self.api_url = _resolve_transcription_url(
|
||||
@@ -151,6 +185,7 @@ class OpenAITranscriptionProvider:
|
||||
"https://api.openai.com/v1/audio/transcriptions",
|
||||
)
|
||||
self.language = language or None
|
||||
self.model = model or "whisper-1"
|
||||
logger.debug("OpenAI transcription endpoint: {}", self.api_url)
|
||||
|
||||
async def transcribe(self, file_path: str | Path) -> str:
|
||||
@@ -165,7 +200,7 @@ class OpenAITranscriptionProvider:
|
||||
self.api_url,
|
||||
api_key=self.api_key,
|
||||
path=path,
|
||||
model="whisper-1",
|
||||
model=self.model,
|
||||
provider_label="OpenAI",
|
||||
language=self.language,
|
||||
)
|
||||
@@ -183,6 +218,7 @@ class GroqTranscriptionProvider:
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
language: str | None = None,
|
||||
model: str | None = None,
|
||||
):
|
||||
self.api_key = api_key or os.environ.get("GROQ_API_KEY")
|
||||
self.api_url = _resolve_transcription_url(
|
||||
@@ -190,6 +226,7 @@ class GroqTranscriptionProvider:
|
||||
"https://api.groq.com/openai/v1/audio/transcriptions",
|
||||
)
|
||||
self.language = language or None
|
||||
self.model = model or "whisper-large-v3"
|
||||
logger.debug("Groq transcription endpoint: {}", self.api_url)
|
||||
|
||||
async def transcribe(self, file_path: str | Path) -> str:
|
||||
@@ -215,7 +252,7 @@ class GroqTranscriptionProvider:
|
||||
self.api_url,
|
||||
api_key=self.api_key,
|
||||
path=path,
|
||||
model="whisper-large-v3",
|
||||
model=self.model,
|
||||
provider_label="Groq",
|
||||
language=self.language,
|
||||
)
|
||||
|
||||
@@ -18,13 +18,30 @@ from nanobot.utils.helpers import safe_filename
|
||||
DEFAULT_MAX_BYTES = 10 * 1024 * 1024
|
||||
MAX_FILE_SIZE = DEFAULT_MAX_BYTES
|
||||
|
||||
_DATA_URL_RE = re.compile(r"^data:([^;]+);base64,(.+)$", re.DOTALL)
|
||||
_DATA_URL_RE = re.compile(r"^data:([^;,]+)(?:;[^,]*)*;base64,(.+)$", re.DOTALL)
|
||||
_MIME_EXTENSION_OVERRIDES = {
|
||||
# Python's ``mimetypes`` maps browser-recorded audio/webm to ``.weba`` and
|
||||
# audio/ogg to ``.oga`` on macOS. Some transcription APIs validate by the
|
||||
# file extension and accept the canonical container extensions instead.
|
||||
"application/ogg": ".ogg",
|
||||
"audio/ogg": ".ogg",
|
||||
"audio/mpga": ".mpga",
|
||||
"audio/wav": ".wav",
|
||||
"audio/webm": ".webm",
|
||||
"audio/x-m4a": ".m4a",
|
||||
"audio/x-wav": ".wav",
|
||||
"audio/vnd.wave": ".wav",
|
||||
"video/webm": ".webm",
|
||||
}
|
||||
|
||||
|
||||
class FileSizeExceeded(Exception):
|
||||
class FileSizeExceededError(Exception):
|
||||
"""Raised when a decoded payload exceeds the caller's size limit."""
|
||||
|
||||
|
||||
FileSizeExceeded = FileSizeExceededError
|
||||
|
||||
|
||||
def save_base64_data_url(
|
||||
data_url: str,
|
||||
media_dir: Path,
|
||||
@@ -40,7 +57,7 @@ def save_base64_data_url(
|
||||
m = _DATA_URL_RE.match(data_url)
|
||||
if not m:
|
||||
return None
|
||||
mime_type, b64_payload = m.group(1), m.group(2)
|
||||
mime_type, b64_payload = m.group(1).strip().lower(), m.group(2)
|
||||
try:
|
||||
raw = base64.b64decode(b64_payload)
|
||||
except Exception:
|
||||
@@ -48,7 +65,7 @@ def save_base64_data_url(
|
||||
limit = DEFAULT_MAX_BYTES if max_bytes is None else max_bytes
|
||||
if len(raw) > limit:
|
||||
raise FileSizeExceeded(f"File exceeds {limit // (1024 * 1024)}MB limit")
|
||||
ext = mimetypes.guess_extension(mime_type) or ".bin"
|
||||
ext = _MIME_EXTENSION_OVERRIDES.get(mime_type) or mimetypes.guess_extension(mime_type) or ".bin"
|
||||
filename = f"{uuid.uuid4().hex[:12]}{ext}"
|
||||
dest = media_dir / safe_filename(filename)
|
||||
dest.write_bytes(raw)
|
||||
|
||||
@@ -15,6 +15,7 @@ from zoneinfo import ZoneInfo
|
||||
|
||||
import httpx
|
||||
|
||||
from nanobot.audio.transcription import resolve_transcription_config
|
||||
from nanobot.config.loader import get_config_path, load_config, save_config
|
||||
from nanobot.config.schema import ModelPresetConfig
|
||||
from nanobot.providers.image_generation import (
|
||||
@@ -90,6 +91,7 @@ _IMAGE_GENERATION_ASPECT_RATIOS = {
|
||||
"2:3",
|
||||
"21:9",
|
||||
}
|
||||
_TRANSCRIPTION_PROVIDERS = ("groq", "openai")
|
||||
_CONTEXT_WINDOW_TOKEN_OPTIONS = {65_536, 262_144}
|
||||
_MODEL_CONFIGURATION_SLUG_RE = re.compile(r"[^a-z0-9_-]+")
|
||||
_ENV_REF_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
|
||||
@@ -576,6 +578,22 @@ def _image_generation_provider_rows(config: Any) -> list[dict[str, Any]]:
|
||||
return rows
|
||||
|
||||
|
||||
def _transcription_provider_rows(config: Any) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for name in _TRANSCRIPTION_PROVIDERS:
|
||||
spec = find_by_name(name)
|
||||
provider_config = getattr(config.providers, name, None)
|
||||
rows.append({
|
||||
"name": name,
|
||||
"label": spec.label if spec is not None else name,
|
||||
"configured": bool(getattr(provider_config, "api_key", None)),
|
||||
"api_key_hint": _mask_secret_hint(getattr(provider_config, "api_key", None)),
|
||||
"api_base": getattr(provider_config, "api_base", None),
|
||||
"default_api_base": spec.default_api_base if spec and spec.default_api_base else None,
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def settings_payload(
|
||||
*,
|
||||
requires_restart: bool = False,
|
||||
@@ -633,6 +651,7 @@ def settings_payload(
|
||||
|
||||
search_config = config.tools.web.search
|
||||
image_config = config.tools.image_generation
|
||||
transcription = resolve_transcription_config(config)
|
||||
search_provider = (
|
||||
search_config.provider
|
||||
if search_config.provider in _WEB_SEARCH_PROVIDER_BY_NAME
|
||||
@@ -733,6 +752,16 @@ def settings_payload(
|
||||
"save_dir": image_config.save_dir,
|
||||
"providers": image_providers,
|
||||
},
|
||||
"transcription": {
|
||||
"enabled": transcription.enabled,
|
||||
"provider": transcription.provider,
|
||||
"provider_configured": transcription.configured,
|
||||
"model": transcription.model,
|
||||
"language": transcription.language,
|
||||
"max_duration_sec": transcription.max_duration_sec,
|
||||
"max_upload_mb": transcription.max_upload_mb,
|
||||
"providers": _transcription_provider_rows(config),
|
||||
},
|
||||
"runtime": {
|
||||
"config_path": str(get_config_path().expanduser()),
|
||||
"workspace_path": str(config.workspace_path),
|
||||
@@ -1311,3 +1340,71 @@ def update_image_generation_settings(query: QueryParams) -> dict[str, Any]:
|
||||
if changed:
|
||||
save_config(config)
|
||||
return settings_payload(requires_restart=changed)
|
||||
|
||||
|
||||
def update_transcription_settings(query: QueryParams) -> dict[str, Any]:
|
||||
config = load_config()
|
||||
transcription = config.transcription
|
||||
changed = False
|
||||
|
||||
enabled = _query_first(query, "enabled")
|
||||
if enabled is not None:
|
||||
parsed_enabled = _parse_bool(enabled, "enabled")
|
||||
if transcription.enabled != parsed_enabled:
|
||||
transcription.enabled = parsed_enabled
|
||||
changed = True
|
||||
|
||||
provider = _query_first(query, "provider")
|
||||
if provider is not None:
|
||||
provider = provider.strip().lower()
|
||||
if provider not in _TRANSCRIPTION_PROVIDERS:
|
||||
raise WebUISettingsError("unknown transcription provider")
|
||||
if transcription.provider != provider:
|
||||
transcription.provider = provider # type: ignore[assignment]
|
||||
changed = True
|
||||
|
||||
model = _query_first(query, "model")
|
||||
if model is not None:
|
||||
model = model.strip() or None
|
||||
if model is not None and len(model) > 200:
|
||||
raise WebUISettingsError("transcription model is too long")
|
||||
if transcription.model != model:
|
||||
transcription.model = model
|
||||
changed = True
|
||||
|
||||
language = _query_first(query, "language")
|
||||
if language is not None:
|
||||
language = language.strip().lower() or None
|
||||
if language is not None and not re.fullmatch(r"[a-z]{2,3}", language):
|
||||
raise WebUISettingsError("transcription language must be 2-3 lowercase letters")
|
||||
if transcription.language != language:
|
||||
transcription.language = language
|
||||
changed = True
|
||||
|
||||
max_duration_sec = _query_first_alias(query, "max_duration_sec", "maxDurationSec")
|
||||
if max_duration_sec is not None:
|
||||
try:
|
||||
parsed_duration = int(max_duration_sec)
|
||||
except ValueError:
|
||||
raise WebUISettingsError("max_duration_sec must be an integer") from None
|
||||
if parsed_duration < 1 or parsed_duration > 600:
|
||||
raise WebUISettingsError("max_duration_sec must be between 1 and 600")
|
||||
if transcription.max_duration_sec != parsed_duration:
|
||||
transcription.max_duration_sec = parsed_duration
|
||||
changed = True
|
||||
|
||||
max_upload_mb = _query_first_alias(query, "max_upload_mb", "maxUploadMb")
|
||||
if max_upload_mb is not None:
|
||||
try:
|
||||
parsed_upload = int(max_upload_mb)
|
||||
except ValueError:
|
||||
raise WebUISettingsError("max_upload_mb must be an integer") from None
|
||||
if parsed_upload < 1 or parsed_upload > 100:
|
||||
raise WebUISettingsError("max_upload_mb must be between 1 and 100")
|
||||
if transcription.max_upload_mb != parsed_upload:
|
||||
transcription.max_upload_mb = parsed_upload
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
save_config(config)
|
||||
return settings_payload()
|
||||
|
||||
@@ -33,6 +33,7 @@ from nanobot.webui.settings_api import (
|
||||
update_model_configuration,
|
||||
update_network_safety_settings,
|
||||
update_provider_settings,
|
||||
update_transcription_settings,
|
||||
update_web_search_settings,
|
||||
)
|
||||
|
||||
@@ -100,6 +101,8 @@ class WebUISettingsRouter:
|
||||
return self._handle_settings_web_search_update(request)
|
||||
if path == "/api/settings/image-generation/update":
|
||||
return self._handle_settings_image_generation_update(request)
|
||||
if path == "/api/settings/transcription/update":
|
||||
return self._handle_settings_transcription_update(request)
|
||||
if path == "/api/settings/network-safety/update":
|
||||
return self._handle_settings_network_safety_update(request)
|
||||
if path == "/api/settings/cli-apps":
|
||||
@@ -275,6 +278,15 @@ class WebUISettingsRouter:
|
||||
return self._error_response(e.status, e.message)
|
||||
return self._json_response(self._with_restart_state(payload, section="image"))
|
||||
|
||||
def _handle_settings_transcription_update(self, request: WsRequest) -> Response:
|
||||
if not self._authorized(request):
|
||||
return self._unauthorized()
|
||||
try:
|
||||
payload = update_transcription_settings(self._query(request))
|
||||
except WebUISettingsError as e:
|
||||
return self._error_response(e.status, e.message)
|
||||
return self._json_response(self._with_restart_state(payload))
|
||||
|
||||
def _handle_settings_network_safety_update(self, request: WsRequest) -> Response:
|
||||
if not self._authorized(request):
|
||||
return self._unauthorized()
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
"""WebUI transcription envelope handling.
|
||||
|
||||
The WebSocket channel owns transport and subscription fan-out. This module owns
|
||||
the WebUI-specific audio transcription action carried over that socket.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from nanobot.audio.transcription import (
|
||||
TranscriptionIngressError,
|
||||
resolve_transcription_config,
|
||||
transcribe_audio_data_url,
|
||||
)
|
||||
from nanobot.config.loader import load_config
|
||||
|
||||
_MAX_REQUEST_ID_LENGTH = 80
|
||||
|
||||
|
||||
async def webui_transcription_event(envelope: dict[str, Any]) -> tuple[str, dict[str, Any]]:
|
||||
"""Return the WS event name and payload for one WebUI transcription request."""
|
||||
request_id = envelope.get("request_id")
|
||||
valid_request_id = (
|
||||
isinstance(request_id, str)
|
||||
and 0 < len(request_id) <= _MAX_REQUEST_ID_LENGTH
|
||||
)
|
||||
|
||||
def error(detail: str, **extra: Any) -> tuple[str, dict[str, Any]]:
|
||||
payload: dict[str, Any] = {"detail": detail, **extra}
|
||||
if valid_request_id:
|
||||
payload["request_id"] = request_id
|
||||
return "transcription_error", payload
|
||||
|
||||
if not valid_request_id:
|
||||
return error("invalid_request")
|
||||
|
||||
try:
|
||||
text = await transcribe_audio_data_url(
|
||||
envelope.get("data_url"),
|
||||
resolve_transcription_config(load_config()),
|
||||
duration_ms=envelope.get("duration_ms"),
|
||||
)
|
||||
except TranscriptionIngressError as exc:
|
||||
return error(exc.detail, **exc.extra)
|
||||
return "transcription_result", {"request_id": request_id, "text": text}
|
||||
Reference in New Issue
Block a user