fix(stt): dispatch live-API-only Gemini models over the Live WebSocket transport (#4006)
Addresses #4006 by letting Gemini STT models that only exist on the Live API transcribe instead of failing. transcribeGemini sends audio to :generateContent and that is the only Gemini path. A model that is realtime-only (exposed by the Live API's bidiGenerateContent WebSocket) therefore fails outright, even though the account can transcribe it. open-sse/handlers/geminiLiveStt.js owns the WebSocket lifecycle: opens :bidiGenerateContent, sends setup frame, waits for setupComplete, streams audio as realtimeInput media chunks, and settles on turnComplete. Dispatch is driven by transport marker 'gemini-live'. Adds custom model transport persistence and selection on the dashboard.
This commit is contained in:
Nick Nyanjui
266
open-sse/handlers/geminiLiveStt.js
Normal file
266
open-sse/handlers/geminiLiveStt.js
Normal file
@@ -0,0 +1,266 @@
|
||||
import { Buffer } from "node:buffer";
|
||||
|
||||
// Gemini Live API realtime STT transport.
|
||||
//
|
||||
// The REST generateContent path (sttCore.transcribeGemini) only transcribes
|
||||
// whole files inline. The Live API's `:bidiGenerateContent` WebSocket is the
|
||||
// streaming counterpart: audio goes up as realtimeInput mediaChunks and the
|
||||
// server pushes incremental `serverContent.inputTranscription` events back.
|
||||
// This module owns the socket lifecycle only — envelope/response shaping
|
||||
// stays in sttCore so the engine's single STT exit shape is preserved.
|
||||
//
|
||||
// Marker contract: dispatched from sttCore's format-switch when the model
|
||||
// entry carries `transport: "gemini-live"` (registry) or the caller passes a
|
||||
// transport string (custom models). Never keyed on a hardcoded model id here.
|
||||
//
|
||||
// Transport behavior:
|
||||
// - Node >= 22 global WebSocket (undici). No new dependency.
|
||||
// - Live API expects low-latency PCM; other containers are forwarded with
|
||||
// their declared MIME unchanged (provider-side rejection is surfaced).
|
||||
// - Text accumulation is append-only over inputTranscription segments and
|
||||
// ends on serverContent.turnComplete (or graceful close with partial text).
|
||||
// - Transcription deltas are kept per-frame (chunks[]) so sttCore can shape
|
||||
// verbose_json segments without fabricating timestamps. goAway advisements
|
||||
// rotate the socket once per call: setup replay + byte-offset resume.
|
||||
|
||||
const SETUP_TIMEOUT_MS = 10_000; // open → setupComplete
|
||||
const TURN_TIMEOUT_MS = 60_000; // audio streamed → turnComplete
|
||||
const MAX_TIMEOUT_MS = 300_000; // clamp ceiling for client-supplied lifecycle knobs
|
||||
const CHUNK_BYTES = 16_384; // ~0.5s of 16-bit 16kHz mono PCM
|
||||
const GOAWAY_RECONNECTS = 1; // socket rotations honoured per call
|
||||
|
||||
class GeminiLiveError extends Error {
|
||||
constructor(message, status) {
|
||||
super(message);
|
||||
this.name = "GeminiLiveError";
|
||||
this.status = status || 502;
|
||||
}
|
||||
}
|
||||
|
||||
// REST base (https://host/v1beta/models) → Live WS base
|
||||
// (wss://host/ws/api/v1beta/models), then the bidiGenerateContent endpoint.
|
||||
function toLiveWsUrl(baseUrl, model, token) {
|
||||
const url = new URL(baseUrl);
|
||||
url.protocol = "wss:";
|
||||
if (!url.pathname.startsWith("/ws/")) url.pathname = `/ws/api${url.pathname}`;
|
||||
const base = url.toString().replace(/\/+$/, "");
|
||||
return `${base}/${encodeURIComponent(model)}:bidiGenerateContent?key=${encodeURIComponent(token || "")}`;
|
||||
}
|
||||
|
||||
// Bind socket events supporting BOTH handler styles: addEventListener
|
||||
// (browser WebSocket, undici) and onopen/onmessage property assignment
|
||||
// (minimal polyfills). Whichever the implementation exposes, it works.
|
||||
function bindSocket(ws, { onOpen, onMessage, onError, onClose }) {
|
||||
if (typeof ws.addEventListener === "function") {
|
||||
ws.addEventListener("open", onOpen);
|
||||
ws.addEventListener("message", onMessage);
|
||||
ws.addEventListener("error", onError);
|
||||
ws.addEventListener("close", onClose);
|
||||
return;
|
||||
}
|
||||
ws.onopen = onOpen;
|
||||
ws.onmessage = onMessage;
|
||||
ws.onerror = onError;
|
||||
ws.onclose = onClose;
|
||||
}
|
||||
|
||||
function parseFrame(data) {
|
||||
try {
|
||||
return JSON.parse(typeof data === "string" ? data : String(data));
|
||||
} catch {
|
||||
return null; // non-JSON frames carry no Live API semantics
|
||||
}
|
||||
}
|
||||
|
||||
function firstStringField(formData, key) {
|
||||
const v = typeof formData?.get === "function" ? formData.get(key) : null;
|
||||
return typeof v === "string" && v.trim() ? v.trim() : "";
|
||||
}
|
||||
|
||||
// Lifecycle knobs the live registry entry advertises in params[]
|
||||
// (setup/turn timeouts). They ride the same formData pass-through sttCore
|
||||
// gives every transport — no sttCore change needed to reach this leaf.
|
||||
function firstNumberField(formData, key, fallback) {
|
||||
const n = Number(firstStringField(formData, key));
|
||||
return Number.isFinite(n) && n > 0 ? Math.min(n, MAX_TIMEOUT_MS) : fallback;
|
||||
}
|
||||
|
||||
/**
|
||||
* Transcribe an audio File via the Gemini Live bidirectional stream.
|
||||
* @returns {Promise<{text: string, chunks: string[]}>} transcript plus the raw
|
||||
* incremental inputTranscription deltas (sttCore shapes verbose_json from them).
|
||||
* @throws {GeminiLiveError} with .status for the sttCore error envelope.
|
||||
*/
|
||||
export async function transcribeGeminiLive({ cfg, file, model, token, formData, mimeType }) {
|
||||
const WS = globalThis.WebSocket;
|
||||
if (!WS) throw new GeminiLiveError("Gemini Live transport needs global WebSocket (Node >= 22)", 502);
|
||||
|
||||
const buf = Buffer.from(await file.arrayBuffer());
|
||||
if (!buf.length) throw new GeminiLiveError("Empty audio file", 400);
|
||||
|
||||
const instruction = firstStringField(formData, "prompt") || "Transcribe the spoken audio verbatim.";
|
||||
const language = firstStringField(formData, "language");
|
||||
const setupTimeoutMs = firstNumberField(formData, "setup_timeout_ms", SETUP_TIMEOUT_MS);
|
||||
const turnTimeoutMs = firstNumberField(formData, "turn_timeout_ms", TURN_TIMEOUT_MS);
|
||||
// system_instruction (registry param) overrides the built-in transcription
|
||||
// directive wholesale; prompt/language only shape the default.
|
||||
const instructionOverride = firstStringField(formData, "system_instruction");
|
||||
const systemText = instructionOverride
|
||||
|| (language ? `${instruction} Language: ${language}.` : instruction);
|
||||
const wsUrl = toLiveWsUrl(cfg.baseUrl, model, token);
|
||||
|
||||
return await new Promise((resolve, reject) => {
|
||||
let text = "";
|
||||
const chunks = []; // raw inputTranscription deltas, shaped by sttCore
|
||||
let settled = false;
|
||||
let timer = null;
|
||||
let goAwayTimer = null;
|
||||
let ws = null;
|
||||
let generation = 0; // socket identity: superseded closes never settle
|
||||
let sentBytes = 0; // audio prefix already handed to the live socket
|
||||
let goAwayReconnects = GOAWAY_RECONNECTS;
|
||||
|
||||
const arm = (ms, message) => {
|
||||
if (timer) clearTimeout(timer);
|
||||
timer = setTimeout(() => fail(new GeminiLiveError(message, 504)), ms);
|
||||
};
|
||||
const shutdown = () => {
|
||||
if (timer) { clearTimeout(timer); timer = null; }
|
||||
if (goAwayTimer) { clearTimeout(goAwayTimer); goAwayTimer = null; }
|
||||
// ws is null until the first open() dials (and stays null when the
|
||||
// constructor throws) — fail() runs shutdown() on that path.
|
||||
if (!ws) return;
|
||||
try {
|
||||
if (ws.readyState === WS.OPEN || ws.readyState === WS.CONNECTING) ws.close(1000);
|
||||
} catch { /* socket already dead — outcome is already settled */ }
|
||||
};
|
||||
const succeed = () => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
shutdown();
|
||||
resolve({ text, chunks });
|
||||
};
|
||||
const fail = (err) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
shutdown();
|
||||
reject(err);
|
||||
};
|
||||
const send = (frame) => {
|
||||
if (ws.readyState !== WS.OPEN) return false;
|
||||
try {
|
||||
ws.send(JSON.stringify(frame));
|
||||
} catch {
|
||||
return false; // socket died mid-send — streamAudioAndPrompt maps this to a 502
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
// Streams every byte not yet sent, then the flushing text turn. After a
|
||||
// goAway rotation this resumes from sentBytes — no audio re-upload.
|
||||
const streamAudioAndPrompt = () => {
|
||||
for (let off = sentBytes; off < buf.length; off += CHUNK_BYTES) {
|
||||
const mediaChunk = buf.subarray(off, off + CHUNK_BYTES).toString("base64");
|
||||
if (!send({ realtimeInput: { mediaChunks: [{ mimeType, data: mediaChunk }] } })) {
|
||||
fail(new GeminiLiveError("Gemini Live socket closed while streaming audio", 502));
|
||||
return;
|
||||
}
|
||||
sentBytes = Math.min(off + CHUNK_BYTES, buf.length);
|
||||
}
|
||||
// Final user turn: flushes the recognizer and yields turnComplete.
|
||||
send({ clientContent: { turns: [{ parts: [{ text: systemText }] }], turnComplete: true } });
|
||||
};
|
||||
|
||||
// goAway: the server names the instant it will force-close this socket.
|
||||
// Graceful play = rotate BEFORE the deadline: retire the live socket,
|
||||
// dial a fresh one, replay setup, resume audio from sentBytes — text and
|
||||
// chunks survive the hop. Once the advisory budget is spent a later
|
||||
// goAway is left to the close path, which settles on partial transcript.
|
||||
const scheduleGoAwayReconnect = (goAway) => {
|
||||
if (settled || goAwayTimer || goAwayReconnects <= 0) return;
|
||||
const deadline = Date.parse(typeof goAway?.time === "string" ? goAway.time : "");
|
||||
const delay = Number.isFinite(deadline)
|
||||
? Math.max(0, Math.min(deadline - Date.now(), setupTimeoutMs))
|
||||
: 0;
|
||||
goAwayTimer = setTimeout(() => {
|
||||
goAwayTimer = null;
|
||||
if (settled) return;
|
||||
goAwayReconnects--;
|
||||
generation++;
|
||||
try { ws?.close(1000); } catch { /* deadline crossed mid-flight — re-dial anyway */ }
|
||||
open();
|
||||
}, delay);
|
||||
};
|
||||
|
||||
const open = () => {
|
||||
const gen = ++generation;
|
||||
try {
|
||||
ws = new WS(wsUrl);
|
||||
} catch {
|
||||
fail(new GeminiLiveError("Gemini Live websocket connection failed", 502));
|
||||
return;
|
||||
}
|
||||
bindSocket(ws, {
|
||||
onOpen: () => {
|
||||
if (settled || gen !== generation) return;
|
||||
arm(setupTimeoutMs, "Gemini Live timed out waiting for setupComplete");
|
||||
send({
|
||||
setup: {
|
||||
model: `models/${model}`,
|
||||
generationConfig: {
|
||||
responseModalities: ["TEXT"],
|
||||
inputAudioTranscription: {},
|
||||
},
|
||||
systemInstruction: { parts: [{ text: systemText }] },
|
||||
},
|
||||
});
|
||||
},
|
||||
onMessage: (ev) => {
|
||||
if (settled || gen !== generation) return;
|
||||
const frame = parseFrame(ev?.data);
|
||||
if (!frame) return;
|
||||
|
||||
if (frame.error) {
|
||||
const e = frame.error;
|
||||
fail(new GeminiLiveError(`Gemini Live error${e.status ? ` (${e.status})` : ""}: ${e.message || "unknown"}`, 502));
|
||||
return;
|
||||
}
|
||||
if (frame.goAway) {
|
||||
scheduleGoAwayReconnect(frame.goAway);
|
||||
return;
|
||||
}
|
||||
|
||||
const sc = frame.serverContent;
|
||||
if (!sc) return;
|
||||
|
||||
const delta = typeof sc.inputTranscription?.text === "string" ? sc.inputTranscription.text : "";
|
||||
// Trim before testing: a padding-only frame carries no transcript and
|
||||
// must not make an empty run look like a partial success on close.
|
||||
if (delta.trim()) {
|
||||
text += delta;
|
||||
chunks.push(delta);
|
||||
}
|
||||
|
||||
if (sc.setupComplete) {
|
||||
arm(turnTimeoutMs, "Gemini Live transcription timed out");
|
||||
streamAudioAndPrompt();
|
||||
return;
|
||||
}
|
||||
if (sc.turnComplete) succeed();
|
||||
},
|
||||
onError: () => {
|
||||
if (settled || gen !== generation) return;
|
||||
fail(new GeminiLiveError("Gemini Live websocket connection failed", 502));
|
||||
},
|
||||
onClose: (ev) => {
|
||||
if (settled || gen !== generation) return;
|
||||
// Partial transcript beats a hard error on graceful close; silence is one.
|
||||
if (text.trim()) succeed();
|
||||
else fail(new GeminiLiveError(`Gemini Live socket closed before completion${ev?.code ? ` (code ${ev.code})` : ""}`, 502));
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
open();
|
||||
});
|
||||
}
|
||||
@@ -1,5 +1,7 @@
|
||||
import { Buffer } from "node:buffer";
|
||||
import { createErrorResult } from "../utils/error.js";
|
||||
import { transcribeGeminiLive } from "./geminiLiveStt.js";
|
||||
import { PROVIDER_MODELS, PROVIDER_ID_TO_ALIAS } from "../config/providerModels.js";
|
||||
import { HTTP_STATUS } from "../config/runtimeConfig.js";
|
||||
|
||||
// Build auth headers from sttConfig + token
|
||||
@@ -162,11 +164,26 @@ function jsonResponse(obj) {
|
||||
};
|
||||
}
|
||||
|
||||
// Model-level transport marker (registry models[].transport, e.g. the Gemini
|
||||
// live STT entry's "gemini-live", or a custom model's stored transport).
|
||||
// Dispatch reads the marker — never a hardcoded model id — so new realtime
|
||||
// providers extend sttCore through data, not code.
|
||||
function resolveModelTransport(provider, model) {
|
||||
const key = PROVIDER_ID_TO_ALIAS[provider] || provider;
|
||||
const models = PROVIDER_MODELS[key] || PROVIDER_MODELS[provider];
|
||||
if (!Array.isArray(models)) return null;
|
||||
const entry = models.find((m) => m && m.id === model && (m.kind || "llm") === "stt");
|
||||
const marker = typeof entry?.transport === "string" ? entry.transport.trim() : "";
|
||||
return marker || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* STT core handler — dispatch by sttConfig.format.
|
||||
* STT core handler — dispatch by model transport marker, else sttConfig.format.
|
||||
* `transport` is the caller-supplied marker override (custom models resolve
|
||||
* it in the app layer; built-ins fall back to the registry entry marker).
|
||||
* @returns {Promise<{success, response, status?, error?}>}
|
||||
*/
|
||||
export async function handleSttCore({ provider, model, formData, credentials, sttConfig }) {
|
||||
export async function handleSttCore({ provider, model, formData, credentials, sttConfig, transport }) {
|
||||
const file = formData.get("file");
|
||||
if (!file) return createErrorResult(HTTP_STATUS.BAD_REQUEST, "Missing required field: file");
|
||||
|
||||
@@ -186,8 +203,29 @@ export async function handleSttCore({ provider, model, formData, credentials, st
|
||||
return createErrorResult(HTTP_STATUS.UNAUTHORIZED, `No credentials for STT provider: ${provider}`);
|
||||
}
|
||||
|
||||
// Format-switch extension: an explicit caller marker wins over the registry
|
||||
// marker; with neither, the provider-default sttConfig.format applies.
|
||||
const marker = (typeof transport === "string" && transport.trim()) ? transport.trim() : resolveModelTransport(provider, model);
|
||||
|
||||
try {
|
||||
switch (cfg.format) {
|
||||
switch (marker || cfg.format) {
|
||||
case "gemini-live": {
|
||||
const live = await transcribeGeminiLive({ cfg, file, model, token, formData, mimeType: resolveAudioContentType(file) });
|
||||
// response_format parity with the OpenAI-compatible transport: default
|
||||
// envelope stays {text}; verbose_json adds segments mapped from the
|
||||
// Live API's incremental inputTranscription deltas. Those frames carry
|
||||
// NO timestamps, so segments expose {id,text} only (id = delta order,
|
||||
// Whisper-compatible 0-based) — start/end/duration are deliberately
|
||||
// absent rather than fabricated as zeros, which would misrepresent
|
||||
// provider data to callers diffing transports.
|
||||
const fmt = typeof formData?.get === "function"
|
||||
? String(formData.get("response_format") ?? "").trim().toLowerCase()
|
||||
: "";
|
||||
if (fmt === "verbose_json") {
|
||||
return jsonResponse({ text: live.text, segments: live.chunks.map((segText, id) => ({ id, text: segText })) });
|
||||
}
|
||||
return jsonResponse({ text: live.text });
|
||||
}
|
||||
case "deepgram": return await transcribeDeepgram(cfg, file, model, token, formData);
|
||||
case "assemblyai": return await transcribeAssemblyAI(cfg, file, model, token);
|
||||
case "nvidia-asr": return await transcribeNvidia(cfg, file, model, token);
|
||||
@@ -196,6 +234,6 @@ export async function handleSttCore({ provider, model, formData, credentials, st
|
||||
default: return await transcribeOpenAICompatible(cfg, file, model, token, formData);
|
||||
}
|
||||
} catch (err) {
|
||||
return createErrorResult(HTTP_STATUS.BAD_GATEWAY, err.message || "STT request failed");
|
||||
return createErrorResult(err.status || HTTP_STATUS.BAD_GATEWAY, err.message || "STT request failed");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,6 +58,7 @@ export default {
|
||||
{ id: "gemini-2.5-flash", name: "Gemini 2.5 Flash", params: ["language","prompt"], kind: "stt" },
|
||||
{ id: "gemini-2.5-flash-lite", name: "Gemini 2.5 Flash Lite (Cheapest)", params: ["language","prompt"], kind: "stt" },
|
||||
{ id: "gemini-2.0-flash", name: "Gemini 2.0 Flash", params: ["language","prompt"], kind: "stt" },
|
||||
{ id: "gemini-2.5-flash-native-audio-preview-09-17", name: "Gemini Live Transcription (Realtime)", params: ["language","prompt","system_instruction","setup_timeout_ms","turn_timeout_ms"], kind: "stt", transport: "gemini-live" },
|
||||
{ id: "gemini-3.1-flash-tts-preview", name: "Gemini 3.1 Flash TTS", kind: "tts" },
|
||||
{ id: "gemini-2.5-flash-preview-tts", name: "Gemini 2.5 Flash TTS", kind: "tts" },
|
||||
{ id: "gemini-2.5-pro-preview-tts", name: "Gemini 2.5 Pro TTS", kind: "tts" },
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
import { useState, useEffect } from "react";
|
||||
import PropTypes from "prop-types";
|
||||
import { Button, Modal, Toggle } from "@/shared/components";
|
||||
import { CAPACITY_META } from "@/shared/constants/models";
|
||||
import { Button, Modal, Select, Toggle } from "@/shared/components";
|
||||
import { CAPACITY_META, STT_TRANSPORT_META, STT_TRANSPORTS } from "@/shared/constants/models";
|
||||
|
||||
const defaultCaps = () => Object.fromEntries(Object.keys(CAPACITY_META).map((key) => [key, false]));
|
||||
|
||||
@@ -13,10 +13,12 @@ export default function AddCustomModelModal({ isOpen, providerAlias, providerDis
|
||||
const [testStatus, setTestStatus] = useState(null); // null | "testing" | "ok" | "error"
|
||||
const [testError, setTestError] = useState("");
|
||||
const [saving, setSaving] = useState(false);
|
||||
// Realtime dispatch marker for the transport select; "" = provider default REST.
|
||||
const [transport, setTransport] = useState("");
|
||||
|
||||
// Reset state when modal opens
|
||||
useEffect(() => {
|
||||
if (isOpen) { setModelId(""); setCaps(defaultCaps()); setTestStatus(null); setTestError(""); }
|
||||
if (isOpen) { setModelId(""); setCaps(defaultCaps()); setTransport(""); setTestStatus(null); setTestError(""); }
|
||||
}, [isOpen]);
|
||||
|
||||
// Strip provider's own alias prefix (e.g. "cc/model" -> "model" for cc provider)
|
||||
@@ -50,7 +52,9 @@ export default function AddCustomModelModal({ isOpen, providerAlias, providerDis
|
||||
if (!cleanId || saving) return;
|
||||
setSaving(true);
|
||||
try {
|
||||
await onSave(cleanId, caps);
|
||||
// caps.stt is UI-only; the parent save flow derives the model type from
|
||||
// it and forwards the pinned transport (null unless the caller picked one).
|
||||
await onSave(cleanId, caps, caps.stt ? transport : null);
|
||||
} finally {
|
||||
setSaving(false);
|
||||
}
|
||||
@@ -106,6 +110,32 @@ export default function AddCustomModelModal({ isOpen, providerAlias, providerDis
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* STT is a model TYPE, not a chat capability: the save flow turns this
|
||||
flag into type "stt" (the API honours a transport only on stt
|
||||
records). The select pins the realtime dispatch marker persisted
|
||||
with the model; the whitelist is the shared STT_TRANSPORT_META. */}
|
||||
<div>
|
||||
<Toggle
|
||||
checked={!!caps.stt}
|
||||
onChange={(v) => { setCaps((prev) => ({ ...prev, stt: v })); if (!v) setTransport(""); }}
|
||||
label="Speech to text"
|
||||
description="Transcribes audio via /v1/audio/transcriptions"
|
||||
size="sm"
|
||||
/>
|
||||
{caps.stt && (
|
||||
<div className="mt-3">
|
||||
<Select
|
||||
label="Transport"
|
||||
value={transport}
|
||||
onChange={(e) => setTransport(e.target.value)}
|
||||
placeholder="Provider default (REST)"
|
||||
options={STT_TRANSPORTS.map((t) => ({ value: t, label: STT_TRANSPORT_META[t].label }))}
|
||||
hint="Realtime transport marker for the STT dispatcher. Empty keeps the provider's REST format."
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Test result */}
|
||||
{testStatus === "ok" && (
|
||||
<div className="flex items-center gap-2 text-sm text-green-600">
|
||||
|
||||
@@ -552,12 +552,14 @@ export default function ProviderDetailPage() {
|
||||
}
|
||||
};
|
||||
|
||||
const handleAddCustomModel = async (modelId, type = "llm", providerAliasOverride = providerStorageAlias, caps) => {
|
||||
// `transport` pins a realtime STT dispatch marker (shared whitelist
|
||||
// STT_TRANSPORT_META); the API only honours it on type "stt" records.
|
||||
const handleAddCustomModel = async (modelId, type = "llm", providerAliasOverride = providerStorageAlias, caps, transport) => {
|
||||
try {
|
||||
const res = await fetch("/api/models/custom", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ providerAlias: providerAliasOverride, id: modelId, type, ...(caps ? { caps } : {}) }),
|
||||
body: JSON.stringify({ providerAlias: providerAliasOverride, id: modelId, type, ...(caps ? { caps } : {}), ...(transport ? { transport } : {}) }),
|
||||
});
|
||||
if (res.ok) {
|
||||
await fetchCustomModels();
|
||||
@@ -1904,8 +1906,10 @@ export default function ProviderDetailPage() {
|
||||
isOpen={showAddCustomModel}
|
||||
providerAlias={providerStorageAlias}
|
||||
providerDisplayAlias={providerDisplayAlias}
|
||||
onSave={async (modelId, caps) => {
|
||||
await handleAddCustomModel(modelId, "llm", providerStorageAlias, caps);
|
||||
onSave={async (modelId, caps, transport) => {
|
||||
// caps.stt is a UI-only flag; the API accepts transports only on
|
||||
// type "stt" records, so the save derives the type from it.
|
||||
await handleAddCustomModel(modelId, caps?.stt ? "stt" : "llm", providerStorageAlias, caps, transport);
|
||||
setShowAddCustomModel(false);
|
||||
}}
|
||||
onClose={() => setShowAddCustomModel(false)}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { getCustomModels, addCustomModel, deleteCustomModel } from "@/models";
|
||||
import { CAPACITY_META } from "@/shared/constants/models";
|
||||
import { CAPACITY_META, isSttTransport } from "@/shared/constants/models";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
@@ -14,6 +14,16 @@ function sanitizeCaps(caps) {
|
||||
return Object.keys(clean).length ? clean : null;
|
||||
}
|
||||
|
||||
// Accepted STT transport markers live in the shared whitelist
|
||||
// (src/shared/constants/models STT_TRANSPORT_META) — the dashboard transport
|
||||
// select and this validator must agree on one set, so neither owns a copy.
|
||||
// Unknown or mistyped values are silently dropped, the same policy
|
||||
// sanitizeCaps applies to capability keys.
|
||||
function sanitizeTransport(transport, type) {
|
||||
if (type !== "stt" || !isSttTransport(transport)) return null;
|
||||
return transport.trim();
|
||||
}
|
||||
|
||||
// GET /api/models/custom - List all custom models
|
||||
export async function GET() {
|
||||
try {
|
||||
@@ -28,12 +38,13 @@ export async function GET() {
|
||||
// POST /api/models/custom - Add custom model
|
||||
export async function POST(request) {
|
||||
try {
|
||||
const { providerAlias, id, type, name, caps } = await request.json();
|
||||
const { providerAlias, id, type, name, caps, transport } = await request.json();
|
||||
if (!providerAlias || !id) {
|
||||
return NextResponse.json({ error: "providerAlias and id required" }, { status: 400 });
|
||||
}
|
||||
const cleanCaps = sanitizeCaps(caps);
|
||||
const added = await addCustomModel({ providerAlias, id, type: type || "llm", name, ...(cleanCaps ? { caps: cleanCaps } : {}) });
|
||||
const cleanTransport = sanitizeTransport(transport, type || "llm");
|
||||
const added = await addCustomModel({ providerAlias, id, type: type || "llm", name, ...(cleanCaps ? { caps: cleanCaps } : {}), ...(cleanTransport ? { transport: cleanTransport } : {}) });
|
||||
return NextResponse.json({ success: true, added });
|
||||
} catch (error) {
|
||||
console.log("Error adding custom model:", error);
|
||||
|
||||
@@ -30,8 +30,8 @@ export async function getCustomModels() {
|
||||
}
|
||||
|
||||
// Atomic upsert inside transaction to prevent duplicate races.
|
||||
// Re-adding an existing model updates caps/name without resetting omitted fields.
|
||||
export async function addCustomModel({ providerAlias, id, type = "llm", name, caps }) {
|
||||
// Re-adding an existing model updates caps/name/transport without resetting omitted fields.
|
||||
export async function addCustomModel({ providerAlias, id, type = "llm", name, caps, transport }) {
|
||||
const k = customKey(providerAlias, id, type);
|
||||
const db = await getAdapter();
|
||||
let added = false;
|
||||
@@ -39,11 +39,11 @@ export async function addCustomModel({ providerAlias, id, type = "llm", name, ca
|
||||
const row = db.get(`SELECT value FROM kv WHERE scope = 'customModels' AND key = ?`, [k]);
|
||||
if (row) {
|
||||
const prev = parseJson(row.value) || {};
|
||||
const next = { ...prev, ...(name ? { name } : {}), ...(caps ? { caps } : {}) };
|
||||
const next = { ...prev, ...(name ? { name } : {}), ...(caps ? { caps } : {}), ...(transport ? { transport } : {}) };
|
||||
db.run(`UPDATE kv SET value = ? WHERE scope = 'customModels' AND key = ?`, [stringifyJson(next), k]);
|
||||
return;
|
||||
}
|
||||
const value = stringifyJson({ providerAlias, id, type, name: name || id, ...(caps ? { caps } : {}) });
|
||||
const value = stringifyJson({ providerAlias, id, type, name: name || id, ...(caps ? { caps } : {}), ...(transport ? { transport } : {}) });
|
||||
db.run(`INSERT INTO kv(scope, key, value) VALUES('customModels', ?, ?)`, [k, value]);
|
||||
added = true;
|
||||
});
|
||||
|
||||
@@ -45,3 +45,23 @@ export const CAPACITY_META = {
|
||||
// search: temporarily hidden (feature not wired yet)
|
||||
reasoning: { icon: "neurology", label: "Reasoning", desc: "Supports reasoning / thinking", color: "text-amber-500" },
|
||||
};
|
||||
|
||||
// Realtime STT transport markers accepted on custom models — single source of
|
||||
// truth across layers: the API whitelist (src/app/api/models/custom/route.js
|
||||
// sanitizeTransport) and the dashboard transport select
|
||||
// (providers/[id]/AddCustomModelModal) both import this map, so one new row
|
||||
// here makes a realtime engine dispatch case (open-sse/handlers/sttCore.js)
|
||||
// selectable and validated end-to-end. Keys must mirror a sttCore case.
|
||||
export const STT_TRANSPORT_META = {
|
||||
"gemini-live": {
|
||||
label: "Gemini Live (realtime WebSocket)",
|
||||
desc: "Streams audio over bidiGenerateContent and returns incremental transcription segments",
|
||||
},
|
||||
};
|
||||
|
||||
export const STT_TRANSPORTS = Object.freeze(Object.keys(STT_TRANSPORT_META));
|
||||
|
||||
export function isSttTransport(transport) {
|
||||
if (typeof transport !== "string") return false;
|
||||
return Object.prototype.hasOwnProperty.call(STT_TRANSPORT_META, transport.trim());
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ import {
|
||||
extractApiKey, isValidApiKey,
|
||||
getProviderCredentials, markAccountUnavailable,
|
||||
} from "../services/auth.js";
|
||||
import { getSettings } from "@/lib/localDb";
|
||||
import { getSettings, getCustomModels } from "@/lib/localDb";
|
||||
import { getModelInfo } from "../services/model.js";
|
||||
import { handleSttCore } from "open-sse/handlers/sttCore.js";
|
||||
import { errorResponse, unavailableResponse } from "open-sse/utils/error.js";
|
||||
@@ -17,6 +17,23 @@ const CREDENTIALED_PROVIDERS = new Set(
|
||||
.map(([id]) => id)
|
||||
);
|
||||
|
||||
// Custom-model transport marker: models registered through
|
||||
// /api/models/custom may pin a specialized STT transport (e.g.
|
||||
// "gemini-live"). The engine dispatches on the marker itself, so the app
|
||||
// layer only resolves it — same getModelInfo-style provider+model pairing,
|
||||
// restricted to type "stt" records.
|
||||
async function resolveCustomModelTransport(provider, model) {
|
||||
try {
|
||||
const customModels = await getCustomModels();
|
||||
const hit = customModels.find((c) => c && c.type === "stt"
|
||||
&& c.providerAlias === provider && c.id === model
|
||||
&& typeof c.transport === "string" && c.transport.trim());
|
||||
return hit ? hit.transport.trim() : null;
|
||||
} catch {
|
||||
return null; // DB unreadable → built-in registry marker still applies
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleStt(request) {
|
||||
let formData;
|
||||
try {
|
||||
@@ -45,9 +62,11 @@ export async function handleStt(request) {
|
||||
const { provider, model } = modelInfo;
|
||||
log.info("ROUTING", `Provider: ${provider}, Model: ${model}`);
|
||||
|
||||
const modelTransport = await resolveCustomModelTransport(provider, model);
|
||||
|
||||
// noAuth providers
|
||||
if (!CREDENTIALED_PROVIDERS.has(provider)) {
|
||||
const result = await handleSttCore({ provider, model, formData, sttConfig: AI_PROVIDERS[provider]?.sttConfig });
|
||||
const result = await handleSttCore({ provider, model, formData, sttConfig: AI_PROVIDERS[provider]?.sttConfig, transport: modelTransport });
|
||||
if (result.success) return result.response;
|
||||
return errorResponse(result.status || HTTP_STATUS.BAD_GATEWAY, result.error || "STT failed");
|
||||
}
|
||||
@@ -72,7 +91,7 @@ export async function handleStt(request) {
|
||||
|
||||
log.info("AUTH", `\x1b[32mUsing ${provider} account: ${credentials.connectionName}\x1b[0m`);
|
||||
|
||||
const result = await handleSttCore({ provider, model, formData, credentials, sttConfig: AI_PROVIDERS[provider]?.sttConfig });
|
||||
const result = await handleSttCore({ provider, model, formData, credentials, sttConfig: AI_PROVIDERS[provider]?.sttConfig, transport: modelTransport });
|
||||
|
||||
if (result.success) return result.response;
|
||||
|
||||
|
||||
534
tests/unit/gemini-live-stt.test.js
Normal file
534
tests/unit/gemini-live-stt.test.js
Normal file
@@ -0,0 +1,534 @@
|
||||
// Gemini Live (realtime bidi) STT transport contract.
|
||||
//
|
||||
// Black-box tests against open-sse/handlers/sttCore.js. Wire observables only:
|
||||
// - transport marker drives dispatch (caller param / registry entry), never a
|
||||
// hardcoded model id;
|
||||
// - session opens with a setup frame declaring the model; audio rides
|
||||
// realtimeInput frames only AFTER the server's setup-complete ack;
|
||||
// - inputTranscription deltas accumulate into {text}; verbose_json adds
|
||||
// segments {id,text} with NO timing keys (protocol carries none);
|
||||
// - error frame → gateway error envelope (any 4xx/5xx, shape only);
|
||||
// - system_instruction / prompt override setup instruction (substring);
|
||||
// - client-supplied setup_timeout_ms (tiny) bounds the wait (error occurs);
|
||||
// - response_format never reaches the session setup;
|
||||
// - custom-model transport persists via POST /api/models/custom (whitelist)
|
||||
// with unknown values silently dropped;
|
||||
// - the persisted custom transport is resolved by the app layer (stt.js)
|
||||
// and reaches engine dispatch end-to-end (handleStt → WS, not REST).
|
||||
//
|
||||
// NOT pinned (unstated or implementation-only): goAway/reconnect semantics,
|
||||
// timeout clamp ceilings, specific status codes, byte-exact WS URLs (only
|
||||
// wss:// + bidiGenerateContent + model-id substrings), exact frame JSON paths.
|
||||
import { describe, it, expect, afterEach, vi, beforeEach } from "vitest";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import { handleSttCore } from "open-sse/handlers/sttCore.js";
|
||||
import { PROVIDER_MODELS } from "open-sse/config/providerModels.js";
|
||||
|
||||
// ── fixtures ──────────────────────────────────────────────────────────────
|
||||
|
||||
const STTCFG = {
|
||||
baseUrl: "https://generativelanguage.googleapis.com/v1beta/models",
|
||||
authType: "apikey",
|
||||
authHeader: "key",
|
||||
format: "gemini-stt",
|
||||
};
|
||||
const CRED = { apiKey: "AIza-TEST" };
|
||||
|
||||
const LIVE_ID = "probe-live-capability-1";
|
||||
|
||||
function mkFile() {
|
||||
return new File([new Uint8Array([1, 2, 3, 4])], "a.wav", { type: "audio/wav" });
|
||||
}
|
||||
|
||||
function mkFormData(extra = {}) {
|
||||
const fd = new FormData();
|
||||
fd.set("file", mkFile());
|
||||
for (const [k, v] of Object.entries(extra)) fd.set(k, v);
|
||||
return fd;
|
||||
}
|
||||
|
||||
// ── fake WebSocket ────────────────────────────────────────────────────────
|
||||
|
||||
class FakeWS {
|
||||
static instances = [];
|
||||
static CONNECTING = 0;
|
||||
static OPEN = 1;
|
||||
static CLOSING = 2;
|
||||
static CLOSED = 3;
|
||||
|
||||
constructor(url) {
|
||||
this.url = url;
|
||||
this.sent = []; // JSON.parsed frames, in send order
|
||||
this.readyState = FakeWS.CONNECTING;
|
||||
this.closed = false;
|
||||
this.closeCalls = []; // {code, reason} recordings
|
||||
this._listeners = {};
|
||||
FakeWS.instances.push(this);
|
||||
queueMicrotask(() => {
|
||||
if (this.closed) return;
|
||||
this.readyState = FakeWS.OPEN;
|
||||
if (typeof this.onopen === "function") this.onopen({});
|
||||
(this._listeners.open || []).forEach((f) => f({}));
|
||||
});
|
||||
}
|
||||
|
||||
addEventListener(type, fn) {
|
||||
(this._listeners[type] = this._listeners[type] || []).push(fn);
|
||||
}
|
||||
|
||||
send(data) {
|
||||
this.sent.push(JSON.parse(data));
|
||||
}
|
||||
|
||||
// Fire a server frame through both supported binding styles.
|
||||
emit(obj) {
|
||||
const ev = { data: JSON.stringify(obj) };
|
||||
if (typeof this.onmessage === "function") this.onmessage(ev);
|
||||
(this._listeners.message || []).forEach((f) => f(ev));
|
||||
}
|
||||
|
||||
// Fire a server-initiated close through both supported binding styles.
|
||||
// Distinct from close(), which only records the client-side shutdown.
|
||||
emitClose(code = 1000) {
|
||||
this.closed = true;
|
||||
this.readyState = FakeWS.CLOSED;
|
||||
const ev = { code, reason: "" };
|
||||
if (typeof this.onclose === "function") this.onclose(ev);
|
||||
(this._listeners.close || []).forEach((f) => f(ev));
|
||||
}
|
||||
|
||||
close(code, reason) {
|
||||
this.closed = true;
|
||||
this.readyState = FakeWS.CLOSED;
|
||||
this.closeCalls.push({ code: code ?? 1000, reason: reason ?? "" });
|
||||
}
|
||||
}
|
||||
|
||||
function stubWs() {
|
||||
vi.stubGlobal("WebSocket", FakeWS);
|
||||
}
|
||||
|
||||
// Fetch spy that records calls; handler defaults to "REST must not happen".
|
||||
function stubFetch(handler = () => { throw new Error("REST must not be used for live transport"); }) {
|
||||
const calls = [];
|
||||
vi.stubGlobal("fetch", async (url, opts) => {
|
||||
calls.push(String(url && url.url ? url.url : url));
|
||||
return handler(url, opts);
|
||||
});
|
||||
return calls;
|
||||
}
|
||||
|
||||
// Server drives a completed session: setup ack → transcription deltas → turn done.
|
||||
function serverScript(instance, texts) {
|
||||
instance.emit({ serverContent: { setupComplete: true } });
|
||||
for (const t of texts) instance.emit({ serverContent: { inputTranscription: { text: t } } });
|
||||
instance.emit({ serverContent: { turnComplete: true } });
|
||||
}
|
||||
|
||||
async function liveSession({ model = LIVE_ID, formData = mkFormData(), transport = "gemini-live" } = {}) {
|
||||
stubWs();
|
||||
const fetchCalls = stubFetch();
|
||||
const pending = handleSttCore({ provider: "gemini", model, formData, credentials: CRED, sttConfig: STTCFG, transport });
|
||||
await vi.waitFor(() => expect(FakeWS.instances.length).toBe(1));
|
||||
const ws = FakeWS.instances[0];
|
||||
await vi.waitFor(() => expect(ws.sent.length).toBeGreaterThanOrEqual(1)); // setup frame sent
|
||||
return { pending, ws, fetchCalls };
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
FakeWS.instances.length = 0;
|
||||
});
|
||||
|
||||
// ── S1/S2/S3: transport marker dispatch, text envelope, no REST ───────────
|
||||
|
||||
describe("Live transport dispatch via caller marker", () => {
|
||||
it("T3: transport 'gemini-live' opens a WebSocket, never REST; text = accumulated deltas", async () => {
|
||||
// REST-fallback contrast (folded from T1): a live-capability id with no
|
||||
// transport marker falls to REST and fails cleanly — the live path is opt-in.
|
||||
stubFetch(() => ({
|
||||
ok: false,
|
||||
status: 400,
|
||||
text: async () => JSON.stringify({ error: { message: "live models require the streaming endpoint" } }),
|
||||
}));
|
||||
const restResult = await handleSttCore({
|
||||
provider: "gemini",
|
||||
model: LIVE_ID,
|
||||
formData: mkFormData(),
|
||||
credentials: CRED,
|
||||
sttConfig: STTCFG,
|
||||
});
|
||||
expect(restResult.success).toBe(false);
|
||||
|
||||
// Caller-marker dispatch: explicit transport "gemini-live" opens the WS,
|
||||
// never REST; text = accumulated inputTranscription deltas.
|
||||
const { pending, ws, fetchCalls } = await liveSession();
|
||||
serverScript(ws, ["hello ", "world"]);
|
||||
const result = await pending;
|
||||
expect(result.success).toBe(true);
|
||||
await expect(result.response.json()).resolves.toEqual({ text: "hello world" });
|
||||
expect(fetchCalls).toHaveLength(0);
|
||||
|
||||
// Registry-marker dispatch: the live entry's transport field alone — no
|
||||
// caller transport param — routes to the WS path; id derived from the
|
||||
// registry, never a literal.
|
||||
const regId = (PROVIDER_MODELS.gemini || []).find(
|
||||
(m) => m && m.kind === "stt" && m.transport === "gemini-live",
|
||||
)?.id;
|
||||
FakeWS.instances.length = 0;
|
||||
stubWs();
|
||||
const regFetchCalls = stubFetch();
|
||||
const regPending = handleSttCore({
|
||||
provider: "gemini",
|
||||
model: regId,
|
||||
formData: mkFormData(),
|
||||
credentials: CRED,
|
||||
sttConfig: STTCFG,
|
||||
});
|
||||
await vi.waitFor(() => expect(FakeWS.instances.length).toBe(1));
|
||||
const regWs = FakeWS.instances[0];
|
||||
await vi.waitFor(() => expect(regWs.sent.length).toBeGreaterThanOrEqual(1));
|
||||
serverScript(regWs, ["reg ", "live"]);
|
||||
const regResult = await regPending;
|
||||
expect(regResult.success).toBe(true);
|
||||
await expect(regResult.response.json()).resolves.toEqual({ text: "reg live" });
|
||||
expect(regFetchCalls).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("T4: setup frame first (carries model id); audio only in realtimeInput at index >=1; WS URL is the bidi endpoint", async () => {
|
||||
const { pending, ws, fetchCalls } = await liveSession();
|
||||
const setup = ws.sent[0];
|
||||
expect(setup.setup).toBeTruthy();
|
||||
expect(JSON.stringify(setup.setup)).toContain(LIVE_ID);
|
||||
expect(setup.realtimeInput).toBeUndefined();
|
||||
|
||||
serverScript(ws, ["x"]);
|
||||
const result = await pending;
|
||||
expect(result.success).toBe(true);
|
||||
expect(fetchCalls).toHaveLength(0);
|
||||
|
||||
const audioIdx = ws.sent.findIndex((f) => f.realtimeInput);
|
||||
expect(audioIdx).toBeGreaterThanOrEqual(1);
|
||||
const media = ws.sent[audioIdx].realtimeInput.mediaChunks;
|
||||
expect(Array.isArray(media)).toBe(true);
|
||||
expect(typeof media[0].data).toBe("string");
|
||||
expect(media[0].data.length).toBeGreaterThan(0);
|
||||
expect(Buffer.from(media[0].data, "base64").length).toBeGreaterThan(0);
|
||||
|
||||
expect(ws.url).toContain("wss://");
|
||||
expect(ws.url).toContain("bidiGenerateContent");
|
||||
expect(ws.url).toContain(LIVE_ID);
|
||||
|
||||
// REST contrast (folded from T2): an ordinary gemini model still transcribes
|
||||
// over REST generateContent — the live path is opt-in, never the default.
|
||||
stubFetch(() => ({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({ candidates: [{ content: { parts: [{ text: "hello rest" }] } }] }),
|
||||
text: async () => "",
|
||||
}));
|
||||
const restResult = await handleSttCore({
|
||||
provider: "gemini",
|
||||
model: "gemini-2.0-flash",
|
||||
formData: mkFormData(),
|
||||
credentials: CRED,
|
||||
sttConfig: STTCFG,
|
||||
});
|
||||
expect(restResult.success).toBe(true);
|
||||
await expect(restResult.response.json()).resolves.toEqual({ text: "hello rest" });
|
||||
});
|
||||
|
||||
it("T5: server error frame yields the gateway error envelope (any 4xx/5xx, no text pin)", async () => {
|
||||
const { pending, ws } = await liveSession();
|
||||
ws.emit({ error: { code: "X", message: "Y" } });
|
||||
const result = await pending;
|
||||
expect(result.success).toBe(false);
|
||||
expect(typeof result.status).toBe("number");
|
||||
expect(result.status).toBeGreaterThanOrEqual(400);
|
||||
expect(result.status).toBeLessThanOrEqual(599);
|
||||
});
|
||||
});
|
||||
|
||||
// ── S7: client knobs (instruction overrides ride the setup frame) ─────────
|
||||
|
||||
describe("Setup frame knobs", () => {
|
||||
it("T6: client prompt overrides the setup instruction", async () => {
|
||||
const { pending, ws } = await liveSession({ formData: mkFormData({ prompt: "Say it back" }) });
|
||||
const setupJson = JSON.stringify(ws.sent[0]);
|
||||
expect(setupJson).toContain("Say it back");
|
||||
serverScript(ws, ["ok"]);
|
||||
const result = await pending;
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("T7: system_instruction override appears in the setup frame", async () => {
|
||||
const { pending, ws } = await liveSession({ formData: mkFormData({ system_instruction: "TRANSCRIBE-VERBATIM-OVERRIDE-42" }) });
|
||||
const setupJson = JSON.stringify(ws.sent[0]);
|
||||
expect(setupJson).toContain("TRANSCRIBE-VERBATIM-OVERRIDE-42");
|
||||
serverScript(ws, ["ok"]);
|
||||
const result = await pending;
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("T8: response_format is client-only and never reaches the session setup", async () => {
|
||||
const { pending, ws } = await liveSession({ formData: mkFormData({ response_format: "verbose_json" }) });
|
||||
expect(JSON.stringify(ws.sent[0])).not.toContain("response_format");
|
||||
serverScript(ws, ["a", "b"]);
|
||||
const result = await pending;
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("T9: tiny setup_timeout_ms with no server ack errors out within the bound", async () => {
|
||||
const { pending } = await liveSession({ formData: mkFormData({ setup_timeout_ms: "5" }) });
|
||||
// deliberately emit nothing — the client knob must end the wait
|
||||
const result = await pending;
|
||||
expect(result.success).toBe(false);
|
||||
expect(typeof result.status).toBe("number");
|
||||
expect(result.status).toBeGreaterThanOrEqual(400);
|
||||
expect(result.status).toBeLessThanOrEqual(599);
|
||||
});
|
||||
});
|
||||
|
||||
// ── S8: verbose_json shaping ──────────────────────────────────────────────
|
||||
|
||||
describe("Response shaping", () => {
|
||||
it("T10: verbose_json adds {id,text} segments in arrival order with NO timing fields", async () => {
|
||||
const { pending, ws } = await liveSession({ formData: mkFormData({ response_format: "verbose_json" }) });
|
||||
serverScript(ws, ["hello ", "world"]);
|
||||
const result = await pending;
|
||||
const body = await result.response.json();
|
||||
expect(body.text).toBe("hello world");
|
||||
expect(Array.isArray(body.segments)).toBe(true);
|
||||
expect(body.segments).toHaveLength(2);
|
||||
const [s0, s1] = body.segments;
|
||||
expect(s0.id).toBe(0);
|
||||
expect(s1.id).toBe(1);
|
||||
for (const seg of body.segments) {
|
||||
expect(Object.keys(seg)).toContain("id");
|
||||
expect(Object.keys(seg)).toContain("text");
|
||||
expect("start" in seg).toBe(false);
|
||||
expect("end" in seg).toBe(false);
|
||||
expect("duration" in seg).toBe(false);
|
||||
}
|
||||
expect(s0.text).toBe("hello ");
|
||||
expect(s1.text).toBe("world");
|
||||
expect("duration" in body).toBe(false);
|
||||
});
|
||||
|
||||
it("T11: default format carries text only, no segments", async () => {
|
||||
const { pending, ws } = await liveSession();
|
||||
serverScript(ws, ["one", "two"]);
|
||||
const result = await pending;
|
||||
const body = await result.response.json();
|
||||
expect(Object.keys(body)).toContain("text");
|
||||
expect("segments" in body).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── S2a: registry marks the live family (data, not code) ─────────────────
|
||||
|
||||
describe("Registry family marking", () => {
|
||||
it("T12: gemini stt catalog includes a live-transport entry advertising the lifecycle params", () => {
|
||||
const live = (PROVIDER_MODELS.gemini || []).find(
|
||||
(m) => m && m.kind === "stt" && m.transport === "gemini-live",
|
||||
);
|
||||
expect(live).toBeTruthy();
|
||||
expect(live.id).toBeTruthy();
|
||||
expect(Array.isArray(live.params)).toBe(true);
|
||||
for (const p of ["language", "prompt", "system_instruction", "setup_timeout_ms", "turn_timeout_ms"]) {
|
||||
expect(live.params).toContain(p);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── S2b: custom-model transport persistence (route level) ─────────────────
|
||||
|
||||
describe("Custom-model transport persistence via POST /api/models/custom", () => {
|
||||
let tempDir;
|
||||
const originalDataDir = process.env.DATA_DIR;
|
||||
|
||||
beforeEach(() => {
|
||||
// paths.js freezes DATA_DIR at module load — re-evaluate the db chain per test
|
||||
vi.resetModules();
|
||||
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "9router-gemini-live-"));
|
||||
process.env.DATA_DIR = tempDir;
|
||||
delete global._dbAdapter;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
try { global._dbAdapter?.instance?.close?.(); } catch { /* already closed */ }
|
||||
delete global._dbAdapter;
|
||||
if (tempDir) fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
if (originalDataDir === undefined) delete process.env.DATA_DIR;
|
||||
else process.env.DATA_DIR = originalDataDir;
|
||||
});
|
||||
|
||||
async function postCustom(payload) {
|
||||
const { POST } = await import("@/app/api/models/custom/route.js");
|
||||
const res = await POST({ json: async () => payload });
|
||||
return res.json();
|
||||
}
|
||||
|
||||
async function customRows() {
|
||||
const { getCustomModels } = await import("@/lib/db/repos/aliasRepo.js");
|
||||
return getCustomModels();
|
||||
}
|
||||
|
||||
it("T13: whitelisted transport persists on the saved row", { timeout: 30000 }, async () => {
|
||||
const body = await postCustom({
|
||||
providerAlias: "gemini",
|
||||
id: "probe-custom-capability-9",
|
||||
type: "stt",
|
||||
transport: "gemini-live",
|
||||
});
|
||||
expect(body.success).toBe(true);
|
||||
const row = (await customRows()).find((m) => m && m.providerAlias === "gemini" && m.id === "probe-custom-capability-9");
|
||||
expect(row).toBeTruthy();
|
||||
expect(row.type).toBe("stt");
|
||||
expect(row.transport).toBe("gemini-live");
|
||||
});
|
||||
|
||||
it("T14: unknown transport is silently dropped — prior whitelisted transport survives re-save", { timeout: 30000 }, async () => {
|
||||
const first = await postCustom({
|
||||
providerAlias: "gemini",
|
||||
id: "probe-custom-capability-10",
|
||||
type: "stt",
|
||||
transport: "gemini-live",
|
||||
});
|
||||
expect(first.success).toBe(true);
|
||||
// Re-saving the same model with an unknown transport must not clobber
|
||||
// the persisted marker: silent-drop keeps the stored value (merge keeps
|
||||
// omitted fields, per addCustomModel).
|
||||
const second = await postCustom({
|
||||
providerAlias: "gemini",
|
||||
id: "probe-custom-capability-10",
|
||||
type: "stt",
|
||||
transport: "nope",
|
||||
});
|
||||
expect(second.success).toBe(true);
|
||||
const row = (await customRows()).find((m) => m && m.providerAlias === "gemini" && m.id === "probe-custom-capability-10");
|
||||
expect(row).toBeTruthy();
|
||||
expect(row.transport).toBe("gemini-live");
|
||||
});
|
||||
});
|
||||
|
||||
// ── S2c (T15): app-layer custom-transport resolution, end-to-end ─────────
|
||||
//
|
||||
// Gate C M1 closure. Only handleStt (src/sse/handlers/stt.js) maps a
|
||||
// persisted custom-model transport onto the handleSttCore dispatch; T13/T14
|
||||
// stop at repo persistence. Fake model id is absent from the registry, so a
|
||||
// WebSocket opening here is observable proof the caller-supplied transport
|
||||
// marker was resolved and passed — deleting that resolution fails T15.
|
||||
|
||||
describe("App-layer custom transport resolution (stt.js)", () => {
|
||||
it("T15: persisted custom gemini-live transport reaches WS dispatch through real handleStt", async () => {
|
||||
const LOCALDB = "@/lib/localDb";
|
||||
const AUTH = "../../src/sse/services/auth.js";
|
||||
try {
|
||||
vi.resetModules();
|
||||
vi.doMock(LOCALDB, () => ({
|
||||
getSettings: async () => ({ requireApiKey: false }),
|
||||
getCustomModels: async () => ([{
|
||||
providerAlias: "gemini", id: "probe-sttjs-1", type: "stt", transport: "gemini-live",
|
||||
}]),
|
||||
}));
|
||||
vi.doMock(AUTH, () => ({
|
||||
extractApiKey: () => null,
|
||||
isValidApiKey: async () => true,
|
||||
getProviderCredentials: async () => ({
|
||||
apiKey: "AIza-TEST", connectionId: "c1", connectionName: "t", providerSpecificData: {},
|
||||
}),
|
||||
markAccountUnavailable: async () => ({ shouldFallback: false }),
|
||||
}));
|
||||
// getModelInfo stays REAL: "gemini/..." is a reserved-prefix passthrough,
|
||||
// so the parse→route hop in the chain is exercised, not stubbed.
|
||||
const { handleStt } = await import("../../src/sse/handlers/stt.js");
|
||||
|
||||
stubWs();
|
||||
const fetchCalls = stubFetch(); // default handler throws: REST must not happen
|
||||
const fd = mkFormData();
|
||||
fd.set("model", "gemini/probe-sttjs-1");
|
||||
|
||||
const pending = handleStt({ formData: async () => fd });
|
||||
await vi.waitFor(() => expect(FakeWS.instances.length).toBe(1));
|
||||
const ws = FakeWS.instances[0];
|
||||
await vi.waitFor(() => expect(ws.sent.length).toBeGreaterThanOrEqual(1)); // setup first
|
||||
serverScript(ws, ["hello ", "world"]);
|
||||
const res = await pending;
|
||||
expect(res.status).toBe(200);
|
||||
const body = await res.json();
|
||||
expect(body.text).toBe("hello world");
|
||||
expect(fetchCalls).toHaveLength(0);
|
||||
} finally {
|
||||
vi.doUnmock(LOCALDB);
|
||||
vi.doUnmock(AUTH);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── S5/S6/S9: lifecycle gates (setup ack, turn timeout, turn completion) ──
|
||||
|
||||
// Setup/turn gating, graceful close, and transcript accumulation hygiene:
|
||||
// padding-only frames are dropped, and a whitespace-only run is a failure
|
||||
// rather than a blank success.
|
||||
describe("Lifecycle gates", () => {
|
||||
it("T16: audio streaming waits for the server setup-complete reply", async () => {
|
||||
const { pending, ws } = await liveSession();
|
||||
// deliberately do NOT emit setupComplete
|
||||
await new Promise((r) => setTimeout(r, 80));
|
||||
const audioFrames = ws.sent.filter((f) => f.realtimeInput);
|
||||
expect(audioFrames).toHaveLength(0);
|
||||
// clean up: let the pending promise settle so afterEach unstub works cleanly
|
||||
ws.emit({ serverContent: { setupComplete: true } });
|
||||
ws.emit({ serverContent: { turnComplete: true } });
|
||||
await pending;
|
||||
});
|
||||
|
||||
it("T17: tiny turn_timeout_ms with setup-complete but no turn-complete errors out", async () => {
|
||||
const { pending, ws } = await liveSession({ formData: mkFormData({ turn_timeout_ms: "5" }) });
|
||||
ws.emit({ serverContent: { setupComplete: true } });
|
||||
// deliberately do NOT emit turnComplete
|
||||
const result = await pending;
|
||||
expect(result.success).toBe(false);
|
||||
expect(typeof result.status).toBe("number");
|
||||
expect(result.status).toBeGreaterThanOrEqual(400);
|
||||
expect(result.status).toBeLessThanOrEqual(599);
|
||||
});
|
||||
|
||||
it("T18: server turnComplete closes the WebSocket gracefully", async () => {
|
||||
const { pending, ws } = await liveSession();
|
||||
serverScript(ws, ["done"]);
|
||||
await pending;
|
||||
expect(ws.closeCalls.length).toBeGreaterThanOrEqual(1);
|
||||
expect(ws.closeCalls[0].code).toBe(1000);
|
||||
});
|
||||
|
||||
it("T19: whitespace-only transcription frames are not appended to the transcript", async () => {
|
||||
const { pending, ws } = await liveSession();
|
||||
ws.emit({ serverContent: { setupComplete: true } });
|
||||
// a padding-only frame must contribute nothing to the transcript
|
||||
ws.emit({ serverContent: { inputTranscription: { text: " " } } });
|
||||
ws.emit({ serverContent: { inputTranscription: { text: "done" } } });
|
||||
ws.emit({ serverContent: { turnComplete: true } });
|
||||
const result = await pending;
|
||||
const body = await result.response.json();
|
||||
expect(body.text).toBe("done");
|
||||
});
|
||||
|
||||
it("T20: a run that receives only whitespace frames errors instead of returning a blank transcript", async () => {
|
||||
const { pending, ws } = await liveSession();
|
||||
ws.emit({ serverContent: { setupComplete: true } });
|
||||
ws.emit({ serverContent: { inputTranscription: { text: " " } } });
|
||||
// server closes before any real transcript arrived: a partial success would
|
||||
// hand the client a whitespace-only transcript
|
||||
ws.emitClose(1000);
|
||||
const result = await pending;
|
||||
expect(result.success).toBe(false);
|
||||
expect(typeof result.status).toBe("number");
|
||||
expect(result.status).toBeGreaterThanOrEqual(400);
|
||||
expect(result.status).toBeLessThanOrEqual(599);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user