feat: add STT support, Gemini TTS, and expand usage tracking
- Speech-to-Text: full pipeline with sttCore handler, /v1/audio/transcriptions endpoint, sttConfig for OpenAI, Gemini, Groq, Deepgram, AssemblyAI, HuggingFace, NVIDIA Parakeet; new 9router-stt skill - Gemini TTS: add gemini provider with 30 prebuilt voices and TTS_PROVIDER_CONFIG - Usage: implement GLM (intl/cn) and MiniMax (intl/cn) quota fetchers; refactor Gemini CLI usage to use retrieveUserQuota with per-model buckets - Disabled models: lowdb-backed disabledModelsDb + /api/models/disabled route - Header search: reusable Zustand store (headerSearchStore) wired into Header - CLI tools: add Claude Cowork tool card and cowork-settings API - Providers: introduce mediaPriority sorting in getProvidersByKind, add Kimi K2.6, reorder hermes, drop qwen STT kind - UI: expand media-providers/[kind]/[id] page (+314), enhance OAuthModal, ModelSelectModal, ProviderTopology, ProxyPools, ProviderLimits - Assets: refresh provider PNGs (alicode, byteplus, cloudflare-ai, nvidia, ollama, vertex, volcengine-ark) and add aws-polly, fal-ai, jina-ai, recraft, runwayml, stability-ai, topaz, black-forest-labs
This commit is contained in:
decolua
194
open-sse/handlers/sttCore.js
Normal file
194
open-sse/handlers/sttCore.js
Normal file
@@ -0,0 +1,194 @@
|
||||
import { Buffer } from "node:buffer";
|
||||
import { createErrorResult } from "../utils/error.js";
|
||||
import { HTTP_STATUS } from "../config/runtimeConfig.js";
|
||||
import { AI_PROVIDERS } from "../../src/shared/constants/providers.js";
|
||||
|
||||
// Build auth headers from sttConfig + token
|
||||
function buildAuthHeaders(cfg, token) {
|
||||
if (!token) return {};
|
||||
switch (cfg.authHeader) {
|
||||
case "bearer": return { "Authorization": `Bearer ${token}` };
|
||||
case "token": return { "Authorization": `Token ${token}` };
|
||||
case "x-api-key": return { "x-api-key": token };
|
||||
case "key": return { "Authorization": `Key ${token}` };
|
||||
default: return { "Authorization": `Bearer ${token}` };
|
||||
}
|
||||
}
|
||||
|
||||
// Map browser file MIME / ext → audio MIME for binary formats (deepgram/HF)
|
||||
function resolveAudioContentType(file) {
|
||||
const t = (file.type || "").toLowerCase();
|
||||
if (t.startsWith("audio/")) return t;
|
||||
const name = typeof file.name === "string" ? file.name.toLowerCase() : "";
|
||||
const ext = name.includes(".") ? name.split(".").pop() : "";
|
||||
const map = { mp3: "audio/mpeg", mp4: "audio/mp4", m4a: "audio/mp4", wav: "audio/wav", ogg: "audio/ogg", flac: "audio/flac", webm: "audio/webm", aac: "audio/aac", opus: "audio/opus" };
|
||||
return map[ext] || "application/octet-stream";
|
||||
}
|
||||
|
||||
async function upstreamError(res) {
|
||||
let txt = "";
|
||||
try { txt = await res.text(); } catch {}
|
||||
let msg = txt || `Upstream error (${res.status})`;
|
||||
try { const j = JSON.parse(txt); msg = j?.error?.message || j?.error || j?.message || msg; } catch {}
|
||||
return createErrorResult(res.status, typeof msg === "string" ? msg : JSON.stringify(msg));
|
||||
}
|
||||
|
||||
// Deepgram: raw binary POST + model query param
|
||||
async function transcribeDeepgram(cfg, file, model, token, formData) {
|
||||
const url = new URL(cfg.baseUrl);
|
||||
url.searchParams.set("model", model);
|
||||
url.searchParams.set("smart_format", "true");
|
||||
url.searchParams.set("punctuate", "true");
|
||||
const lang = formData.get("language");
|
||||
if (typeof lang === "string" && lang.trim()) url.searchParams.set("language", lang.trim());
|
||||
else url.searchParams.set("detect_language", "true");
|
||||
|
||||
const buf = await file.arrayBuffer();
|
||||
const res = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: { ...buildAuthHeaders(cfg, token), "Content-Type": resolveAudioContentType(file) },
|
||||
body: buf,
|
||||
});
|
||||
if (!res.ok) return upstreamError(res);
|
||||
const data = await res.json();
|
||||
const text = data.results?.channels?.[0]?.alternatives?.[0]?.transcript ?? "";
|
||||
return jsonResponse({ text });
|
||||
}
|
||||
|
||||
// AssemblyAI: upload → submit → poll (max 120s)
|
||||
async function transcribeAssemblyAI(cfg, file, model, token) {
|
||||
const auth = buildAuthHeaders(cfg, token);
|
||||
const buf = await file.arrayBuffer();
|
||||
const up = await fetch("https://api.assemblyai.com/v2/upload", {
|
||||
method: "POST", headers: { ...auth, "Content-Type": "application/octet-stream" }, body: buf,
|
||||
});
|
||||
if (!up.ok) return upstreamError(up);
|
||||
const { upload_url } = await up.json();
|
||||
|
||||
const sub = await fetch(cfg.baseUrl, {
|
||||
method: "POST",
|
||||
headers: { ...auth, "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ audio_url: upload_url, speech_models: [model], language_detection: true }),
|
||||
});
|
||||
if (!sub.ok) return upstreamError(sub);
|
||||
const { id } = await sub.json();
|
||||
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < 120_000) {
|
||||
await new Promise((r) => setTimeout(r, 2000));
|
||||
const poll = await fetch(`${cfg.baseUrl}/${id}`, { headers: auth });
|
||||
if (!poll.ok) continue;
|
||||
const r = await poll.json();
|
||||
if (r.status === "completed") return jsonResponse({ text: r.text || "" });
|
||||
if (r.status === "error") return createErrorResult(500, r.error || "AssemblyAI failed");
|
||||
}
|
||||
return createErrorResult(504, "AssemblyAI timeout after 120s");
|
||||
}
|
||||
|
||||
// Nvidia NIM: multipart, normalize response
|
||||
async function transcribeNvidia(cfg, file, model, token) {
|
||||
const fd = new FormData();
|
||||
fd.append("file", file, file.name || "audio.wav");
|
||||
fd.append("model", model);
|
||||
const res = await fetch(cfg.baseUrl, { method: "POST", headers: buildAuthHeaders(cfg, token), body: fd });
|
||||
if (!res.ok) return upstreamError(res);
|
||||
const data = await res.json();
|
||||
return jsonResponse({ text: data.text || data.transcript || "" });
|
||||
}
|
||||
|
||||
// Gemini: generateContent with inline_data audio + transcription prompt
|
||||
async function transcribeGemini(cfg, file, model, token, formData) {
|
||||
const buf = await file.arrayBuffer();
|
||||
const b64 = Buffer.from(buf).toString("base64");
|
||||
const mime = resolveAudioContentType(file);
|
||||
const lang = formData.get("language");
|
||||
const userPrompt = formData.get("prompt");
|
||||
let promptText = userPrompt && typeof userPrompt === "string" && userPrompt.trim()
|
||||
? userPrompt.trim()
|
||||
: "Generate a transcript of the speech. Return only the transcribed text, no commentary.";
|
||||
if (typeof lang === "string" && lang.trim()) promptText += ` Language: ${lang.trim()}.`;
|
||||
|
||||
const url = `${cfg.baseUrl}/${model}:generateContent?key=${token}`;
|
||||
const res = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
contents: [{ parts: [{ text: promptText }, { inline_data: { mime_type: mime, data: b64 } }] }],
|
||||
}),
|
||||
});
|
||||
if (!res.ok) return upstreamError(res);
|
||||
const data = await res.json();
|
||||
const text = data?.candidates?.[0]?.content?.parts?.map((p) => p.text).filter(Boolean).join("") || "";
|
||||
return jsonResponse({ text });
|
||||
}
|
||||
|
||||
// HuggingFace: POST raw binary to {baseUrl}/{model_id}
|
||||
async function transcribeHuggingFace(cfg, file, model, token) {
|
||||
if (model.includes("..") || model.includes("//")) return createErrorResult(400, "Invalid model ID");
|
||||
const url = `${cfg.baseUrl.replace(/\/+$/, "")}/${model}`;
|
||||
const buf = await file.arrayBuffer();
|
||||
const res = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: { ...buildAuthHeaders(cfg, token), "Content-Type": resolveAudioContentType(file) },
|
||||
body: buf,
|
||||
});
|
||||
if (!res.ok) return upstreamError(res);
|
||||
const data = await res.json();
|
||||
return jsonResponse({ text: data.text || "" });
|
||||
}
|
||||
|
||||
// Default: OpenAI/Groq/Whisper-compatible multipart
|
||||
async function transcribeOpenAICompatible(cfg, file, model, token, formData) {
|
||||
const fd = new FormData();
|
||||
fd.append("file", file, file.name || "audio.wav");
|
||||
fd.append("model", model);
|
||||
for (const k of ["language", "prompt", "response_format", "temperature"]) {
|
||||
const v = formData.get(k);
|
||||
if (v !== null && v !== undefined && v !== "") fd.append(k, v);
|
||||
}
|
||||
const res = await fetch(cfg.baseUrl, { method: "POST", headers: buildAuthHeaders(cfg, token), body: fd });
|
||||
if (!res.ok) return upstreamError(res);
|
||||
const ct = res.headers.get("content-type") || "application/json";
|
||||
const txt = await res.text();
|
||||
return { success: true, response: new Response(txt, { status: 200, headers: { "Content-Type": ct, "Access-Control-Allow-Origin": "*" } }) };
|
||||
}
|
||||
|
||||
function jsonResponse(obj) {
|
||||
return {
|
||||
success: true,
|
||||
response: new Response(JSON.stringify(obj), {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json", "Access-Control-Allow-Origin": "*" },
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* STT core handler — dispatch by sttConfig.format.
|
||||
* @returns {Promise<{success, response, status?, error?}>}
|
||||
*/
|
||||
export async function handleSttCore({ provider, model, formData, credentials }) {
|
||||
const file = formData.get("file");
|
||||
if (!file) return createErrorResult(HTTP_STATUS.BAD_REQUEST, "Missing required field: file");
|
||||
|
||||
const cfg = AI_PROVIDERS[provider]?.sttConfig;
|
||||
if (!cfg) return createErrorResult(HTTP_STATUS.BAD_REQUEST, `Provider '${provider}' does not support STT`);
|
||||
|
||||
const token = cfg.authType === "none" ? null : (credentials?.apiKey || credentials?.accessToken);
|
||||
if (cfg.authType !== "none" && !token) {
|
||||
return createErrorResult(HTTP_STATUS.UNAUTHORIZED, `No credentials for STT provider: ${provider}`);
|
||||
}
|
||||
|
||||
try {
|
||||
switch (cfg.format) {
|
||||
case "deepgram": return await transcribeDeepgram(cfg, file, model, token, formData);
|
||||
case "assemblyai": return await transcribeAssemblyAI(cfg, file, model, token);
|
||||
case "nvidia-asr": return await transcribeNvidia(cfg, file, model, token);
|
||||
case "huggingface-asr": return await transcribeHuggingFace(cfg, file, model, token);
|
||||
case "gemini-stt": return await transcribeGemini(cfg, file, model, token, formData);
|
||||
default: return await transcribeOpenAICompatible(cfg, file, model, token, formData);
|
||||
}
|
||||
} catch (err) {
|
||||
return createErrorResult(HTTP_STATUS.BAD_GATEWAY, err.message || "STT request failed");
|
||||
}
|
||||
}
|
||||
@@ -48,16 +48,16 @@ function createTtsResponse(base64Audio, format, responseFormat) {
|
||||
*
|
||||
* @returns {Promise<{success, response, status?, error?}>}
|
||||
*/
|
||||
export async function handleTtsCore({ provider, model, input, credentials, responseFormat = "mp3" }) {
|
||||
export async function handleTtsCore({ provider, model, input, credentials, responseFormat = "mp3", language }) {
|
||||
if (!input?.trim()) {
|
||||
return createErrorResult(HTTP_STATUS.BAD_REQUEST, "Missing required field: input");
|
||||
}
|
||||
|
||||
try {
|
||||
// Special-case adapters (google-tts, edge-tts, local-device, elevenlabs, openai, openrouter)
|
||||
// Special-case adapters (google-tts, edge-tts, local-device, elevenlabs, openai, openrouter, gemini)
|
||||
const adapter = getTtsAdapter(provider);
|
||||
if (adapter) {
|
||||
const result = await adapter.synthesize(input.trim(), model, credentials, responseFormat);
|
||||
const result = await adapter.synthesize(input.trim(), model, credentials, responseFormat, { language });
|
||||
// Adapter may return a full {success, response} (legacy) or {base64, format}
|
||||
if (result.success !== undefined) return result;
|
||||
return createTtsResponse(result.base64, result.format, responseFormat);
|
||||
|
||||
117
open-sse/handlers/ttsProviders/gemini.js
Normal file
117
open-sse/handlers/ttsProviders/gemini.js
Normal file
@@ -0,0 +1,117 @@
|
||||
// Gemini TTS — generateContent with AUDIO modality returns PCM L16, wrap as WAV
|
||||
import { Buffer } from "node:buffer";
|
||||
|
||||
const DEFAULT_MODEL = "gemini-2.5-flash-preview-tts";
|
||||
const DEFAULT_VOICE = "Kore";
|
||||
const KNOWN_MODELS = ["gemini-2.5-flash-preview-tts", "gemini-2.5-pro-preview-tts"];
|
||||
|
||||
// Parse "model/voice" — if input doesn't match a known TTS model, treat it as voice with default model
|
||||
function parseGeminiModelVoice(input) {
|
||||
if (!input) return { modelId: DEFAULT_MODEL, voiceId: DEFAULT_VOICE };
|
||||
for (const id of KNOWN_MODELS) {
|
||||
if (input === id) return { modelId: id, voiceId: DEFAULT_VOICE };
|
||||
if (input.startsWith(`${id}/`)) return { modelId: id, voiceId: input.slice(id.length + 1) };
|
||||
}
|
||||
return { modelId: DEFAULT_MODEL, voiceId: input };
|
||||
}
|
||||
// Gemini returns PCM 16-bit signed mono @ 24kHz
|
||||
const SAMPLE_RATE = 24000;
|
||||
const CHANNELS = 1;
|
||||
const BITS_PER_SAMPLE = 16;
|
||||
|
||||
// Build WAV header for raw PCM payload
|
||||
function pcmToWav(pcmBuffer) {
|
||||
const dataSize = pcmBuffer.length;
|
||||
const byteRate = SAMPLE_RATE * CHANNELS * BITS_PER_SAMPLE / 8;
|
||||
const blockAlign = CHANNELS * BITS_PER_SAMPLE / 8;
|
||||
const header = Buffer.alloc(44);
|
||||
header.write("RIFF", 0);
|
||||
header.writeUInt32LE(36 + dataSize, 4);
|
||||
header.write("WAVE", 8);
|
||||
header.write("fmt ", 12);
|
||||
header.writeUInt32LE(16, 16);
|
||||
header.writeUInt16LE(1, 20);
|
||||
header.writeUInt16LE(CHANNELS, 22);
|
||||
header.writeUInt32LE(SAMPLE_RATE, 24);
|
||||
header.writeUInt32LE(byteRate, 28);
|
||||
header.writeUInt16LE(blockAlign, 32);
|
||||
header.writeUInt16LE(BITS_PER_SAMPLE, 34);
|
||||
header.write("data", 36);
|
||||
header.writeUInt32LE(dataSize, 40);
|
||||
return Buffer.concat([header, pcmBuffer]);
|
||||
}
|
||||
|
||||
// Build TTS prompt: add "Say [in {language}]:" prefix to force TTS mode
|
||||
function buildPrompt(text, language) {
|
||||
if (/:\s/.test(text)) return text; // user already provided style instruction
|
||||
return language ? `Say in ${language}: ${text}` : `Say: ${text}`;
|
||||
}
|
||||
|
||||
export default {
|
||||
async synthesize(text, model, credentials, _responseFormat, opts = {}) {
|
||||
if (!credentials?.apiKey) throw new Error("No Gemini API key configured");
|
||||
const { modelId, voiceId } = parseGeminiModelVoice(model);
|
||||
const url = `https://generativelanguage.googleapis.com/v1beta/models/${modelId}:generateContent?key=${credentials.apiKey}`;
|
||||
const res = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
contents: [{ parts: [{ text: buildPrompt(text, opts.language) }] }],
|
||||
generationConfig: {
|
||||
responseModalities: ["AUDIO"],
|
||||
speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: voiceId } } },
|
||||
},
|
||||
}),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const err = await res.json().catch(() => ({}));
|
||||
throw new Error(err?.error?.message || `Gemini TTS failed: ${res.status}`);
|
||||
}
|
||||
const data = await res.json();
|
||||
const b64 = data?.candidates?.[0]?.content?.parts?.find((p) => p.inlineData?.data)?.inlineData?.data;
|
||||
if (!b64) {
|
||||
const reason = data?.candidates?.[0]?.finishReason || data?.promptFeedback?.blockReason || "unknown";
|
||||
throw new Error(`Gemini TTS returned no audio (finishReason: ${reason}, voice: ${voiceId}, model: ${modelId})`);
|
||||
}
|
||||
const wav = pcmToWav(Buffer.from(b64, "base64"));
|
||||
return { base64: wav.toString("base64"), format: "wav" };
|
||||
},
|
||||
};
|
||||
|
||||
// Voice fetcher — return prebuilt voices (Gemini has no list API)
|
||||
const PREBUILT_VOICES = [
|
||||
{ id: "Zephyr", lang: "en", gender: "Female" },
|
||||
{ id: "Puck", lang: "en", gender: "Male" },
|
||||
{ id: "Charon", lang: "en", gender: "Male" },
|
||||
{ id: "Kore", lang: "en", gender: "Female" },
|
||||
{ id: "Fenrir", lang: "en", gender: "Male" },
|
||||
{ id: "Leda", lang: "en", gender: "Female" },
|
||||
{ id: "Orus", lang: "en", gender: "Male" },
|
||||
{ id: "Aoede", lang: "en", gender: "Female" },
|
||||
{ id: "Callirrhoe", lang: "en", gender: "Female" },
|
||||
{ id: "Autonoe", lang: "en", gender: "Female" },
|
||||
{ id: "Enceladus", lang: "en", gender: "Male" },
|
||||
{ id: "Iapetus", lang: "en", gender: "Male" },
|
||||
{ id: "Umbriel", lang: "en", gender: "Male" },
|
||||
{ id: "Algieba", lang: "en", gender: "Male" },
|
||||
{ id: "Despina", lang: "en", gender: "Female" },
|
||||
{ id: "Erinome", lang: "en", gender: "Female" },
|
||||
{ id: "Algenib", lang: "en", gender: "Male" },
|
||||
{ id: "Rasalgethi", lang: "en", gender: "Male" },
|
||||
{ id: "Laomedeia", lang: "en", gender: "Female" },
|
||||
{ id: "Achernar", lang: "en", gender: "Female" },
|
||||
{ id: "Alnilam", lang: "en", gender: "Male" },
|
||||
{ id: "Schedar", lang: "en", gender: "Male" },
|
||||
{ id: "Gacrux", lang: "en", gender: "Female" },
|
||||
{ id: "Pulcherrima", lang: "en", gender: "Female" },
|
||||
{ id: "Achird", lang: "en", gender: "Male" },
|
||||
{ id: "Zubenelgenubi", lang: "en", gender: "Male" },
|
||||
{ id: "Vindemiatrix", lang: "en", gender: "Female" },
|
||||
{ id: "Sadachbia", lang: "en", gender: "Male" },
|
||||
{ id: "Sadaltager", lang: "en", gender: "Male" },
|
||||
{ id: "Sulafat", lang: "en", gender: "Female" },
|
||||
];
|
||||
|
||||
export async function fetchGeminiVoices() {
|
||||
return PREBUILT_VOICES.map((v) => ({ voice_id: v.id, name: v.id, labels: { language: v.lang, gender: v.gender } }));
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import localDevice, { fetchLocalDeviceVoices } from "./localDevice.js";
|
||||
import elevenlabs, { fetchElevenLabsVoices } from "./elevenlabs.js";
|
||||
import openai from "./openai.js";
|
||||
import openrouter from "./openrouter.js";
|
||||
import gemini, { fetchGeminiVoices } from "./gemini.js";
|
||||
import { FORMAT_HANDLERS } from "./genericFormats.js";
|
||||
import { parseModelVoice } from "./_base.js";
|
||||
|
||||
@@ -16,6 +17,7 @@ const SPECIAL_ADAPTERS = {
|
||||
elevenlabs,
|
||||
openai,
|
||||
openrouter,
|
||||
gemini,
|
||||
};
|
||||
|
||||
export function getTtsAdapter(provider) {
|
||||
@@ -41,7 +43,8 @@ export const VOICE_FETCHERS = {
|
||||
"edge-tts": fetchEdgeTtsVoices,
|
||||
"local-device": fetchLocalDeviceVoices,
|
||||
elevenlabs: fetchElevenLabsVoices,
|
||||
gemini: fetchGeminiVoices,
|
||||
};
|
||||
|
||||
// Re-export for backward compat
|
||||
export { fetchEdgeTtsVoices, fetchLocalDeviceVoices, fetchElevenLabsVoices };
|
||||
export { fetchEdgeTtsVoices, fetchLocalDeviceVoices, fetchElevenLabsVoices, fetchGeminiVoices };
|
||||
|
||||
Reference in New Issue
Block a user