fix(tts): resolve Gemini TTS models from catalog

Resolve Gemini TTS models from shared TTS catalog and provider registry
with a safe fallback, fixing requests resolving to models/undefined when
ttsConfig.models is empty. Add gemini-3.1-flash-tts-preview to catalogs.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
nguyenha935
2026-06-26 10:31:22 +07:00
committed by decolua
parent 644bff4cdd
commit ce844899ed
4 changed files with 109 additions and 3 deletions

View File

@@ -96,10 +96,12 @@ export const TTS_MODELS_CONFIG = {
},
gemini: {
models: [
{ id: "gemini-3.1-flash-tts-preview", name: "Gemini 3.1 Flash TTS", type: "tts" },
{ id: "gemini-2.5-flash-preview-tts", name: "Gemini 2.5 Flash TTS", type: "tts" },
{ id: "gemini-2.5-pro-preview-tts", name: "Gemini 2.5 Pro TTS", type: "tts" },
],
voices: {
"gemini-3.1-flash-tts-preview": GEMINI_VOICES,
"gemini-2.5-flash-preview-tts": GEMINI_VOICES,
"gemini-2.5-pro-preview-tts": GEMINI_VOICES,
},

View File

@@ -1,11 +1,19 @@
// Gemini TTS — generateContent with AUDIO modality returns PCM L16, wrap as WAV
import { Buffer } from "node:buffer";
import { PROVIDER_MEDIA } from "../../providers/index.js";
import { PROVIDER_MEDIA, PROVIDER_MODELS } from "../../providers/index.js";
const TTS_CFG = PROVIDER_MEDIA["gemini"]?.ttsConfig || {};
const TTS_BASE = TTS_CFG.baseUrl;
const KNOWN_MODELS = (TTS_CFG.models || []).map((m) => m.id);
const DEFAULT_MODEL = KNOWN_MODELS[0];
const FALLBACK_MODEL = "gemini-3.1-flash-tts-preview";
const KNOWN_MODELS = [
...(TTS_CFG.models || []),
...(PROVIDER_MODELS["gemini-tts-models"] || []),
...(PROVIDER_MODELS.gemini || []).filter((m) => (m.kind || m.type) === "tts"),
]
.map((m) => m?.id)
.filter(Boolean)
.filter((id, index, list) => list.indexOf(id) === index);
const DEFAULT_MODEL = KNOWN_MODELS[0] || FALLBACK_MODEL;
const DEFAULT_VOICE = "Kore";
// Parse "model/voice" — if input doesn't match a known TTS model, treat it as voice with default model

View File

@@ -52,6 +52,7 @@ export default {
{ id: "gemini-2.5-flash", name: "Gemini 2.5 Flash", params: ["language","prompt"], kind: "stt" },
{ id: "gemini-2.5-flash-lite", name: "Gemini 2.5 Flash Lite (Cheapest)", params: ["language","prompt"], kind: "stt" },
{ id: "gemini-2.0-flash", name: "Gemini 2.0 Flash", params: ["language","prompt"], kind: "stt" },
{ id: "gemini-3.1-flash-tts-preview", name: "Gemini 3.1 Flash TTS", kind: "tts" },
{ id: "gemini-2.5-flash-preview-tts", name: "Gemini 2.5 Flash TTS", kind: "tts" },
{ id: "gemini-2.5-pro-preview-tts", name: "Gemini 2.5 Pro TTS", kind: "tts" },
{ id: "embedding-001", name: "Embedding 001", dimensions: 768, kind: "embedding" },

View File

@@ -0,0 +1,95 @@
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import { handleTtsCore } from "../../open-sse/handlers/ttsCore.js";
import { buildTtsProviderModels } from "../../open-sse/config/ttsModels.js";
const originalFetch = global.fetch;
function mockGeminiAudioResponse() {
global.fetch.mockResolvedValueOnce(
new Response(
JSON.stringify({
candidates: [
{
content: {
parts: [
{
inlineData: {
mimeType: "audio/pcm",
data: Buffer.from([0, 1, 2, 3]).toString("base64"),
},
},
],
},
},
],
}),
{ status: 200, headers: { "Content-Type": "application/json" } }
)
);
}
describe("Gemini TTS", () => {
beforeEach(() => {
global.fetch = vi.fn();
});
afterEach(() => {
global.fetch = originalFetch;
});
it("uses the default Gemini TTS model when only a voice is provided", async () => {
mockGeminiAudioResponse();
const result = await handleTtsCore({
provider: "gemini",
model: "Zephyr",
input: "Hello from Gemini",
credentials: { apiKey: "test-key" },
responseFormat: "json",
});
expect(result.success).toBe(true);
expect(global.fetch.mock.calls[0][0]).toBe(
"https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-tts-preview:generateContent?key=test-key"
);
const sent = JSON.parse(global.fetch.mock.calls[0][1].body);
expect(sent.generationConfig.speechConfig.voiceConfig.prebuiltVoiceConfig.voiceName).toBe("Zephyr");
const body = await result.response.json();
expect(body.format).toBe("wav");
expect(body.audio).toEqual(expect.any(String));
});
it("preserves an explicit Gemini TTS model and voice pair", async () => {
mockGeminiAudioResponse();
const result = await handleTtsCore({
provider: "gemini",
model: "gemini-2.5-flash-preview-tts/Puck",
input: "Hello from Gemini",
credentials: { apiKey: "test-key" },
responseFormat: "json",
});
expect(result.success).toBe(true);
expect(global.fetch.mock.calls[0][0]).toBe(
"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-preview-tts:generateContent?key=test-key"
);
const sent = JSON.parse(global.fetch.mock.calls[0][1].body);
expect(sent.generationConfig.speechConfig.voiceConfig.prebuiltVoiceConfig.voiceName).toBe("Puck");
});
it("exposes current Gemini TTS models in the TTS catalog", () => {
const entries = buildTtsProviderModels();
expect(entries["gemini-tts-models"].map((model) => model.id)).toEqual([
"gemini-3.1-flash-tts-preview",
"gemini-2.5-flash-preview-tts",
"gemini-2.5-pro-preview-tts",
]);
expect(entries["gemini-tts-voices"]).toContainEqual(
expect.objectContaining({ id: "Zephyr", type: "tts" })
);
});
});