feat: add STT support, Gemini TTS, and expand usage tracking

- Speech-to-Text: full pipeline with sttCore handler, /v1/audio/transcriptions
  endpoint, sttConfig for OpenAI, Gemini, Groq, Deepgram, AssemblyAI,
  HuggingFace, NVIDIA Parakeet; new 9router-stt skill
- Gemini TTS: add gemini provider with 30 prebuilt voices and TTS_PROVIDER_CONFIG
- Usage: implement GLM (intl/cn) and MiniMax (intl/cn) quota fetchers; refactor
  Gemini CLI usage to use retrieveUserQuota with per-model buckets
- Disabled models: lowdb-backed disabledModelsDb + /api/models/disabled route
- Header search: reusable Zustand store (headerSearchStore) wired into Header
- CLI tools: add Claude Cowork tool card and cowork-settings API
- Providers: introduce mediaPriority sorting in getProvidersByKind, add
  Kimi K2.6, reorder hermes, drop qwen STT kind
- UI: expand media-providers/[kind]/[id] page (+314), enhance OAuthModal,
  ModelSelectModal, ProviderTopology, ProxyPools, ProviderLimits
- Assets: refresh provider PNGs (alicode, byteplus, cloudflare-ai, nvidia,
  ollama, vertex, volcengine-ark) and add aws-polly, fal-ai, jina-ai, recraft,
  runwayml, stability-ai, topaz, black-forest-labs
This commit is contained in:
decolua
2026-05-05 10:32:59 +07:00
parent bfb7d42164
commit d4bc42e1f5
67 changed files with 2930 additions and 234 deletions

View File

@@ -0,0 +1,246 @@
"use server";
import { NextResponse } from "next/server";
import fs from "fs/promises";
import path from "path";
import os from "os";
import crypto from "crypto";
const PROVIDER = "gateway";
// Candidate user-data roots — Cowork can run from either Claude-3p (3p mode) or Claude (1p mode w/ cowork features)
const getCandidateRoots = () => {
if (os.platform() === "darwin") {
const base = path.join(os.homedir(), "Library", "Application Support");
return [path.join(base, "Claude-3p"), path.join(base, "Claude")];
}
if (os.platform() === "win32") {
const localApp = process.env.LOCALAPPDATA || path.join(os.homedir(), "AppData", "Local");
const roaming = process.env.APPDATA || path.join(os.homedir(), "AppData", "Roaming");
return [
path.join(localApp, "Claude-3p"),
path.join(roaming, "Claude-3p"),
path.join(localApp, "Claude"),
path.join(roaming, "Claude"),
];
}
return [
path.join(os.homedir(), ".config", "Claude-3p"),
path.join(os.homedir(), ".config", "Claude"),
];
};
// Claude.app/exe install paths — fallback detect when no user-data folder yet
const getAppInstallPaths = () => {
if (os.platform() === "darwin") {
return ["/Applications/Claude.app", path.join(os.homedir(), "Applications", "Claude.app")];
}
if (os.platform() === "win32") {
const localApp = process.env.LOCALAPPDATA || path.join(os.homedir(), "AppData", "Local");
const programFiles = process.env["ProgramFiles"] || "C:\\Program Files";
return [
path.join(localApp, "AnthropicClaude"),
path.join(programFiles, "Claude"),
path.join(programFiles, "AnthropicClaude"),
];
}
return [];
};
// For READ: prefer existing configLibrary (any root). For WRITE: always Claude-3p (first candidate).
const resolveAppRootForRead = async () => {
const candidates = getCandidateRoots();
for (const dir of candidates) {
try {
await fs.access(path.join(dir, "configLibrary"));
return dir;
} catch { /* try next */ }
}
return candidates[0];
};
const getWriteRoot = () => getCandidateRoots()[0]; // always Claude-3p
const getConfigDir = async () => path.join(await resolveAppRootForRead(), "configLibrary");
const getWriteConfigDir = () => path.join(getWriteRoot(), "configLibrary");
const getMetaPath = async () => path.join(await getConfigDir(), "_meta.json");
const getWriteMetaPath = () => path.join(getWriteConfigDir(), "_meta.json");
// Locate Claude (1p) folder for claude_desktop_config.json bootstrap
const get1pRoot = () => {
if (os.platform() === "darwin") {
return path.join(os.homedir(), "Library", "Application Support", "Claude");
}
if (os.platform() === "win32") {
const localApp = process.env.LOCALAPPDATA || path.join(os.homedir(), "AppData", "Local");
const roaming = process.env.APPDATA || path.join(os.homedir(), "AppData", "Roaming");
return path.join(roaming, "Claude"); // 1p uses roaming on Win
}
return path.join(os.homedir(), ".config", "Claude");
};
// Set deploymentMode="3p" in Claude/claude_desktop_config.json (preserve existing keys)
const bootstrapDeploymentMode = async () => {
const cfgPath = path.join(get1pRoot(), "claude_desktop_config.json");
let cfg = {};
try {
const content = await fs.readFile(cfgPath, "utf-8");
cfg = JSON.parse(content);
} catch (error) {
if (error.code !== "ENOENT") throw error;
}
if (cfg.deploymentMode === "3p") return false; // no change
cfg.deploymentMode = "3p";
await fs.mkdir(get1pRoot(), { recursive: true });
await fs.writeFile(cfgPath, JSON.stringify(cfg, null, 2));
return true;
};
// Cowork is available if either (a) any user-data root exists or (b) Claude app is installed
const checkInstalled = async () => {
for (const dir of [...getCandidateRoots(), ...getAppInstallPaths()]) {
try {
await fs.access(dir);
return true;
} catch { /* try next */ }
}
return false;
};
const isLocalhostUrl = (url) => /localhost|127\.0\.0\.1|0\.0\.0\.0/i.test(url || "");
const readJson = async (filePath) => {
try {
const content = await fs.readFile(filePath, "utf-8");
return JSON.parse(content);
} catch (error) {
if (error.code === "ENOENT") return null;
throw error;
}
};
// Ensure meta exists in Claude-3p/configLibrary (write target). If meta already exists in Claude/ (1p), copy appliedId.
const ensureMeta = async () => {
const writeMetaPath = getWriteMetaPath();
let meta = await readJson(writeMetaPath);
if (!meta || !meta.appliedId) {
// Try to inherit from any existing root
const existingRead = await readJson(await getMetaPath());
if (existingRead?.appliedId) {
meta = existingRead;
} else {
const newId = crypto.randomUUID();
meta = { appliedId: newId, entries: [{ id: newId, name: "Default" }] };
}
await fs.mkdir(getWriteConfigDir(), { recursive: true });
await fs.writeFile(writeMetaPath, JSON.stringify(meta, null, 2));
}
return meta;
};
export async function GET() {
try {
const installed = await checkInstalled();
if (!installed) {
return NextResponse.json({
installed: false,
config: null,
message: "Claude Desktop (Cowork mode) not detected",
});
}
const meta = await readJson(await getMetaPath());
const appliedId = meta?.appliedId || null;
const configDir = await getConfigDir();
const configPath = appliedId ? path.join(configDir, `${appliedId}.json`) : null;
const config = configPath ? await readJson(configPath) : null;
const baseUrl = config?.inferenceGatewayBaseUrl || null;
const models = Array.isArray(config?.inferenceModels)
? config.inferenceModels.map((m) => (typeof m === "string" ? m : m?.name)).filter(Boolean)
: [];
const has9Router = !!(config?.inferenceProvider === PROVIDER && baseUrl);
return NextResponse.json({
installed: true,
config,
has9Router,
configPath,
cowork: {
appliedId,
baseUrl,
models,
provider: config?.inferenceProvider || null,
},
});
} catch (error) {
console.log("Error reading cowork settings:", error);
return NextResponse.json({ error: "Failed to read cowork settings" }, { status: 500 });
}
}
export async function POST(request) {
try {
const { baseUrl, apiKey, models } = await request.json();
if (!baseUrl || !apiKey) {
return NextResponse.json({ error: "baseUrl and apiKey are required" }, { status: 400 });
}
if (isLocalhostUrl(baseUrl)) {
return NextResponse.json({
error: "Claude Cowork sandbox cannot reach localhost. Enable Tunnel/Cloud Endpoint or use Tailscale/VPS.",
}, { status: 400 });
}
const modelsArray = Array.isArray(models) ? models.filter((m) => typeof m === "string" && m.trim()) : [];
if (modelsArray.length === 0) {
return NextResponse.json({ error: "At least one model is required" }, { status: 400 });
}
const bootstrapped = await bootstrapDeploymentMode();
const meta = await ensureMeta();
const configPath = path.join(getWriteConfigDir(), `${meta.appliedId}.json`);
const newConfig = {
inferenceProvider: PROVIDER,
inferenceGatewayBaseUrl: baseUrl,
inferenceGatewayApiKey: apiKey,
inferenceModels: modelsArray.map((name) => ({ name })),
};
await fs.writeFile(configPath, JSON.stringify(newConfig, null, 2));
return NextResponse.json({
success: true,
bootstrapped,
message: bootstrapped
? "Cowork enabled (3p mode set). Quit & reopen Claude Desktop."
: "Cowork settings applied. Quit & reopen Claude Desktop.",
configPath,
});
} catch (error) {
console.log("Error applying cowork settings:", error);
return NextResponse.json({ error: "Failed to apply cowork settings" }, { status: 500 });
}
}
export async function DELETE() {
try {
const meta = await readJson(await getMetaPath());
if (!meta?.appliedId) {
return NextResponse.json({ success: true, message: "No active config to reset" });
}
const configPath = path.join(await getConfigDir(), `${meta.appliedId}.json`);
try {
await fs.writeFile(configPath, JSON.stringify({}, null, 2));
} catch (error) {
if (error.code !== "ENOENT") throw error;
}
return NextResponse.json({ success: true, message: "Cowork config reset" });
} catch (error) {
console.log("Error resetting cowork settings:", error);
return NextResponse.json({ error: "Failed to reset cowork settings" }, { status: 500 });
}
}

View File

@@ -33,6 +33,7 @@ export async function GET(request) {
// ElevenLabs requires API key
const raw = provider === "elevenlabs" ? await fetcher(apiKey) : await fetcher();
const useElevenShape = provider === "elevenlabs" || provider === "gemini";
let voices;
if (provider === "local-device") {
@@ -46,7 +47,7 @@ export async function GET(request) {
langName: langName(v.lang),
gender: v.gender,
}));
} else if (provider === "elevenlabs") {
} else if (useElevenShape) {
voices = raw.map((v) => ({
id: v.voice_id,
name: v.name,

View File

@@ -0,0 +1,50 @@
import { NextResponse } from "next/server";
import { getDisabledModels, disableModels, enableModels } from "@/lib/disabledModelsDb";
export const dynamic = "force-dynamic";
// GET /api/models/disabled?providerAlias=xxx
export async function GET(request) {
try {
const { searchParams } = new URL(request.url);
const providerAlias = searchParams.get("providerAlias");
const all = await getDisabledModels();
if (providerAlias) return NextResponse.json({ ids: all[providerAlias] || [] });
return NextResponse.json({ disabled: all });
} catch (error) {
console.log("Error fetching disabled models:", error);
return NextResponse.json({ error: "Failed to fetch disabled models" }, { status: 500 });
}
}
// POST /api/models/disabled body: { providerAlias, ids: [...] }
export async function POST(request) {
try {
const { providerAlias, ids } = await request.json();
if (!providerAlias || !Array.isArray(ids)) {
return NextResponse.json({ error: "providerAlias and ids[] required" }, { status: 400 });
}
await disableModels(providerAlias, ids);
return NextResponse.json({ success: true });
} catch (error) {
console.log("Error disabling models:", error);
return NextResponse.json({ error: "Failed to disable models" }, { status: 500 });
}
}
// DELETE /api/models/disabled?providerAlias=xxx[&id=yyy]
export async function DELETE(request) {
try {
const { searchParams } = new URL(request.url);
const providerAlias = searchParams.get("providerAlias");
const id = searchParams.get("id");
if (!providerAlias) {
return NextResponse.json({ error: "providerAlias required" }, { status: 400 });
}
await enableModels(providerAlias, id ? [id] : []);
return NextResponse.json({ success: true });
} catch (error) {
console.log("Error enabling models:", error);
return NextResponse.json({ error: "Failed to enable models" }, { status: 500 });
}
}

View File

@@ -1,20 +1,29 @@
import { NextResponse } from "next/server";
import { getModelAliases, setModelAlias } from "@/models";
import { getDisabledModels } from "@/lib/disabledModelsDb";
import { AI_MODELS } from "@/shared/constants/config";
import { getProviderAlias } from "@/shared/constants/providers";
// GET /api/models - Get models with aliases
export async function GET() {
try {
const modelAliases = await getModelAliases();
const models = AI_MODELS.map((m) => {
const fullModel = `${m.provider}/${m.model}`;
return {
...m,
fullModel,
alias: modelAliases[fullModel] || m.model,
};
});
const disabled = await getDisabledModels();
const models = AI_MODELS
.filter((m) => {
const alias = getProviderAlias(m.provider) || m.provider;
const list = disabled[alias] || disabled[m.provider] || [];
return !list.includes(m.model);
})
.map((m) => {
const fullModel = `${m.provider}/${m.model}`;
return {
...m,
fullModel,
alias: modelAliases[fullModel] || m.model,
};
});
return NextResponse.json({ models });
} catch (error) {

View File

@@ -7,7 +7,13 @@ import {
pollForToken
} from "@/lib/oauth/providers";
import { createProviderConnection } from "@/models";
import { startCodexProxy, stopCodexProxy } from "@/lib/oauth/utils/server";
import {
startCodexProxy,
stopCodexProxy,
registerCodexSession,
getCodexSessionStatus,
clearCodexSession,
} from "@/lib/oauth/utils/server";
/**
* Dynamic OAuth API Route
@@ -39,8 +45,34 @@ export async function GET(request, { params }) {
if (!appPort) {
return NextResponse.json({ error: "Missing app_port" }, { status: 400 });
}
// Optional server-side mode params: register session for auto-exchange
const state = searchParams.get("state");
const codeVerifier = searchParams.get("code_verifier");
const redirectUri = searchParams.get("redirect_uri");
const result = await startCodexProxy(Number(appPort));
return NextResponse.json(result);
let serverSide = false;
if (result.success && state && codeVerifier && redirectUri) {
serverSide = registerCodexSession({ state, codeVerifier, redirectUri });
}
return NextResponse.json({ ...result, serverSide });
}
if (action === "poll-status") {
if (provider !== "codex") {
return NextResponse.json({ error: "Poll only supported for codex" }, { status: 400 });
}
const state = searchParams.get("state");
if (!state) {
return NextResponse.json({ error: "Missing state" }, { status: 400 });
}
const session = getCodexSessionStatus(state);
if (!session) return NextResponse.json({ status: "unknown" });
if (session.status === "done" || session.status === "error") {
const payload = { ...session };
clearCodexSession(state);
return NextResponse.json(payload);
}
return NextResponse.json({ status: session.status });
}
if (action === "stop-proxy") {

View File

@@ -110,7 +110,7 @@ export async function POST(request) {
if (!provider || !isValidProvider) {
return NextResponse.json({ error: "Invalid provider" }, { status: 400 });
}
if (!apiKey) {
if (!apiKey && provider !== "ollama-local") {
return NextResponse.json({ error: `${isWebCookieProvider ? "Cookie value" : "API Key"} is required` }, { status: 400 });
}
if (!name) {
@@ -185,7 +185,7 @@ export async function POST(request) {
provider,
authType: isWebCookieProvider ? "cookie" : "apikey",
name,
apiKey,
apiKey: apiKey || "",
priority: priority || 1,
globalPriority: globalPriority || null,
defaultModel: defaultModel || null,

View File

@@ -49,7 +49,7 @@ async function probeMediaProvider(provider, apiKey) {
const kinds = p.serviceKinds || ["llm"];
const isMediaOnly = kinds.every((k) => MEDIA_KINDS.has(k));
if (!isMediaOnly) return null;
const cfg = p.ttsConfig || p.embeddingConfig || p.imageConfig || p.videoConfig || p.musicConfig;
const cfg = p.ttsConfig || p.sttConfig || p.embeddingConfig || p.imageConfig || p.videoConfig || p.musicConfig;
// No probe config → best-effort accept (validate at usage time)
if (!cfg) return true;
if (p.noAuth || cfg.authType === "none") return true;

View File

@@ -5,6 +5,7 @@ import { getProviderConnectionById, updateProviderConnection } from "@/lib/local
import { getUsageForProvider } from "open-sse/services/usage.js";
import { getExecutor } from "open-sse/executors/index.js";
import { resolveConnectionProxyConfig } from "@/lib/network/connectionProxy";
import { USAGE_APIKEY_PROVIDERS } from "@/shared/constants/providers";
// Detect auth-expired messages returned by usage providers instead of throwing
const AUTH_EXPIRED_PATTERNS = ["expired", "authentication", "unauthorized", "401", "re-authorize"];
@@ -113,9 +114,14 @@ export async function GET(request, { params }) {
return Response.json({ error: "Connection not found" }, { status: 404 });
}
// Only OAuth connections have usage APIs
if (connection.authType !== "oauth") {
return Response.json({ message: "Usage not available for API key connections" });
// Allow OAuth connections, plus whitelisted apikey providers (glm/minimax/...)
const isOAuth = connection.authType === "oauth";
const isApikeyEligible =
connection.authType === "apikey" &&
USAGE_APIKEY_PROVIDERS.includes(connection.provider);
if (!isOAuth && !isApikeyEligible) {
return Response.json({ message: "Usage not available for this connection" });
}
// Resolve connection proxy config; force strictProxy=false so quota/refresh fall back to direct on failure
@@ -128,23 +134,25 @@ export async function GET(request, { params }) {
strictProxy: false,
};
// Refresh credentials if needed using executor
try {
const result = await refreshAndUpdateCredentials(connection, false, proxyOptions);
connection = result.connection;
} catch (refreshError) {
console.error("[Usage API] Credential refresh failed:", refreshError);
return Response.json({
error: `Credential refresh failed: ${refreshError.message}`
}, { status: 401 });
// Refresh credentials only for OAuth connections (apikey has no token refresh)
if (isOAuth) {
try {
const result = await refreshAndUpdateCredentials(connection, false, proxyOptions);
connection = result.connection;
} catch (refreshError) {
console.error("[Usage API] Credential refresh failed:", refreshError);
return Response.json({
error: `Credential refresh failed: ${refreshError.message}`
}, { status: 401 });
}
}
// Fetch usage from provider API
let usage = await getUsageForProvider(connection, proxyOptions);
// If provider returned an auth-expired message instead of throwing,
// force-refresh token and retry once
if (isAuthExpiredMessage(usage) && connection.refreshToken) {
// force-refresh token and retry once (OAuth only)
if (isOAuth && isAuthExpiredMessage(usage) && connection.refreshToken) {
try {
const retryResult = await refreshAndUpdateCredentials(connection, true, proxyOptions);
connection = retryResult.connection;

View File

@@ -0,0 +1,19 @@
import { handleStt } from "@/sse/handlers/stt.js";
// Allow large audio uploads — 5min for processing large files
export const maxDuration = 300;
export async function OPTIONS() {
return new Response(null, {
headers: {
"Access-Control-Allow-Origin": "*",
"Access-Control-Allow-Methods": "POST, OPTIONS",
"Access-Control-Allow-Headers": "*",
},
});
}
/** POST /v1/audio/transcriptions - OpenAI Whisper compatible STT */
export async function POST(request) {
return await handleStt(request);
}