...and 20+ more providers including Nebius, Chutes, Hyperbolic, and custom OpenAI/Anthropic compatible endpoints
+### 🏠 Self-hosted Providers + +For speech and embeddings served from **your own** machine — whisper.cpp, +faster-whisper, Speaches, Kokoro-FastAPI, openedai-speech, llama.cpp/llama-server, +vLLM, Infinity, text-embeddings-inference, or anything else that speaks the OpenAI +shape. + +| Provider | Endpoint used | Typical server | +| --- | --- | --- | +| **Self-hosted STT** | `/v1/audio/transcriptions` | whisper.cpp, faster-whisper | +| **Self-hosted TTS** | `/v1/audio/speech` | Kokoro-FastAPI, openedai-speech | +| **Self-hosted Embedding** | `/v1/embeddings` | llama-server, vLLM, Infinity | + +Every other speech provider is a named cloud service with a fixed endpoint. These +three read their address from **each connection**, so one provider can front +several machines and load-balance across them like any other. + +Set it on the connection as `providerSpecificData.baseUrl`: + +| Provider | Give it | Result | +| --- | --- | --- | +| Self-hosted STT | the full URL — `http://host:8080/v1/audio/transcriptions` | used as-is | +| Self-hosted TTS | the server root — `http://host:8880` | `+ /v1/audio/speech` | +| Self-hosted Embedding | the **OpenAI base**, `/v1` included — `http://host:8080/v1` | `+ /embeddings` | + +> **Mind the `/v1` on embeddings.** The adapter appends `/embeddings`, so +> `http://host:8080` resolves to `http://host:8080/embeddings` and misses the +> OpenAI route — llama-server answers **501**. Give it the same base URL an OpenAI +> client would use. A full `.../v1/embeddings` is also accepted, so a value pasted +> from a `curl` example works too. + +The API key is not checked by most local servers, but the field must be non-empty: +it is what gives the connection a credentials record, and `baseUrl` lives there. +Any placeholder works. + +Self-hosted Embedding has **no cloud fallback by design** — a connection saved +without a `baseUrl` is reported as a configuration error rather than quietly +falling back to `api.openai.com`, which would send your input text and API key to +a third party under a provider named "Self-hosted". + --- ## 💡 Key Features diff --git a/open-sse/handlers/embeddingProviders/index.js b/open-sse/handlers/embeddingProviders/index.js index 62e52778..9dd6a604 100644 --- a/open-sse/handlers/embeddingProviders/index.js +++ b/open-sse/handlers/embeddingProviders/index.js @@ -2,6 +2,7 @@ import createOpenAIEmbeddingAdapter from "./openai.js"; import gemini from "./gemini.js"; import openaiCompatNode from "./openaiCompatNode.js"; +import selfhostedEmbedding from "./selfhostedEmbedding.js"; const OPENAI_COMPAT_PROVIDERS = [ "openai", "openrouter", "mistral", "voyage-ai", "fireworks", @@ -13,6 +14,12 @@ const ADAPTERS = { ...Object.fromEntries(OPENAI_COMPAT_PROVIDERS.map((id) => [id, createOpenAIEmbeddingAdapter(id)])), gemini, google_ai_studio: gemini, + // Self-hosted reads creds.providerSpecificData.baseUrl (one provider, many + // servers) — but via its OWN adapter, not openaiCompatNode: that one falls back + // to api.openai.com when no baseUrl is set, which under a provider called + // "Self-hosted Embedding" means silently shipping the input and API key to + // OpenAI. selfhostedEmbedding refuses instead. + "selfhosted-embedding": selfhostedEmbedding, }; export function getEmbeddingAdapter(provider) { diff --git a/open-sse/handlers/embeddingProviders/selfhostedEmbedding.js b/open-sse/handlers/embeddingProviders/selfhostedEmbedding.js new file mode 100644 index 00000000..aa030439 --- /dev/null +++ b/open-sse/handlers/embeddingProviders/selfhostedEmbedding.js @@ -0,0 +1,46 @@ +// Self-hosted embeddings — like openaiCompatNode, but the baseUrl is REQUIRED. +// +// openaiCompatNode falls back to https://api.openai.com/v1 when a connection +// carries no providerSpecificData.baseUrl. For a custom NODE that default is +// defensible: the node was created by pointing at some OpenAI-compatible URL, and +// OpenAI is the archetype. For a provider whose entire purpose is "my own +// server", it is actively harmful — a connection saved without a baseUrl sends +// the INPUT TEXT and the API KEY to OpenAI, silently, under a provider named +// "Self-hosted Embedding". +// +// Observed exactly that with a placeholder connection (2026-08-04): +// +// [selfhosted-embedding/embedding] [401]: Incorrect API key provided: abc. +// You can find your API key at https://platform.openai.com/account/api-keys. +// +// The key "abc" was typed as a throwaway for a LOCAL server and left the network. +// A self-hosted provider must never have a cloud fallback, so this one refuses +// instead: no baseUrl means a configuration error, reported as such. +import createOpenAIEmbeddingAdapter from "./openai.js"; + +const baseAdapter = createOpenAIEmbeddingAdapter("openai"); + +export class MissingBaseUrlError extends Error { + constructor() { + super( + "Self-hosted Embedding needs an endpoint: set this connection's baseUrl to " + + "the OpenAI base URL of your server, e.g. http://host:8080/v1 (note the /v1 — " + + "\"/embeddings\" is appended to it). Refusing to fall back to api.openai.com, " + + "which would send your input and API key to OpenAI." + ); + this.name = "MissingBaseUrlError"; + this.isConfigError = true; + } +} + +export default { + ...baseAdapter, + buildUrl: (_model, creds) => { + const rawBaseUrl = creds?.providerSpecificData?.baseUrl; + if (!rawBaseUrl || !String(rawBaseUrl).trim()) throw new MissingBaseUrlError(); + // Accept either the OpenAI base or a full embeddings URL, so a value pasted + // from a curl example works as well as one typed from the help text. + const baseUrl = String(rawBaseUrl).trim().replace(/\/$/, "").replace(/\/embeddings$/, ""); + return `${baseUrl}/embeddings`; + }, +}; diff --git a/open-sse/handlers/embeddingsCore.js b/open-sse/handlers/embeddingsCore.js index aa81117c..1ee0f4cd 100644 --- a/open-sse/handlers/embeddingsCore.js +++ b/open-sse/handlers/embeddingsCore.js @@ -1,5 +1,5 @@ import { createErrorResult, parseUpstreamError, formatProviderError } from "../utils/error.js"; -import { HTTP_STATUS } from "../config/runtimeConfig.js"; +import { HTTP_STATUS, FETCH_CONNECT_TIMEOUT_MS } from "../config/runtimeConfig.js"; import { getExecutor } from "../executors/index.js"; import { refreshWithRetry } from "../services/tokenRefresh.js"; import { getEmbeddingAdapter } from "./embeddingProviders/index.js"; @@ -38,13 +38,24 @@ export async function handleEmbeddingsCore({ } const ctx = { input }; - const url = adapter.buildUrl(model, credentials, ctx); - const headers = adapter.buildHeaders(credentials, ctx); - const requestBody = adapter.buildBody(model, { - input, - encoding_format: body.encoding_format || "float", - dimensions: body.dimensions, - }); + // buildUrl/buildHeaders/buildBody were called bare. An adapter that rejects a + // misconfigured connection — selfhosted-embedding throws when no baseUrl is set + // rather than silently falling back to api.openai.com — would have escaped this + // function uncaught, surfacing as a 500 or a request that never settles. A + // configuration mistake is a 400 with the reason in it. + let url, headers, requestBody; + try { + url = adapter.buildUrl(model, credentials, ctx); + headers = adapter.buildHeaders(credentials, ctx); + requestBody = adapter.buildBody(model, { + input, + encoding_format: body.encoding_format || "float", + dimensions: body.dimensions, + }); + } catch (error) { + log?.debug?.("EMBEDDINGS", `Request build failed: ${error.message}`); + return createErrorResult(HTTP_STATUS.BAD_REQUEST, `[${provider}/${model}] ${error.message}`); + } log?.debug?.("EMBEDDINGS", `${provider.toUpperCase()} | ${model} | input_type=${Array.isArray(input) ? `array[${input.length}]` : "string"}`); @@ -54,6 +65,9 @@ export async function handleEmbeddingsCore({ method: "POST", headers, body: JSON.stringify(requestBody), + ...(typeof AbortSignal?.timeout === "function" + ? { signal: AbortSignal.timeout(FETCH_CONNECT_TIMEOUT_MS) } + : {}), }); } catch (error) { const errMsg = formatProviderError(error, provider, model, HTTP_STATUS.BAD_GATEWAY); diff --git a/open-sse/handlers/sttCore.js b/open-sse/handlers/sttCore.js index acb7d13b..8127782a 100644 --- a/open-sse/handlers/sttCore.js +++ b/open-sse/handlers/sttCore.js @@ -170,9 +170,17 @@ export async function handleSttCore({ provider, model, formData, credentials, st const file = formData.get("file"); if (!file) return createErrorResult(HTTP_STATUS.BAD_REQUEST, "Missing required field: file"); - const cfg = sttConfig; + let cfg = sttConfig; if (!cfg) return createErrorResult(HTTP_STATUS.BAD_REQUEST, `Provider '${provider}' does not support STT`); + // Per-connection endpoint override. Registry entries carry a fixed baseUrl, + // which is right for a named cloud service but useless for a self-hosted one + // whose address only the operator knows. Opt-in: absent unless the connection + // sets it, so cloud providers are untouched. Mirrors the custom embedding + // providers, which already resolve baseUrl the same way. + const overrideUrl = credentials?.providerSpecificData?.baseUrl; + if (overrideUrl) cfg = { ...cfg, baseUrl: String(overrideUrl).replace(/\/+$/, "") }; + const token = cfg.authType === "none" ? null : (credentials?.apiKey || credentials?.accessToken); if (cfg.authType !== "none" && !token) { return createErrorResult(HTTP_STATUS.UNAUTHORIZED, `No credentials for STT provider: ${provider}`); diff --git a/open-sse/handlers/ttsProviders/index.js b/open-sse/handlers/ttsProviders/index.js index 80e104d2..d2be03fa 100644 --- a/open-sse/handlers/ttsProviders/index.js +++ b/open-sse/handlers/ttsProviders/index.js @@ -7,6 +7,7 @@ import openai from "./openai.js"; import openrouter from "./openrouter.js"; import gemini, { fetchGeminiVoices } from "./gemini.js"; import xiaomiMimo from "./xiaomi-mimo.js"; +import selfhostedTts from "./selfhostedTts.js"; import { FORMAT_HANDLERS } from "./genericFormats.js"; import { parseModelVoice } from "./_base.js"; @@ -20,6 +21,7 @@ const SPECIAL_ADAPTERS = { openrouter, gemini, "xiaomi-mimo": xiaomiMimo, + "selfhosted-tts": selfhostedTts, }; export function getTtsAdapter(provider) { diff --git a/open-sse/handlers/ttsProviders/selfhostedTts.js b/open-sse/handlers/ttsProviders/selfhostedTts.js new file mode 100644 index 00000000..fdfdbf76 --- /dev/null +++ b/open-sse/handlers/ttsProviders/selfhostedTts.js @@ -0,0 +1,69 @@ +// Self-hosted OpenAI-compatible TTS — POST {baseUrl}/v1/audio/speech. +// +// A SPECIAL_ADAPTER rather than a genericFormats handler on purpose: the generic +// dispatcher resolves baseUrl from the static registry entry +// (`synthesizeViaConfig` reads `cfg.baseUrl`) and never looks at the connection, +// which is exactly the limitation this provider exists to lift. +import { Buffer } from "node:buffer"; + +const DEFAULT_BASE_URL = "http://localhost:8880"; +const DEFAULT_MODEL = "kokoro"; +const DEFAULT_VOICE = "af_heart"; + +export default { + async synthesize(text, model, credentials, responseFormat = "mp3") { + // Accept either providerSpecificData.baseUrl (how the custom embedding and + // STT providers carry it) or a bare credentials.baseUrl (how the OpenAI TTS + // adapter does), so a connection configured either way works. + const raw = credentials?.providerSpecificData?.baseUrl || credentials?.baseUrl || DEFAULT_BASE_URL; + // Tolerate a baseUrl given as the full endpoint or with a trailing /v1 — + // both are natural things to paste, and silently double-appending the path + // would 404 with nothing pointing at the cause. + const base = String(raw) + .replace(/\/+$/, "") + .replace(/\/v1\/audio\/speech$/, "") + .replace(/\/v1$/, ""); + + // The provider prefix is already stripped by getModelInfo, so `model` here is + // "kokoro" or "kokoro/af_heart" — NOT "selfhosted-tts/...". + // + // A bare value is the MODEL, not the voice. The OpenAI adapter reads a bare + // value as a voice, which is right for a service whose model is fixed + // ("tts-1") and whose voice varies — but wrong here, where the model is the + // variable part. Treating it as a voice sent voice="kokoro" upstream and + // Kokoro answered 400, so `selfhosted-tts/kokoro` — the obvious way to + // address this provider — was the one form that did not work (verified + // against a live Kokoro through 9router, 2026-08-03). + let ttsModel = DEFAULT_MODEL; + let voice = DEFAULT_VOICE; + if (model) { + const parts = String(model).split("/").filter(Boolean); + if (parts.length >= 2) { + ttsModel = parts[0]; + voice = parts.slice(1).join("/"); + } else if (parts.length === 1) { + ttsModel = parts[0]; + } + } + + const res = await fetch(`${base}/v1/audio/speech`, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...(credentials?.apiKey ? { Authorization: `Bearer ${credentials.apiKey}` } : {}), + }, + body: JSON.stringify({ + model: ttsModel, + voice, + input: text, + response_format: responseFormat, + }), + }); + if (!res.ok) { + const err = await res.json().catch(() => ({})); + throw new Error(err?.error?.message || `Self-hosted TTS failed: ${res.status}`); + } + const buf = await res.arrayBuffer(); + return { base64: Buffer.from(buf).toString("base64"), format: responseFormat }; + }, +}; diff --git a/open-sse/providers/registry/index.js b/open-sse/providers/registry/index.js index ba8a4c17..c08c0597 100644 --- a/open-sse/providers/registry/index.js +++ b/open-sse/providers/registry/index.js @@ -117,6 +117,9 @@ import p113 from "./morph.js"; // import p104 from "./windsurf.js"; import p115 from "./poolside.js"; import p116 from "./tokenrouter.js"; +import p117 from "./selfhosted-stt.js"; +import p118 from "./selfhosted-tts.js"; +import p119 from "./selfhosted-embedding.js"; export default [ p0, @@ -235,4 +238,7 @@ export default [ // p104, // windsurf — hidden, no tool calling p115, p116, + p117, + p118, + p119, ]; diff --git a/open-sse/providers/registry/selfhosted-embedding.js b/open-sse/providers/registry/selfhosted-embedding.js new file mode 100644 index 00000000..90fa79c7 --- /dev/null +++ b/open-sse/providers/registry/selfhosted-embedding.js @@ -0,0 +1,73 @@ +// Self-hosted, OpenAI-compatible embeddings (llama.cpp / llama-server, vLLM, +// Infinity, text-embeddings-inference, ...) — the embeddings counterpart of +// selfhosted-stt and selfhosted-tts. +// +// Routing a self-hosted embeddings server already WORKS today, via a custom +// provider node: getEmbeddingAdapter() matches `openai-compatible-*` and +// `custom-embedding-*` and returns openaiCompatNode, whose buildUrl reads +// creds.providerSpecificData.baseUrl. What is missing is a first-class provider, +// and the gap is visible rather than functional: +// +// /v1/embeddings on such a node -> 200, correct vectors +// the Embedding page in the dashboard -> the node is not listed at all +// +// The page renders getProvidersByKind("embedding") plus provider nodes filtered +// to `type === "custom-embedding"`. A node created as `openai-compatible` — the +// natural choice when ONE endpoint serves chat and embeddings behind the same +// front door — satisfies neither, so a working self-hosted embeddings endpoint is +// invisible on the page whose job is to show embeddings providers. Diagnosed on a +// deployment serving Qwen3-Embedding-8B at 4096 dimensions through exactly that +// shape (2026-08-04). +// +// Declaring it as a provider with serviceKinds: ["embedding"] puts it on the page +// beside Voyage, Jina and the rest, and keeps the per-connection baseUrl that +// makes self-hosting possible at all. +// +// authType is "apikey" rather than "none" for the same reason as the STT and TTS +// entries: it is what gives the connection a credentials record, and +// providerSpecificData.baseUrl lives there. Local servers ignore the key itself; +// any non-empty value works. +export default { + id: "selfhosted-embedding", + priority: 50, + hasFree: true, + alias: "selfhosted-embedding", + display: { + name: "Self-hosted Embedding", + icon: "cloud", + color: "#ffffffff", + textIcon: "SE", + website: "https://github.com/ggml-org/llama.cpp", + }, + category: "apikey", + auth: { + apiKey: { + // Note the /v1: the adapter appends "/embeddings" to whatever it is given, + // so a bare http://host:8080 resolves to http://host:8080/embeddings and + // misses the OpenAI route entirely. Give it the OpenAI base, the same value + // an OpenAI client would use. A trailing /embeddings is tolerated. + text: "Set providerSpecificData.baseUrl to the OpenAI base URL, e.g. http://host:8080/v1 — /embeddings is appended. The API key is not checked by local servers; any value works.", + }, + }, + // A self-hosted server serves whatever model it was started with, so the id + // here is a placeholder for the UI: the request passes `model` straight + // through, and llama-server ignores an unknown value rather than rejecting it. + // Dimensions are deliberately NOT declared — they are a property of the loaded + // weights, and asserting a number here would be a guess that silently + // contradicts the server. + models: [ + { id: "embedding", name: "Self-hosted embedding model", kind: "embedding" }, + ], + serviceKinds: ["embedding"], + embeddingConfig: { + // Declared for shape-consistency with the other embedding providers, and + // read by the UI — but NOT by the request path. openaiCompatNode resolves the + // URL purely from creds.providerSpecificData.baseUrl (falling back to + // api.openai.com), so unlike a fixed cloud provider this baseUrl never + // reaches the wire. Stated plainly because a reader would otherwise + // reasonably assume it is the default endpoint. + baseUrl: "http://localhost:8080/v1/embeddings", + authType: "apikey", + authHeader: "bearer", + }, +}; diff --git a/open-sse/providers/registry/selfhosted-stt.js b/open-sse/providers/registry/selfhosted-stt.js new file mode 100644 index 00000000..d2d9bc80 --- /dev/null +++ b/open-sse/providers/registry/selfhosted-stt.js @@ -0,0 +1,48 @@ +// Self-hosted, OpenAI-compatible speech-to-text (whisper.cpp, faster-whisper, +// Speaches, vLLM-served Whisper, ...). +// +// Every other STT provider here is a named cloud service with a fixed endpoint. +// This one exists so a locally-served /v1/audio/transcriptions can be used at +// all: set the connection's providerSpecificData.baseUrl to the full URL of the +// endpoint, exactly as the custom embedding providers already work. +// +// sttCore dispatches on `format`; anything that is not one of the five named +// cloud shapes falls through to transcribeOpenAICompatible, which POSTs the +// standard multipart body (file, model, and optional language / prompt / +// response_format / temperature). That is precisely what whisper.cpp's OpenAI +// endpoint accepts. +// +// authType is "apikey" rather than "none" so the connection carries a +// credentials record — which is where providerSpecificData.baseUrl lives. Local +// servers ignore the key itself; any non-empty value works. +export default { + id: "selfhosted-stt", + priority: 50, + hasFree: true, + alias: "selfhosted-stt", + display: { + name: "Self-hosted STT", + icon: "cloud", + color: "#ffffffff", + textIcon: "ST", + website: "https://github.com/ggml-org/whisper.cpp", + }, + category: "apikey", + auth: { + apiKey: { + text: "Set providerSpecificData.baseUrl to the full transcriptions URL, e.g. http://host:8080/v1/audio/transcriptions. The API key is not checked by local servers; any value works.", + }, + }, + models: [ + { id: "whisper-1", name: "Whisper (self-hosted)", params: ["language", "response_format", "temperature", "prompt"], kind: "stt" }, + ], + serviceKinds: ["stt"], + sttConfig: { + // Overridden per connection by providerSpecificData.baseUrl; this default + // only makes the provider usable out of the box on a same-host deployment. + baseUrl: "http://localhost:8080/v1/audio/transcriptions", + authType: "apikey", + authHeader: "bearer", + format: "openai", + }, +}; diff --git a/open-sse/providers/registry/selfhosted-tts.js b/open-sse/providers/registry/selfhosted-tts.js new file mode 100644 index 00000000..ff0a67b1 --- /dev/null +++ b/open-sse/providers/registry/selfhosted-tts.js @@ -0,0 +1,44 @@ +// Self-hosted, OpenAI-compatible text-to-speech (Kokoro-FastAPI, openedai-speech, +// vLLM-served TTS, ...) — the TTS counterpart of selfhosted-stt. +// +// Every other self-hostable TTS provider here (coqui, tortoise) carries a FIXED +// localhost baseUrl in its registry entry and `authType: "none"`, and the generic +// dispatcher reads `ttsConfig.baseUrl` from that entry rather than from the +// connection. So there was no way to point TTS at a server on another host. +// +// `authType: "apikey"` is what makes the override possible at all: it gives the +// connection a credentials record, which is where providerSpecificData.baseUrl +// lives. Local servers ignore the key; any non-empty value works. +export default { + id: "selfhosted-tts", + priority: 50, + hasFree: true, + alias: "selfhosted-tts", + display: { + name: "Self-hosted TTS", + icon: "cloud", + color: "#ffffffff", + textIcon: "TT", + website: "https://github.com/remsky/Kokoro-FastAPI", + }, + category: "apikey", + auth: { + apiKey: { + text: "Set providerSpecificData.baseUrl to the server root, e.g. http://host:8080 — /v1/audio/speech is appended. The API key is not checked by local servers; any value works.", + }, + }, + // Voice is selected as "