From 5e5e78d3e88a2bc16eae83441eee351079fed997 Mon Sep 17 00:00:00 2001 From: Sutarto Jordan Chrisfivo Date: Thu, 18 Jun 2026 09:13:27 +0700 Subject: [PATCH] fix: show custom vision models in LLM selector and model list Expose user-added imageToText custom models as vision-capable chat models in the default LLM selector and /v1/models, map custom service kinds to runtime capabilities, and keep typed filtering for /v1/models/{kind}. Co-authored-by: Cursor --- open-sse/providers/capabilities.js | 15 ++++++++++ src/app/api/v1/models/route.js | 30 +++++++++++++++----- src/shared/components/ModelSelectModal.js | 5 +++- tests/unit/capabilities-service-kind.test.js | 15 ++++++++++ 4 files changed, 57 insertions(+), 8 deletions(-) create mode 100644 tests/unit/capabilities-service-kind.test.js diff --git a/open-sse/providers/capabilities.js b/open-sse/providers/capabilities.js index 5f04ec23..77220317 100644 --- a/open-sse/providers/capabilities.js +++ b/open-sse/providers/capabilities.js @@ -51,6 +51,21 @@ export const DEFAULT_CAPABILITIES = { maxOutput: 64000, }; +// User-added model metadata can carry dashboard service kinds instead of the +// runtime capability names used here. Map those typed model kinds into input / +// output capabilities so custom vision models are not treated as text-only. +const SERVICE_KIND_CAPABILITIES = { + imageToText: { vision: true }, + image: { imageOutput: true }, + stt: { audioInput: true }, + tts: { audioOutput: true }, + embedding: { tools: false }, +}; + +export function capabilitiesFromServiceKind(kind) { + return SERVICE_KIND_CAPABILITIES[kind] || null; +} + /** * Canonical exact-id overrides — used for exceptions that patterns would * otherwise mis-match. Only declare deltas vs DEFAULT. diff --git a/src/app/api/v1/models/route.js b/src/app/api/v1/models/route.js index 81560148..e573805b 100644 --- a/src/app/api/v1/models/route.js +++ b/src/app/api/v1/models/route.js @@ -9,6 +9,7 @@ import { getProviderConnections, getCombos, getCustomModels, getModelAliases } f import { getDisabledModels } from "@/lib/disabledModelsDb"; import { resolveKiroModels } from "open-sse/services/kiroModels.js"; import { resolveQoderModels } from "open-sse/services/qoderModels.js"; +import { capabilitiesFromServiceKind } from "open-sse/providers/capabilities.js"; // Per-provider live model resolvers. Each receives a connection record and // returns { models: [{ id, name? }, ...] } | null on failure. @@ -314,13 +315,22 @@ export async function buildModelsList(kindFilter) { }) .filter((modelId) => typeof modelId === "string" && modelId.trim() !== ""); + const customModelKindById = new Map(); const customModelIds = customModels .filter((m) => { - if (!m?.id || (getModelKind(m) && getModelKind(m) !== "llm")) return false; + if (!m?.id) return false; + const kind = getModelKind(m) || LLM_KIND; + // imageToText custom models are vision-capable chat models: expose them + // both in the default LLM list and in /v1/models/image-to-text. + if (!kindFilter.includes(kind) && !(kind === "imageToText" && kindFilter.includes(LLM_KIND))) return false; const alias = m.providerAlias; return alias === staticAlias || alias === outputAlias || alias === providerId; }) - .map((m) => String(m.id).trim()) + .map((m) => { + const modelId = String(m.id).trim(); + if (modelId) customModelKindById.set(modelId, getModelKind(m) || LLM_KIND); + return modelId; + }) .filter((modelId) => modelId !== ""); const aliasModelIds = Object.values(modelAliases || {}) @@ -349,16 +359,22 @@ export async function buildModelsList(kindFilter) { const mergedModelIds = Array.from(new Set([...modelIds, ...customModelIds, ...aliasModelIds])); for (const modelId of mergedModelIds) { - // Resolve kind: prefer static metadata, otherwise infer from ID heuristics - const kind = staticModelKindById.get(modelId) || inferKindFromUnknownModelId(modelId); - if (!kindFilter.includes(kind)) continue; + // Resolve kind: prefer static/custom metadata, otherwise infer from ID heuristics + const customKind = customModelKindById.get(modelId); + const kind = staticModelKindById.get(modelId) || customKind || inferKindFromUnknownModelId(modelId); + // imageToText custom models stay in the LLM list (vision-capable chat models) + const allowAsLlm = kind === "imageToText" && kindFilter.includes(LLM_KIND); + if (!kindFilter.includes(kind) && !allowAsLlm) continue; if (isDisabled(outputAlias, modelId) || isDisabled(staticAlias, modelId)) continue; - models.push({ + const model = { id: `${outputAlias}/${modelId}`, object: "model", owned_by: outputAlias, - }); + }; + const caps = capabilitiesFromServiceKind(customKind); + if (caps) model.capabilities = caps; + models.push(model); } // Web search/fetch — provider IS the model, expose as {alias}/search and/or {alias}/fetch with explicit kind diff --git a/src/shared/components/ModelSelectModal.js b/src/shared/components/ModelSelectModal.js index d5866012..06e5bff5 100644 --- a/src/shared/components/ModelSelectModal.js +++ b/src/shared/components/ModelSelectModal.js @@ -128,7 +128,10 @@ export default function ModelSelectModal({ // Filter a models[] array by kindFilter (keep only matching kind) const filterByKind = (models) => { - if (!kindFilter) return models.filter((m) => m.isPlaceholder || !getModelKind(m) || getModelKind(m) === "llm"); + // No kindFilter means the LLM selector. Keep custom models visible because + // user-added models may have typed capabilities (for example imageToText) + // while still being valid chat/combo targets. + if (!kindFilter) return models.filter((m) => m.isPlaceholder || m.isCustom || !getModelKind(m) || getModelKind(m) === "llm"); if (!TYPED_KINDS.has(kindFilter)) return models; return models.filter((m) => m.isPlaceholder || getModelKind(m) === kindFilter); }; diff --git a/tests/unit/capabilities-service-kind.test.js b/tests/unit/capabilities-service-kind.test.js new file mode 100644 index 00000000..e5019af9 --- /dev/null +++ b/tests/unit/capabilities-service-kind.test.js @@ -0,0 +1,15 @@ +import { describe, expect, it } from "vitest"; + +import { capabilitiesFromServiceKind } from "../../open-sse/providers/capabilities.js"; + +describe("capabilitiesFromServiceKind", () => { + it("maps imageToText custom models to vision-capable runtime models", () => { + expect(capabilitiesFromServiceKind("imageToText")).toMatchObject({ vision: true }); + }); + + it("maps media output/input custom model kinds to runtime capabilities", () => { + expect(capabilitiesFromServiceKind("image")).toMatchObject({ imageOutput: true }); + expect(capabilitiesFromServiceKind("stt")).toMatchObject({ audioInput: true }); + expect(capabilitiesFromServiceKind("tts")).toMatchObject({ audioOutput: true }); + }); +});