diff --git a/open-sse/handlers/chatCore.js b/open-sse/handlers/chatCore.js index cb6ce961..88356471 100644 --- a/open-sse/handlers/chatCore.js +++ b/open-sse/handlers/chatCore.js @@ -57,7 +57,7 @@ export function stripContinuityFields(body) { return body; } -export async function handleChatCore({ body, modelInfo, credentials, log, onCredentialsRefreshed, onRequestSuccess, onDisconnect, clientRawRequest, connectionId, userAgent, apiKey, ccFilterNaming, rtkEnabled, headroomEnabled, headroomUrl, headroomCompressUserMessages, cavemanEnabled, cavemanLevel, ponytailEnabled, ponytailLevel, pxpipeEnabled, pxpipeMinChars, pxpipeTimeoutMs, pxpipeTransform, onPxpipeEvent, sourceFormatOverride, providerThinking }) { +export async function handleChatCore({ body, modelInfo, credentials, log, onCredentialsRefreshed, onRequestSuccess, onDisconnect, clientRawRequest, connectionId, userAgent, apiKey, ccFilterNaming, rtkEnabled, headroomEnabled, headroomUrl, headroomCompressUserMessages, cavemanEnabled, cavemanLevel, ponytailEnabled, ponytailLevel, pxpipeEnabled, pxpipeMinChars, pxpipeTimeoutMs, pxpipeTransform, onPxpipeEvent, sourceFormatOverride, providerThinking, capsOverride }) { const { provider, model } = modelInfo; const requestStartTime = Date.now(); // Stable per-session color so all lines of one CLI conversation share a tag @@ -149,8 +149,10 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred if (credentials) credentials.rawHeaders = clientRawRequest?.headers || {}; // Auto-strip media blocks the model can't read (vision/audio/pdf) before translation. + // capsOverride lets the app layer assert per-model capabilities (e.g. user-registered + // models) on top of the static tables. if (!passthrough) { - const caps = getCapabilitiesForModel(provider, model); + const caps = { ...getCapabilitiesForModel(provider, model), ...(capsOverride || {}) }; if (stripUnsupportedModalities(body, sourceFormat, caps)) { log?.debug?.("MODALITY", `stripped unsupported media for ${provider}/${model}`); } diff --git a/open-sse/providers/capabilities.js b/open-sse/providers/capabilities.js index 24a03b04..024eda65 100644 --- a/open-sse/providers/capabilities.js +++ b/open-sse/providers/capabilities.js @@ -317,6 +317,11 @@ export const PATTERN_CAPABILITIES = [ { pattern: "*ling-*", caps: { reasoning: true, contextWindow: 128000 } }, ]; +// OpenRouter-style gateways validate modalities upstream — a text-only model +// sent an image gets a clear upstream error instead of silent corruption. So for +// unknown models on these providers, trust vision instead of stripping images. +const TRUST_UPSTREAM_VISION = new Set(["openrouter"]); + /** * Resolve capabilities for a model using the 4-step fallback chain, * merged over DEFAULT_CAPABILITIES so the result is always complete. @@ -349,6 +354,9 @@ export function getCapabilitiesForModel(provider, model) { } } - // 4. Floor + // 4. Floor (upstream-validated gateways keep vision on for unknown models) + if (provider && TRUST_UPSTREAM_VISION.has(provider)) { + return { ...DEFAULT_CAPABILITIES, vision: true }; + } return { ...DEFAULT_CAPABILITIES }; } diff --git a/src/sse/handlers/chat.js b/src/sse/handlers/chat.js index 88353c50..8b7a89dd 100644 --- a/src/sse/handlers/chat.js +++ b/src/sse/handlers/chat.js @@ -7,7 +7,8 @@ import { extractApiKey, isValidApiKey, } from "../services/auth.js"; -import { getSettings } from "@/lib/localDb"; +import { getSettings, getCustomModels } from "@/lib/localDb"; +import { capabilitiesFromServiceKind } from "open-sse/providers/capabilities.js"; import { getModelInfo, getComboModels } from "../services/model.js"; import { handleChatCore } from "open-sse/handlers/chatCore.js"; import { DEFAULT_HEADROOM_URL } from "@/lib/headroom/detect"; @@ -259,6 +260,13 @@ async function handleSingleModelChat(body, modelStr, clientRawRequest = null, re // Use shared chatCore const chatSettings = await getSettings(); const providerThinking = (chatSettings.providerThinking || {})[provider] || null; + // User-registered models (Provider page → "Add Model") trust upstream: keep + // images/media in the request instead of stripping by static-capability guess. + let capsOverride = null; + try { + const cm = (await getCustomModels()).find((m) => m.providerAlias === provider && m.id === model); + if (cm) capsOverride = { ...capabilitiesFromServiceKind(cm.type), vision: true }; + } catch { /* fail-open to static caps */ } const result = await handleChatCore({ body: { ...body, model: `${provider}/${model}` }, modelInfo: { provider, model }, @@ -284,6 +292,7 @@ async function handleSingleModelChat(body, modelStr, clientRawRequest = null, re pxpipeTransform: chatSettings.pxpipeEnabled ? await getPxpipeTransform() : null, onPxpipeEvent: appendPxpipeEvent, providerThinking, + capsOverride, // Detect source format by endpoint + body sourceFormatOverride: request?.url ? detectFormatByEndpoint(new URL(request.url).pathname, body) : null, onCredentialsRefreshed: async (newCreds) => {