feat(caps): user-registered models trust upstream vision instead of stripping media

This commit is contained in:
2026-08-22 14:33:53 +07:00
parent 144dda2ac2
commit bb8d67ba9c
3 changed files with 23 additions and 4 deletions

View File

@@ -57,7 +57,7 @@ export function stripContinuityFields(body) {
return body; return body;
} }
export async function handleChatCore({ body, modelInfo, credentials, log, onCredentialsRefreshed, onRequestSuccess, onDisconnect, clientRawRequest, connectionId, userAgent, apiKey, ccFilterNaming, rtkEnabled, headroomEnabled, headroomUrl, headroomCompressUserMessages, cavemanEnabled, cavemanLevel, ponytailEnabled, ponytailLevel, pxpipeEnabled, pxpipeMinChars, pxpipeTimeoutMs, pxpipeTransform, onPxpipeEvent, sourceFormatOverride, providerThinking }) { export async function handleChatCore({ body, modelInfo, credentials, log, onCredentialsRefreshed, onRequestSuccess, onDisconnect, clientRawRequest, connectionId, userAgent, apiKey, ccFilterNaming, rtkEnabled, headroomEnabled, headroomUrl, headroomCompressUserMessages, cavemanEnabled, cavemanLevel, ponytailEnabled, ponytailLevel, pxpipeEnabled, pxpipeMinChars, pxpipeTimeoutMs, pxpipeTransform, onPxpipeEvent, sourceFormatOverride, providerThinking, capsOverride }) {
const { provider, model } = modelInfo; const { provider, model } = modelInfo;
const requestStartTime = Date.now(); const requestStartTime = Date.now();
// Stable per-session color so all lines of one CLI conversation share a tag // Stable per-session color so all lines of one CLI conversation share a tag
@@ -149,8 +149,10 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
if (credentials) credentials.rawHeaders = clientRawRequest?.headers || {}; if (credentials) credentials.rawHeaders = clientRawRequest?.headers || {};
// Auto-strip media blocks the model can't read (vision/audio/pdf) before translation. // Auto-strip media blocks the model can't read (vision/audio/pdf) before translation.
// capsOverride lets the app layer assert per-model capabilities (e.g. user-registered
// models) on top of the static tables.
if (!passthrough) { if (!passthrough) {
const caps = getCapabilitiesForModel(provider, model); const caps = { ...getCapabilitiesForModel(provider, model), ...(capsOverride || {}) };
if (stripUnsupportedModalities(body, sourceFormat, caps)) { if (stripUnsupportedModalities(body, sourceFormat, caps)) {
log?.debug?.("MODALITY", `stripped unsupported media for ${provider}/${model}`); log?.debug?.("MODALITY", `stripped unsupported media for ${provider}/${model}`);
} }

View File

@@ -317,6 +317,11 @@ export const PATTERN_CAPABILITIES = [
{ pattern: "*ling-*", caps: { reasoning: true, contextWindow: 128000 } }, { pattern: "*ling-*", caps: { reasoning: true, contextWindow: 128000 } },
]; ];
// OpenRouter-style gateways validate modalities upstream — a text-only model
// sent an image gets a clear upstream error instead of silent corruption. So for
// unknown models on these providers, trust vision instead of stripping images.
const TRUST_UPSTREAM_VISION = new Set(["openrouter"]);
/** /**
* Resolve capabilities for a model using the 4-step fallback chain, * Resolve capabilities for a model using the 4-step fallback chain,
* merged over DEFAULT_CAPABILITIES so the result is always complete. * merged over DEFAULT_CAPABILITIES so the result is always complete.
@@ -349,6 +354,9 @@ export function getCapabilitiesForModel(provider, model) {
} }
} }
// 4. Floor // 4. Floor (upstream-validated gateways keep vision on for unknown models)
if (provider && TRUST_UPSTREAM_VISION.has(provider)) {
return { ...DEFAULT_CAPABILITIES, vision: true };
}
return { ...DEFAULT_CAPABILITIES }; return { ...DEFAULT_CAPABILITIES };
} }

View File

@@ -7,7 +7,8 @@ import {
extractApiKey, extractApiKey,
isValidApiKey, isValidApiKey,
} from "../services/auth.js"; } from "../services/auth.js";
import { getSettings } from "@/lib/localDb"; import { getSettings, getCustomModels } from "@/lib/localDb";
import { capabilitiesFromServiceKind } from "open-sse/providers/capabilities.js";
import { getModelInfo, getComboModels } from "../services/model.js"; import { getModelInfo, getComboModels } from "../services/model.js";
import { handleChatCore } from "open-sse/handlers/chatCore.js"; import { handleChatCore } from "open-sse/handlers/chatCore.js";
import { DEFAULT_HEADROOM_URL } from "@/lib/headroom/detect"; import { DEFAULT_HEADROOM_URL } from "@/lib/headroom/detect";
@@ -259,6 +260,13 @@ async function handleSingleModelChat(body, modelStr, clientRawRequest = null, re
// Use shared chatCore // Use shared chatCore
const chatSettings = await getSettings(); const chatSettings = await getSettings();
const providerThinking = (chatSettings.providerThinking || {})[provider] || null; const providerThinking = (chatSettings.providerThinking || {})[provider] || null;
// User-registered models (Provider page → "Add Model") trust upstream: keep
// images/media in the request instead of stripping by static-capability guess.
let capsOverride = null;
try {
const cm = (await getCustomModels()).find((m) => m.providerAlias === provider && m.id === model);
if (cm) capsOverride = { ...capabilitiesFromServiceKind(cm.type), vision: true };
} catch { /* fail-open to static caps */ }
const result = await handleChatCore({ const result = await handleChatCore({
body: { ...body, model: `${provider}/${model}` }, body: { ...body, model: `${provider}/${model}` },
modelInfo: { provider, model }, modelInfo: { provider, model },
@@ -284,6 +292,7 @@ async function handleSingleModelChat(body, modelStr, clientRawRequest = null, re
pxpipeTransform: chatSettings.pxpipeEnabled ? await getPxpipeTransform() : null, pxpipeTransform: chatSettings.pxpipeEnabled ? await getPxpipeTransform() : null,
onPxpipeEvent: appendPxpipeEvent, onPxpipeEvent: appendPxpipeEvent,
providerThinking, providerThinking,
capsOverride,
// Detect source format by endpoint + body // Detect source format by endpoint + body
sourceFormatOverride: request?.url ? detectFormatByEndpoint(new URL(request.url).pathname, body) : null, sourceFormatOverride: request?.url ? detectFormatByEndpoint(new URL(request.url).pathname, body) : null,
onCredentialsRefreshed: async (newCreds) => { onCredentialsRefreshed: async (newCreds) => {