feat(caps): user-registered models trust upstream vision instead of stripping media
This commit is contained in:
@@ -57,7 +57,7 @@ export function stripContinuityFields(body) {
|
||||
return body;
|
||||
}
|
||||
|
||||
export async function handleChatCore({ body, modelInfo, credentials, log, onCredentialsRefreshed, onRequestSuccess, onDisconnect, clientRawRequest, connectionId, userAgent, apiKey, ccFilterNaming, rtkEnabled, headroomEnabled, headroomUrl, headroomCompressUserMessages, cavemanEnabled, cavemanLevel, ponytailEnabled, ponytailLevel, pxpipeEnabled, pxpipeMinChars, pxpipeTimeoutMs, pxpipeTransform, onPxpipeEvent, sourceFormatOverride, providerThinking }) {
|
||||
export async function handleChatCore({ body, modelInfo, credentials, log, onCredentialsRefreshed, onRequestSuccess, onDisconnect, clientRawRequest, connectionId, userAgent, apiKey, ccFilterNaming, rtkEnabled, headroomEnabled, headroomUrl, headroomCompressUserMessages, cavemanEnabled, cavemanLevel, ponytailEnabled, ponytailLevel, pxpipeEnabled, pxpipeMinChars, pxpipeTimeoutMs, pxpipeTransform, onPxpipeEvent, sourceFormatOverride, providerThinking, capsOverride }) {
|
||||
const { provider, model } = modelInfo;
|
||||
const requestStartTime = Date.now();
|
||||
// Stable per-session color so all lines of one CLI conversation share a tag
|
||||
@@ -149,8 +149,10 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
|
||||
if (credentials) credentials.rawHeaders = clientRawRequest?.headers || {};
|
||||
|
||||
// Auto-strip media blocks the model can't read (vision/audio/pdf) before translation.
|
||||
// capsOverride lets the app layer assert per-model capabilities (e.g. user-registered
|
||||
// models) on top of the static tables.
|
||||
if (!passthrough) {
|
||||
const caps = getCapabilitiesForModel(provider, model);
|
||||
const caps = { ...getCapabilitiesForModel(provider, model), ...(capsOverride || {}) };
|
||||
if (stripUnsupportedModalities(body, sourceFormat, caps)) {
|
||||
log?.debug?.("MODALITY", `stripped unsupported media for ${provider}/${model}`);
|
||||
}
|
||||
|
||||
@@ -317,6 +317,11 @@ export const PATTERN_CAPABILITIES = [
|
||||
{ pattern: "*ling-*", caps: { reasoning: true, contextWindow: 128000 } },
|
||||
];
|
||||
|
||||
// OpenRouter-style gateways validate modalities upstream — a text-only model
|
||||
// sent an image gets a clear upstream error instead of silent corruption. So for
|
||||
// unknown models on these providers, trust vision instead of stripping images.
|
||||
const TRUST_UPSTREAM_VISION = new Set(["openrouter"]);
|
||||
|
||||
/**
|
||||
* Resolve capabilities for a model using the 4-step fallback chain,
|
||||
* merged over DEFAULT_CAPABILITIES so the result is always complete.
|
||||
@@ -349,6 +354,9 @@ export function getCapabilitiesForModel(provider, model) {
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Floor
|
||||
// 4. Floor (upstream-validated gateways keep vision on for unknown models)
|
||||
if (provider && TRUST_UPSTREAM_VISION.has(provider)) {
|
||||
return { ...DEFAULT_CAPABILITIES, vision: true };
|
||||
}
|
||||
return { ...DEFAULT_CAPABILITIES };
|
||||
}
|
||||
|
||||
@@ -7,7 +7,8 @@ import {
|
||||
extractApiKey,
|
||||
isValidApiKey,
|
||||
} from "../services/auth.js";
|
||||
import { getSettings } from "@/lib/localDb";
|
||||
import { getSettings, getCustomModels } from "@/lib/localDb";
|
||||
import { capabilitiesFromServiceKind } from "open-sse/providers/capabilities.js";
|
||||
import { getModelInfo, getComboModels } from "../services/model.js";
|
||||
import { handleChatCore } from "open-sse/handlers/chatCore.js";
|
||||
import { DEFAULT_HEADROOM_URL } from "@/lib/headroom/detect";
|
||||
@@ -259,6 +260,13 @@ async function handleSingleModelChat(body, modelStr, clientRawRequest = null, re
|
||||
// Use shared chatCore
|
||||
const chatSettings = await getSettings();
|
||||
const providerThinking = (chatSettings.providerThinking || {})[provider] || null;
|
||||
// User-registered models (Provider page → "Add Model") trust upstream: keep
|
||||
// images/media in the request instead of stripping by static-capability guess.
|
||||
let capsOverride = null;
|
||||
try {
|
||||
const cm = (await getCustomModels()).find((m) => m.providerAlias === provider && m.id === model);
|
||||
if (cm) capsOverride = { ...capabilitiesFromServiceKind(cm.type), vision: true };
|
||||
} catch { /* fail-open to static caps */ }
|
||||
const result = await handleChatCore({
|
||||
body: { ...body, model: `${provider}/${model}` },
|
||||
modelInfo: { provider, model },
|
||||
@@ -284,6 +292,7 @@ async function handleSingleModelChat(body, modelStr, clientRawRequest = null, re
|
||||
pxpipeTransform: chatSettings.pxpipeEnabled ? await getPxpipeTransform() : null,
|
||||
onPxpipeEvent: appendPxpipeEvent,
|
||||
providerThinking,
|
||||
capsOverride,
|
||||
// Detect source format by endpoint + body
|
||||
sourceFormatOverride: request?.url ? detectFormatByEndpoint(new URL(request.url).pathname, body) : null,
|
||||
onCredentialsRefreshed: async (newCreds) => {
|
||||
|
||||
Reference in New Issue
Block a user