feat(caps): user-registered models trust upstream vision instead of stripping media
This commit is contained in:
@@ -57,7 +57,7 @@ export function stripContinuityFields(body) {
|
|||||||
return body;
|
return body;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function handleChatCore({ body, modelInfo, credentials, log, onCredentialsRefreshed, onRequestSuccess, onDisconnect, clientRawRequest, connectionId, userAgent, apiKey, ccFilterNaming, rtkEnabled, headroomEnabled, headroomUrl, headroomCompressUserMessages, cavemanEnabled, cavemanLevel, ponytailEnabled, ponytailLevel, pxpipeEnabled, pxpipeMinChars, pxpipeTimeoutMs, pxpipeTransform, onPxpipeEvent, sourceFormatOverride, providerThinking }) {
|
export async function handleChatCore({ body, modelInfo, credentials, log, onCredentialsRefreshed, onRequestSuccess, onDisconnect, clientRawRequest, connectionId, userAgent, apiKey, ccFilterNaming, rtkEnabled, headroomEnabled, headroomUrl, headroomCompressUserMessages, cavemanEnabled, cavemanLevel, ponytailEnabled, ponytailLevel, pxpipeEnabled, pxpipeMinChars, pxpipeTimeoutMs, pxpipeTransform, onPxpipeEvent, sourceFormatOverride, providerThinking, capsOverride }) {
|
||||||
const { provider, model } = modelInfo;
|
const { provider, model } = modelInfo;
|
||||||
const requestStartTime = Date.now();
|
const requestStartTime = Date.now();
|
||||||
// Stable per-session color so all lines of one CLI conversation share a tag
|
// Stable per-session color so all lines of one CLI conversation share a tag
|
||||||
@@ -149,8 +149,10 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
|
|||||||
if (credentials) credentials.rawHeaders = clientRawRequest?.headers || {};
|
if (credentials) credentials.rawHeaders = clientRawRequest?.headers || {};
|
||||||
|
|
||||||
// Auto-strip media blocks the model can't read (vision/audio/pdf) before translation.
|
// Auto-strip media blocks the model can't read (vision/audio/pdf) before translation.
|
||||||
|
// capsOverride lets the app layer assert per-model capabilities (e.g. user-registered
|
||||||
|
// models) on top of the static tables.
|
||||||
if (!passthrough) {
|
if (!passthrough) {
|
||||||
const caps = getCapabilitiesForModel(provider, model);
|
const caps = { ...getCapabilitiesForModel(provider, model), ...(capsOverride || {}) };
|
||||||
if (stripUnsupportedModalities(body, sourceFormat, caps)) {
|
if (stripUnsupportedModalities(body, sourceFormat, caps)) {
|
||||||
log?.debug?.("MODALITY", `stripped unsupported media for ${provider}/${model}`);
|
log?.debug?.("MODALITY", `stripped unsupported media for ${provider}/${model}`);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -317,6 +317,11 @@ export const PATTERN_CAPABILITIES = [
|
|||||||
{ pattern: "*ling-*", caps: { reasoning: true, contextWindow: 128000 } },
|
{ pattern: "*ling-*", caps: { reasoning: true, contextWindow: 128000 } },
|
||||||
];
|
];
|
||||||
|
|
||||||
|
// OpenRouter-style gateways validate modalities upstream — a text-only model
|
||||||
|
// sent an image gets a clear upstream error instead of silent corruption. So for
|
||||||
|
// unknown models on these providers, trust vision instead of stripping images.
|
||||||
|
const TRUST_UPSTREAM_VISION = new Set(["openrouter"]);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resolve capabilities for a model using the 4-step fallback chain,
|
* Resolve capabilities for a model using the 4-step fallback chain,
|
||||||
* merged over DEFAULT_CAPABILITIES so the result is always complete.
|
* merged over DEFAULT_CAPABILITIES so the result is always complete.
|
||||||
@@ -349,6 +354,9 @@ export function getCapabilitiesForModel(provider, model) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 4. Floor
|
// 4. Floor (upstream-validated gateways keep vision on for unknown models)
|
||||||
|
if (provider && TRUST_UPSTREAM_VISION.has(provider)) {
|
||||||
|
return { ...DEFAULT_CAPABILITIES, vision: true };
|
||||||
|
}
|
||||||
return { ...DEFAULT_CAPABILITIES };
|
return { ...DEFAULT_CAPABILITIES };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,7 +7,8 @@ import {
|
|||||||
extractApiKey,
|
extractApiKey,
|
||||||
isValidApiKey,
|
isValidApiKey,
|
||||||
} from "../services/auth.js";
|
} from "../services/auth.js";
|
||||||
import { getSettings } from "@/lib/localDb";
|
import { getSettings, getCustomModels } from "@/lib/localDb";
|
||||||
|
import { capabilitiesFromServiceKind } from "open-sse/providers/capabilities.js";
|
||||||
import { getModelInfo, getComboModels } from "../services/model.js";
|
import { getModelInfo, getComboModels } from "../services/model.js";
|
||||||
import { handleChatCore } from "open-sse/handlers/chatCore.js";
|
import { handleChatCore } from "open-sse/handlers/chatCore.js";
|
||||||
import { DEFAULT_HEADROOM_URL } from "@/lib/headroom/detect";
|
import { DEFAULT_HEADROOM_URL } from "@/lib/headroom/detect";
|
||||||
@@ -259,6 +260,13 @@ async function handleSingleModelChat(body, modelStr, clientRawRequest = null, re
|
|||||||
// Use shared chatCore
|
// Use shared chatCore
|
||||||
const chatSettings = await getSettings();
|
const chatSettings = await getSettings();
|
||||||
const providerThinking = (chatSettings.providerThinking || {})[provider] || null;
|
const providerThinking = (chatSettings.providerThinking || {})[provider] || null;
|
||||||
|
// User-registered models (Provider page → "Add Model") trust upstream: keep
|
||||||
|
// images/media in the request instead of stripping by static-capability guess.
|
||||||
|
let capsOverride = null;
|
||||||
|
try {
|
||||||
|
const cm = (await getCustomModels()).find((m) => m.providerAlias === provider && m.id === model);
|
||||||
|
if (cm) capsOverride = { ...capabilitiesFromServiceKind(cm.type), vision: true };
|
||||||
|
} catch { /* fail-open to static caps */ }
|
||||||
const result = await handleChatCore({
|
const result = await handleChatCore({
|
||||||
body: { ...body, model: `${provider}/${model}` },
|
body: { ...body, model: `${provider}/${model}` },
|
||||||
modelInfo: { provider, model },
|
modelInfo: { provider, model },
|
||||||
@@ -284,6 +292,7 @@ async function handleSingleModelChat(body, modelStr, clientRawRequest = null, re
|
|||||||
pxpipeTransform: chatSettings.pxpipeEnabled ? await getPxpipeTransform() : null,
|
pxpipeTransform: chatSettings.pxpipeEnabled ? await getPxpipeTransform() : null,
|
||||||
onPxpipeEvent: appendPxpipeEvent,
|
onPxpipeEvent: appendPxpipeEvent,
|
||||||
providerThinking,
|
providerThinking,
|
||||||
|
capsOverride,
|
||||||
// Detect source format by endpoint + body
|
// Detect source format by endpoint + body
|
||||||
sourceFormatOverride: request?.url ? detectFormatByEndpoint(new URL(request.url).pathname, body) : null,
|
sourceFormatOverride: request?.url ? detectFormatByEndpoint(new URL(request.url).pathname, body) : null,
|
||||||
onCredentialsRefreshed: async (newCreds) => {
|
onCredentialsRefreshed: async (newCreds) => {
|
||||||
|
|||||||
Reference in New Issue
Block a user