diff --git a/open-sse/services/combo.js b/open-sse/services/combo.js index 9216ab2f..fdfc189a 100644 --- a/open-sse/services/combo.js +++ b/open-sse/services/combo.js @@ -106,15 +106,32 @@ export function detectRequiredCapabilities(body) { const required = new Set(); if (!body || typeof body !== "object") return required; + const addByMime = (mime) => { + if (typeof mime !== "string") return; + if (mime.startsWith("image/")) required.add("vision"); + else if (mime === "application/pdf") required.add("pdf"); + else if (mime.startsWith("audio/")) required.add("audioInput"); + else if (mime.startsWith("video/")) required.add("videoInput"); + }; + const scanBlock = (b) => { if (!b || typeof b !== "object") return; const t = b.type; if (t === "image_url" || t === "image" || t === "input_image") required.add("vision"); - if (t === "file" || t === "document" || t === "input_file") required.add("pdf"); + if (t === "input_audio" || t === "audio_url" || t === "audio") required.add("audioInput"); + if (t === "input_video" || t === "video_url" || t === "video") required.add("videoInput"); + if (t === "file" || t === "document" || t === "input_file") { + // Infer modality from embedded mime when available; fall back to pdf for generic files. + let fmime = null; + if (b.input_audio?.format) fmime = `audio/${b.input_audio.format}`; + else if (b.file?.file_data) fmime = String(b.file.file_data).match(/^data:([^;,]+)/)?.[1]; + else if (b.source?.media_type) fmime = b.source.media_type; + else if (b.source?.data) fmime = String(b.source.data).match(/^data:([^;,]+)/)?.[1]; + if (fmime) addByMime(fmime); + else required.add("pdf"); + } // gemini parts: inlineData/fileData carry a mime - const mime = b.inlineData?.mimeType || b.fileData?.mimeType; - if (typeof mime === "string" && mime.startsWith("image/")) required.add("vision"); - if (mime === "application/pdf") required.add("pdf"); + addByMime(b.inlineData?.mimeType || b.fileData?.mimeType); }; const scanContent = (content) => { diff --git a/src/sse/handlers/chat.js b/src/sse/handlers/chat.js index b44727ab..88353c50 100644 --- a/src/sse/handlers/chat.js +++ b/src/sse/handlers/chat.js @@ -14,7 +14,8 @@ import { DEFAULT_HEADROOM_URL } from "@/lib/headroom/detect"; import { getTransform as getPxpipeTransform } from "@/lib/pxpipe/loader.js"; import { appendPxpipeEvent } from "@/lib/pxpipe/events.js"; import { errorResponse, unavailableResponse } from "open-sse/utils/error.js"; -import { handleComboChat, handleFusionChat } from "open-sse/services/combo.js"; +import { handleComboChat, handleFusionChat, detectRequiredCapabilities } from "open-sse/services/combo.js"; +import { augmentModelsWithCapacityAdapter, withCapacityAdapterStripping, getActiveAdapterStrategy } from "open-sse/services/capacityAdapter.js"; import { handleBypassRequest } from "open-sse/utils/bypassHandler.js"; import { HTTP_STATUS } from "open-sse/config/runtimeConfig.js"; import { detectFormatByEndpoint } from "open-sse/translator/formats.js"; @@ -83,6 +84,8 @@ export async function handleChat(request, clientRawRequest = null) { const bypassResponse = handleBypassRequest(body, modelStr, userAgent, !!settings.ccFilterNaming); if (bypassResponse) return bypassResponse.response || bypassResponse; + const requiredCapabilities = detectRequiredCapabilities(body); + // Check if model is a combo (has multiple models with fallback) const comboModels = await getComboModels(modelStr); if (comboModels) { @@ -90,6 +93,8 @@ export async function handleChat(request, clientRawRequest = null) { const comboStrategies = settings.comboStrategies || {}; const comboSpecificStrategy = comboStrategies[modelStr]?.fallbackStrategy; const comboStrategy = comboSpecificStrategy || settings.comboStrategy || "fallback"; + const augmentedModels = augmentModelsWithCapacityAdapter(comboModels, requiredCapabilities, settings); + const adapterAdded = augmentedModels.filter((m) => !comboModels.includes(m)); if (comboStrategy === "fusion") { log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models (strategy: fusion)`); @@ -112,11 +117,14 @@ export async function handleChat(request, clientRawRequest = null) { } const comboStickyLimit = settings.comboStickyRoundRobinLimit; - log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models (strategy: ${comboStrategy}, sticky: ${comboStickyLimit})`); + log.info("CHAT", `Combo "${modelStr}" with ${augmentedModels.length} models (strategy: ${comboStrategy}, sticky: ${comboStickyLimit})`); return handleComboChat({ body, - models: comboModels, - handleSingleModel: (b, m) => handleSingleModelChat(b, m, clientRawRequest, request, apiKey), + models: augmentedModels, + handleSingleModel: withCapacityAdapterStripping( + (b, m) => handleSingleModelChat(b, m, clientRawRequest, request, apiKey), + adapterAdded + ), log, comboName: modelStr, comboStrategy, @@ -124,7 +132,25 @@ export async function handleChat(request, clientRawRequest = null) { }); } - // Single model request + // Single model request — may still switch to a capacity-adapter model if the + // target lacks a capability the request needs (e.g. no vision, request has an image). + const soloAugmented = augmentModelsWithCapacityAdapter([modelStr], requiredCapabilities, settings); + if (soloAugmented.length > 1) { + const adapterAdded = soloAugmented.filter((m) => m !== modelStr); + log.info("CHAT", `Capacity adapter for [${[...requiredCapabilities].join(",")}] on "${modelStr}" → trying ${soloAugmented.join(", ")}`); + return handleComboChat({ + body, + models: soloAugmented, + handleSingleModel: withCapacityAdapterStripping( + (b, m) => handleSingleModelChat(b, m, clientRawRequest, request, apiKey), + adapterAdded + ), + log, + comboName: modelStr, + comboStrategy: getActiveAdapterStrategy(requiredCapabilities, settings) + }); + } + return handleSingleModelChat(body, modelStr, clientRawRequest, request, apiKey); } @@ -143,6 +169,9 @@ async function handleSingleModelChat(body, modelStr, clientRawRequest = null, re const comboStrategies = chatSettings.comboStrategies || {}; const comboSpecificStrategy = comboStrategies[modelStr]?.fallbackStrategy; const comboStrategy = comboSpecificStrategy || chatSettings.comboStrategy || "fallback"; + const requiredCapabilities = detectRequiredCapabilities(body); + const augmentedModels = augmentModelsWithCapacityAdapter(comboModels, requiredCapabilities, chatSettings); + const adapterAdded = augmentedModels.filter((m) => !comboModels.includes(m)); if (comboStrategy === "fusion") { log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models (strategy: fusion)`); @@ -165,11 +194,14 @@ async function handleSingleModelChat(body, modelStr, clientRawRequest = null, re } const comboStickyLimit = chatSettings.comboStickyRoundRobinLimit; - log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models (strategy: ${comboStrategy}, sticky: ${comboStickyLimit})`); + log.info("CHAT", `Combo "${modelStr}" with ${augmentedModels.length} models (strategy: ${comboStrategy}, sticky: ${comboStickyLimit})`); return handleComboChat({ body, - models: comboModels, - handleSingleModel: (b, m) => handleSingleModelChat(b, m, clientRawRequest, request, apiKey), + models: augmentedModels, + handleSingleModel: withCapacityAdapterStripping( + (b, m) => handleSingleModelChat(b, m, clientRawRequest, request, apiKey), + adapterAdded + ), log, comboName: modelStr, comboStrategy,