From 333e704b2afd3ff414809658167e751c368364a6 Mon Sep 17 00:00:00 2001 From: decolua Date: Sat, 4 Apr 2026 23:24:24 +0700 Subject: [PATCH] MODEL_CAPS --- open-sse/config/providerModels.js | 191 +++++++++++++++++- open-sse/handlers/chatCore.js | 5 +- open-sse/translator/index.js | 29 ++- .../translator/request/openai-to-gemini.js | 10 +- open-sse/translator/request/openai-to-kiro.js | 4 +- .../dashboard/endpoint/EndpointPageClient.js | 8 +- src/shared/constants/models.js | 2 + 7 files changed, 239 insertions(+), 10 deletions(-) diff --git a/open-sse/config/providerModels.js b/open-sse/config/providerModels.js index 12b08ded..e5496aa0 100644 --- a/open-sse/config/providerModels.js +++ b/open-sse/config/providerModels.js @@ -1,5 +1,191 @@ import { PROVIDERS } from "./providers.js"; +// Global model capabilities registry +// Only define models that have non-default capabilities +// Default: { thinking: false, multimodal: { image: false, audio: false, video: false, pdf: false } } +export const MODEL_CAPS = { + // Claude models — full capabilities + "claude-opus-4-6": { thinking: true, multimodal: { image: true, pdf: true } }, + "claude-sonnet-4-6": { thinking: true, multimodal: { image: true, pdf: true } }, + "claude-opus-4-6-thinking": { thinking: true, multimodal: { image: true, pdf: true } }, + "claude-opus-4-5-20251101": { thinking: true, multimodal: { image: true, pdf: true } }, + "claude-sonnet-4-5-20250929": { thinking: true, multimodal: { image: true, pdf: true } }, + "claude-haiku-4-5-20251001": { multimodal: { image: true } }, + "claude-sonnet-4-20250514": { thinking: true, multimodal: { image: true, pdf: true } }, + "claude-opus-4-20250514": { thinking: true, multimodal: { image: true, pdf: true } }, + "claude-3-5-sonnet-20241022": { multimodal: { image: true } }, + // Cursor Claude aliases + "claude-4.5-opus-high-thinking": { thinking: true, multimodal: { image: true } }, + "claude-4.5-opus-high": { multimodal: { image: true } }, + "claude-4.5-sonnet-thinking": { thinking: true, multimodal: { image: true } }, + "claude-4.5-sonnet": { multimodal: { image: true } }, + "claude-4.5-haiku": { multimodal: { image: true } }, + "claude-4.5-opus": { multimodal: { image: true } }, + "claude-4.6-opus-max": { thinking: true, multimodal: { image: true } }, + "claude-4.6-sonnet-medium-thinking": { thinking: true, multimodal: { image: true } }, + // GitHub Copilot Claude aliases + "claude-haiku-4.5": { multimodal: { image: true } }, + "claude-opus-4.1": { thinking: true, multimodal: { image: true } }, + "claude-opus-4.5": { thinking: true, multimodal: { image: true } }, + "claude-sonnet-4": { multimodal: { image: true } }, + "claude-sonnet-4.5": { thinking: true, multimodal: { image: true } }, + "claude-sonnet-4.6": { thinking: true, multimodal: { image: true } }, + "claude-opus-4.6": { thinking: true, multimodal: { image: true } }, + // Kiro aliases + "claude-sonnet-4.5": { thinking: true, multimodal: { image: true } }, + + // Gemini models — full multimodal + "gemini-3.1-pro-preview": { thinking: true, multimodal: { image: true, audio: true, video: true, pdf: true } }, + "gemini-3.1-flash-lite-preview": { multimodal: { image: true, audio: true, video: true } }, + "gemini-3.1-flash-image-preview": { multimodal: { image: true, audio: true, video: true } }, + "gemini-3-flash-preview": { thinking: true, multimodal: { image: true, audio: true, video: true } }, + "gemini-3-pro-preview": { thinking: true, multimodal: { image: true, audio: true, video: true, pdf: true } }, + "gemini-3-flash": { thinking: true, multimodal: { image: true, audio: true, video: true } }, + "gemini-3.1-pro-high": { thinking: true, multimodal: { image: true, audio: true, video: true, pdf: true } }, + "gemini-3.1-pro-low": { thinking: true, multimodal: { image: true, audio: true, video: true, pdf: true } }, + "gemini-2.5-pro": { thinking: true, multimodal: { image: true, audio: true, video: true, pdf: true } }, + "gemini-2.5-flash": { thinking: true, multimodal: { image: true, audio: true, video: true } }, + "gemini-2.5-flash-lite": { multimodal: { image: true } }, + "gemini-2.0-flash": { multimodal: { image: true, audio: true, video: true } }, + "gemini-2.0-flash-lite": { multimodal: { image: true } }, + + // GPT models + "gpt-5.4": { multimodal: { image: true } }, + "gpt-5.4-mini": { multimodal: { image: true } }, + "gpt-5.3-codex": { thinking: true, multimodal: { image: true } }, + "gpt-5.3-codex-xhigh": { thinking: true }, + "gpt-5.3-codex-high": { thinking: true }, + "gpt-5.3-codex-low": { thinking: true }, + "gpt-5.3-codex-none": {}, + "gpt-5.3-codex-spark": {}, + "gpt-5.2-codex": { thinking: true }, + "gpt-5.2": { multimodal: { image: true } }, + "gpt-5.1-codex": { thinking: true }, + "gpt-5.1-codex-mini": { thinking: true }, + "gpt-5.1-codex-high": { thinking: true }, + "gpt-5.1-codex-max": { thinking: true }, + "gpt-5.1": { multimodal: { image: true } }, + "gpt-5-codex": { thinking: true }, + "gpt-5-codex-mini": {}, + "gpt-5": { multimodal: { image: true } }, + "gpt-5-mini": { multimodal: { image: true } }, + "gpt-4o": { multimodal: { image: true, audio: true } }, + "gpt-4o-mini": { multimodal: { image: true } }, + "gpt-4-turbo": { multimodal: { image: true } }, + "gpt-4.1": { multimodal: { image: true } }, + "gpt-4.1-mini": { multimodal: { image: true } }, + "gpt-4.1-nano": {}, + "o3": { thinking: true, multimodal: { image: true } }, + "o3-mini": { thinking: true }, + "o3-pro": { thinking: true, multimodal: { image: true } }, + "o4-mini": { thinking: true, multimodal: { image: true } }, + "o1": { thinking: true, multimodal: { image: true } }, + "o1-mini": { thinking: true }, + + // DeepSeek models + "deepseek-chat": {}, + "deepseek-reasoner": { thinking: true }, + "deepseek-r1": { thinking: true }, + "deepseek-v3": {}, + "deepseek-v3.1": {}, + "deepseek-v3.2": {}, + "deepseek-3.1": {}, + "deepseek-3.2": {}, + "deepseek-ai/DeepSeek-R1": { thinking: true }, + "deepseek-ai/DeepSeek-V3": {}, + "deepseek-ai/DeepSeek-V3.2": {}, + "deepseek-ai/DeepSeek-V3.1": {}, + "deepseek-ai/deepseek-v3.2-maas": {}, + + // Qwen models + "qwen3-vl-plus": { multimodal: { image: true } }, + "vision-model": { multimodal: { image: true } }, + "qwen3-coder-plus": {}, + "qwen3-coder-flash": {}, + "qwen3-max": { thinking: true }, + "qwen3-max-preview": { thinking: true }, + "qwen3-235b": { thinking: true }, + "qwen3-235b-a22b-instruct": {}, + "qwen3-235b-a22b-thinking-2507": { thinking: true }, + "qwen3-32b": { thinking: true }, + "qwen3-coder-next": {}, + "qwen3.5-plus": {}, + "qwen/qwen3-32b": { thinking: true }, + "qwen/qwen3-next-80b-a3b-thinking-maas": { thinking: true }, + "qwen/qwen3-next-80b-a3b-instruct-maas": {}, + "Qwen/Qwen3-235B-A22B": { thinking: true }, + "Qwen/Qwen3-235B-A22B-Instruct-2507": {}, + "Qwen/Qwen3-Coder-480B-A35B-Instruct": {}, + "Qwen/Qwen3-32B": { thinking: true }, + "qwen-3-235b-a22b-instruct-2507": {}, + "qwen-3-32b": { thinking: true }, + + // Kimi models + "kimi-k2": {}, + "kimi-k2.5": {}, + "kimi-k2.5-thinking": { thinking: true }, + "kimi-latest": {}, + "moonshotai/Kimi-K2.5": {}, + "moonshotai/kimi-k2.5": {}, + + // GLM models + "glm-5.1": {}, + "glm-5": {}, + "glm-4.7": {}, + "glm-4.6v": { multimodal: { image: true } }, + "glm-4.6": {}, + "glm-4.5-air": {}, + "glm-4.7-flash": {}, + "z-ai/glm4.7": {}, + "zai-org/GLM-4.7": {}, + "zai-glm-4.7": {}, + "zai-org/glm-5-maas": {}, + + // Grok models + "grok-4": { thinking: true, multimodal: { image: true } }, + "grok-4-fast-reasoning": { thinking: true }, + "grok-code-fast-1": {}, + "grok-3": { multimodal: { image: true } }, + + // GPT-OSS (no toolUse) + "gpt-oss-120b": {}, + "gpt-oss-120b-medium": {}, + "openai/gpt-oss-120b": {}, + "gpt-oss:120b": {}, + + // Llama models + "meta-llama/Llama-3.3-70B-Instruct-Turbo": {}, + "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": { multimodal: { image: true } }, + "meta-llama/llama-4-maverick-17b-128e-instruct": { multimodal: { image: true } }, + "meta-llama/Llama-3.3-70B-Instruct": {}, + "meta-llama/Llama-3.2-3B-Instruct": {}, + "llama-3.3-70b-versatile": {}, + "llama-3.3-70b": {}, + "llama-4-scout-17b-16e-instruct": { multimodal: { image: true } }, +}; + +// Default capabilities for unknown models +const DEFAULT_CAPS = { thinking: false, multimodal: { image: false, audio: false, video: false, pdf: false } }; + +// Merge caps: global as base, provider entry overrides +function mergeCaps(global, override) { + if (!override) return global; + return { + thinking: override.thinking ?? global.thinking, + multimodal: { ...global.multimodal, ...override.multimodal } + }; +} + +// Resolve model capabilities: provider override → global → default +export function getModelCaps(alias, modelId) { + const entry = PROVIDER_MODELS[alias]?.find(m => m.id === modelId); + const global = MODEL_CAPS[modelId] ?? DEFAULT_CAPS; + // Extract caps fields from entry (exclude id, name, type, targetFormat) + const { id, name, type, targetFormat, ...overrideCaps } = entry || {}; + const hasOverride = Object.keys(overrideCaps).length > 0; + return mergeCaps({ ...DEFAULT_CAPS, ...global }, hasOverride ? overrideCaps : null); +} + // Provider models - Single source of truth // Key = alias (cc, cx, gc, qw, if, ag, gh for OAuth; id for API Key) // Field "provider" for special cases (e.g. AntiGravity models that call different backends) @@ -65,7 +251,7 @@ export const PROVIDER_MODELS = { ag: [ // Antigravity - special case: models call different backends { id: "gemini-3.1-pro-high", name: "Gemini 3 Pro High" }, { id: "gemini-3.1-pro-low", name: "Gemini 3 Pro Low" }, - { id: "gemini-3-flash", name: "Gemini 3 Flash" }, + { id: "gemini-3-flash", name: "Gemini 3 Flash", thinking: false }, // AG strips thinking for this model { id: "claude-sonnet-4-6", name: "Claude Sonnet 4.6" }, { id: "claude-opus-4-6-thinking", name: "Claude Opus 4.6 Thinking" }, { id: "gpt-oss-120b-medium", name: "GPT OSS 120B Medium" }, @@ -418,3 +604,6 @@ export function getModelsByProviderId(providerId) { const alias = PROVIDER_ID_TO_ALIAS[providerId] || providerId; return PROVIDER_MODELS[alias] || []; } + +// Re-export getModelCaps here for convenience (defined above PROVIDER_MODELS) +// getModelCaps is already exported above diff --git a/open-sse/handlers/chatCore.js b/open-sse/handlers/chatCore.js index b61ed427..5a0193a5 100644 --- a/open-sse/handlers/chatCore.js +++ b/open-sse/handlers/chatCore.js @@ -5,7 +5,7 @@ import { COLORS } from "../utils/stream.js"; import { createStreamController } from "../utils/streamHandler.js"; import { refreshWithRetry } from "../services/tokenRefresh.js"; import { createRequestLogger } from "../utils/requestLogger.js"; -import { getModelTargetFormat, PROVIDER_ID_TO_ALIAS } from "../config/providerModels.js"; +import { getModelTargetFormat, getModelCaps, PROVIDER_ID_TO_ALIAS } from "../config/providerModels.js"; import { createErrorResult, parseUpstreamError, formatProviderError } from "../utils/error.js"; import { HTTP_STATUS } from "../config/runtimeConfig.js"; import { handleBypassRequest } from "../utils/bypassHandler.js"; @@ -36,6 +36,7 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred const alias = PROVIDER_ID_TO_ALIAS[provider] || provider; const modelTargetFormat = getModelTargetFormat(alias, model); const targetFormat = modelTargetFormat || getTargetFormat(provider); + const modelCaps = getModelCaps(alias, model); const clientRequestedStreaming = body.stream === true || sourceFormat === FORMATS.ANTIGRAVITY || sourceFormat === FORMATS.GEMINI || sourceFormat === FORMATS.GEMINI_CLI; const providerRequiresStreaming = provider === "openai" || provider === "codex"; @@ -55,7 +56,7 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred reqLogger.logRawRequest(body); log?.debug?.("FORMAT", `${sourceFormat} → ${targetFormat} | stream=${stream}`); - let translatedBody = translateRequest(sourceFormat, targetFormat, model, body, stream, credentials, provider, reqLogger); + let translatedBody = translateRequest(sourceFormat, targetFormat, model, body, stream, credentials, provider, reqLogger, modelCaps); if (!translatedBody) { trackPendingRequest(model, provider, connectionId, false, true); return createErrorResult(HTTP_STATUS.BAD_REQUEST, `Failed to translate request for ${sourceFormat} → ${targetFormat}`); diff --git a/open-sse/translator/index.js b/open-sse/translator/index.js index b29686ac..dc6fa4b5 100644 --- a/open-sse/translator/index.js +++ b/open-sse/translator/index.js @@ -51,11 +51,38 @@ function ensureInitialized() { require("./response/ollama-to-openai.js"); } +// Strip multimodal content blocks (image/audio/video) from messages if model doesn't support them +function stripUnsupportedMultimodal(body, multimodal = {}) { + if (!body.messages || !Array.isArray(body.messages)) return; + for (const msg of body.messages) { + if (!Array.isArray(msg.content)) continue; + msg.content = msg.content.filter(part => { + if (part.type === "image_url" || part.type === "image") return multimodal.image === true; + if (part.type === "audio_url" || part.type === "input_audio") return multimodal.audio === true; + return true; // keep text, tool_use, tool_result, etc. + }); + // If content array becomes empty after filtering, replace with empty string to avoid API errors + if (msg.content.length === 0) msg.content = ""; + } +} + // Translate request: source -> openai -> target -export function translateRequest(sourceFormat, targetFormat, model, body, stream = true, credentials = null, provider = null, reqLogger = null) { +export function translateRequest(sourceFormat, targetFormat, model, body, stream = true, credentials = null, provider = null, reqLogger = null, caps = null) { ensureInitialized(); let result = body; + // Apply model capability guards before translation + if (caps) { + // Strip multimodal content if model doesn't support it + stripUnsupportedMultimodal(result, caps.multimodal || {}); + + // Strip thinking config if model doesn't support thinking + if (!caps.thinking) { + delete result.thinking; + delete result.reasoning_effort; + } + } + // Normalize thinking config: remove if lastMessage is not user normalizeThinkingConfig(result); diff --git a/open-sse/translator/request/openai-to-gemini.js b/open-sse/translator/request/openai-to-gemini.js index bfa84d13..c76895ee 100644 --- a/open-sse/translator/request/openai-to-gemini.js +++ b/open-sse/translator/request/openai-to-gemini.js @@ -428,11 +428,15 @@ function wrapInCloudCodeEnvelopeForClaude(model, claudeRequest, credentials = nu return envelope; } +// Detect if model should use Claude backend in Antigravity +// Claude models have specific ID patterns — more reliable than caps at routing level +function isClaudeModel(model) { + return model.toLowerCase().includes("claude"); +} + // OpenAI -> Antigravity (Sandbox Cloud Code with wrapper) export function openaiToAntigravityRequest(model, body, stream, credentials = null) { - const isClaude = model.toLowerCase().includes("claude"); - - if (isClaude) { + if (isClaudeModel(model)) { const claudeRequest = openaiToClaudeRequestForAntigravity(model, body, stream); return wrapInCloudCodeEnvelopeForClaude(model, claudeRequest, credentials); } diff --git a/open-sse/translator/request/openai-to-kiro.js b/open-sse/translator/request/openai-to-kiro.js index f07259ca..add9e88f 100644 --- a/open-sse/translator/request/openai-to-kiro.js +++ b/open-sse/translator/request/openai-to-kiro.js @@ -20,8 +20,8 @@ function convertMessages(messages, tools, model) { let pendingImages = []; let currentRole = null; - // Only Claude models support images in Kiro - const supportsImages = model && model.toLowerCase().includes("claude"); + // Image support is pre-filtered by caps in translateRequest before reaching here + const supportsImages = true; const flushPending = () => { if (currentRole === "user") { diff --git a/src/app/(dashboard)/dashboard/endpoint/EndpointPageClient.js b/src/app/(dashboard)/dashboard/endpoint/EndpointPageClient.js index a48eaa14..5b192563 100644 --- a/src/app/(dashboard)/dashboard/endpoint/EndpointPageClient.js +++ b/src/app/(dashboard)/dashboard/endpoint/EndpointPageClient.js @@ -446,7 +446,13 @@ export default function APIPageClient({ machineId }) {