diff --git a/open-sse/services/combo.js b/open-sse/services/combo.js index b90b56dc..e3b9a8e6 100644 --- a/open-sse/services/combo.js +++ b/open-sse/services/combo.js @@ -138,8 +138,42 @@ export function detectRequiredCapabilities(body) { if (Array.isArray(content)) for (const b of content) scanBlock(b); }; + const scanMessage = (m) => { + if (!m || typeof m !== "object") return; + + // Ollama / Hermes images array (strings or objects) + if (Array.isArray(m.images) && m.images.length > 0) { + required.add("vision"); + } + + // Vercel AI SDK / Hermes attachments / experimental_attachments + const attachments = m.experimental_attachments || m.attachments; + if (Array.isArray(attachments)) { + for (const att of attachments) { + if (!att) continue; + const mime = att.contentType || att.mediaType || (typeof att.url === "string" && att.url.match(/^data:([^;,]+)/)?.[1]); + if (mime) addByMime(mime); + else if (att.url || att.data) required.add("vision"); + } + } + + // Direct message-level modality properties + if (m.image_url || m.image) required.add("vision"); + if (m.audio_url || m.audio) required.add("audioInput"); + + // Scan array content blocks + scanContent(m.content); + + // Scan string content for embedded data URIs + if (typeof m.content === "string") { + if (m.content.includes("data:image/")) required.add("vision"); + else if (m.content.includes("data:audio/")) required.add("audioInput"); + else if (m.content.includes("data:application/pdf")) required.add("pdf"); + } + }; + // Modalities: current user turn only (trailing user run across each known shape). - for (const m of trailingUserItems(body.messages)) scanContent(m.content); // openai / claude + for (const m of trailingUserItems(body.messages)) scanMessage(m); // openai / claude / hermes / ollama for (const it of trailingUserItems(body.input)) scanContent(it.content); // responses const contents = body.contents || body.request?.contents; // gemini / antigravity for (const c of trailingUserItems(contents)) scanContent(c.parts); diff --git a/open-sse/translator/concerns/modality.js b/open-sse/translator/concerns/modality.js index b3bba2de..d2d62f4e 100644 --- a/open-sse/translator/concerns/modality.js +++ b/open-sse/translator/concerns/modality.js @@ -62,6 +62,19 @@ function stripOpenAI(body, caps) { if (!Array.isArray(body.messages)) return; const last = body.messages.length - 1; body.messages.forEach((msg, i) => { + if (caps.vision === false) { + if (Array.isArray(msg.images)) delete msg.images; + if (Array.isArray(msg.experimental_attachments)) { + msg.experimental_attachments = msg.experimental_attachments.filter( + (a) => !(a?.contentType?.startsWith("image/") || (typeof a?.url === "string" && a.url.startsWith("data:image/"))) + ); + } + if (Array.isArray(msg.attachments)) { + msg.attachments = msg.attachments.filter( + (a) => !(a?.contentType?.startsWith("image/") || (typeof a?.url === "string" && a.url.startsWith("data:image/"))) + ); + } + } if (!Array.isArray(msg.content)) return; const removed = new Set(); msg.content = filterBlocks(msg.content, capForOpenAIBlock, caps, removed, i === last); diff --git a/tests/unit/hermes-vision-detection.test.js b/tests/unit/hermes-vision-detection.test.js new file mode 100644 index 00000000..40669614 --- /dev/null +++ b/tests/unit/hermes-vision-detection.test.js @@ -0,0 +1,115 @@ +import { describe, it, expect } from "vitest"; +import { detectRequiredCapabilities } from "../../open-sse/services/combo.js"; +import { augmentModelsWithCapacityAdapter } from "../../open-sse/services/capacityAdapter.js"; +import { stripUnsupportedModalities } from "../../open-sse/translator/concerns/modality.js"; +import { FORMATS } from "../../open-sse/translator/formats.js"; + +describe("Hermes Vision Image Detection", () => { + it("detects vision from Ollama / Hermes images array", () => { + const body = { + messages: [ + { + role: "user", + content: "Please analyze this image from Hermes", + images: ["iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="], + }, + ], + }; + const caps = detectRequiredCapabilities(body); + expect(caps.has("vision")).toBe(true); + }); + + it("detects vision from Vercel AI SDK / Hermes experimental_attachments", () => { + const body = { + messages: [ + { + role: "user", + content: "Describe this attachment", + experimental_attachments: [ + { + contentType: "image/png", + url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", + }, + ], + }, + ], + }; + const caps = detectRequiredCapabilities(body); + expect(caps.has("vision")).toBe(true); + }); + + it("detects vision from Hermes attachments array", () => { + const body = { + messages: [ + { + role: "user", + content: "Look at this photo", + attachments: [ + { + mediaType: "image/jpeg", + url: "https://example.com/photo.jpg", + }, + ], + }, + ], + }; + const caps = detectRequiredCapabilities(body); + expect(caps.has("vision")).toBe(true); + }); + + it("detects vision from embedded data:image URI in string content", () => { + const body = { + messages: [ + { + role: "user", + content: "Here is an inline image: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", + }, + ], + }; + const caps = detectRequiredCapabilities(body); + expect(caps.has("vision")).toBe(true); + }); + + it("auto-switches non-vision model (deepseek-v4-pro) to Vision Adapter model (Kimi-K3)", () => { + const body = { + messages: [ + { + role: "user", + content: "Analyze image", + images: ["base64data..."], + }, + ], + }; + const reqCaps = detectRequiredCapabilities(body); + const settings = { + capacityAdapter: { + vision: { + enabled: true, + models: ["cmc/moonshotai/Kimi-K3"], + }, + }, + }; + + const augmented = augmentModelsWithCapacityAdapter(["cmc/deepseek/deepseek-v4-pro"], reqCaps, settings); + expect(augmented).toEqual(["cmc/moonshotai/Kimi-K3", "cmc/deepseek/deepseek-v4-pro"]); + }); + + it("strips msg.images and attachments when model does not support vision", () => { + const body = { + messages: [ + { + role: "user", + content: "Test text", + images: ["base64..."], + experimental_attachments: [{ contentType: "image/png", url: "data:image/png;base64,..." }], + }, + ], + }; + const noVisionCaps = { vision: false, pdf: false, audioInput: false }; + + stripUnsupportedModalities(body, FORMATS.OPENAI, noVisionCaps); + + expect(body.messages[0].images).toBeUndefined(); + expect(body.messages[0].experimental_attachments).toHaveLength(0); + }); +});