fix(combo): detect images from Hermes and attachment payloads for Vision Adapter

Inspect images[], experimental_attachments/attachments, message-level
image/image_url/audio_url, and inline data:image|audio|pdf URIs on trailing
user turns so Vision Adapter auto-switch fires for Hermes/Ollama/Vercel AI
SDK shapes. stripOpenAI now also drops msg.images and image attachments when
the active model lacks vision support.

open-sse/services/combo.js

@@ -138,8 +138,42 @@ export function detectRequiredCapabilities(body) {
     if (Array.isArray(content)) for (const b of content) scanBlock(b);
   };
 
+  const scanMessage = (m) => {
+    if (!m || typeof m !== "object") return;
+
+    // Ollama / Hermes images array (strings or objects)
+    if (Array.isArray(m.images) && m.images.length > 0) {
+      required.add("vision");
+    }
+
+    // Vercel AI SDK / Hermes attachments / experimental_attachments
+    const attachments = m.experimental_attachments || m.attachments;
+    if (Array.isArray(attachments)) {
+      for (const att of attachments) {
+        if (!att) continue;
+        const mime = att.contentType || att.mediaType || (typeof att.url === "string" && att.url.match(/^data:([^;,]+)/)?.[1]);
+        if (mime) addByMime(mime);
+        else if (att.url || att.data) required.add("vision");
+      }
+    }
+
+    // Direct message-level modality properties
+    if (m.image_url || m.image) required.add("vision");
+    if (m.audio_url || m.audio) required.add("audioInput");
+
+    // Scan array content blocks
+    scanContent(m.content);
+
+    // Scan string content for embedded data URIs
+    if (typeof m.content === "string") {
+      if (m.content.includes("data:image/")) required.add("vision");
+      else if (m.content.includes("data:audio/")) required.add("audioInput");
+      else if (m.content.includes("data:application/pdf")) required.add("pdf");
+    }
+  };
+
   // Modalities: current user turn only (trailing user run across each known shape).
-  for (const m of trailingUserItems(body.messages)) scanContent(m.content);      // openai / claude
+  for (const m of trailingUserItems(body.messages)) scanMessage(m);              // openai / claude / hermes / ollama
   for (const it of trailingUserItems(body.input)) scanContent(it.content);       // responses
   const contents = body.contents || body.request?.contents;                      // gemini / antigravity
   for (const c of trailingUserItems(contents)) scanContent(c.parts);

open-sse/translator/concerns/modality.js

@@ -62,6 +62,19 @@ function stripOpenAI(body, caps) {
   if (!Array.isArray(body.messages)) return;
   const last = body.messages.length - 1;
   body.messages.forEach((msg, i) => {
+    if (caps.vision === false) {
+      if (Array.isArray(msg.images)) delete msg.images;
+      if (Array.isArray(msg.experimental_attachments)) {
+        msg.experimental_attachments = msg.experimental_attachments.filter(
+          (a) => !(a?.contentType?.startsWith("image/") || (typeof a?.url === "string" && a.url.startsWith("data:image/")))
+        );
+      }
+      if (Array.isArray(msg.attachments)) {
+        msg.attachments = msg.attachments.filter(
+          (a) => !(a?.contentType?.startsWith("image/") || (typeof a?.url === "string" && a.url.startsWith("data:image/")))
+        );
+      }
+    }
     if (!Array.isArray(msg.content)) return;
     const removed = new Set();
     msg.content = filterBlocks(msg.content, capForOpenAIBlock, caps, removed, i === last);

tests/unit/hermes-vision-detection.test.js

@@ -0,0 +1,115 @@
+import { describe, it, expect } from "vitest";
+import { detectRequiredCapabilities } from "../../open-sse/services/combo.js";
+import { augmentModelsWithCapacityAdapter } from "../../open-sse/services/capacityAdapter.js";
+import { stripUnsupportedModalities } from "../../open-sse/translator/concerns/modality.js";
+import { FORMATS } from "../../open-sse/translator/formats.js";
+
+describe("Hermes Vision Image Detection", () => {
+  it("detects vision from Ollama / Hermes images array", () => {
+    const body = {
+      messages: [
+        {
+          role: "user",
+          content: "Please analyze this image from Hermes",
+          images: ["iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="],
+        },
+      ],
+    };
+    const caps = detectRequiredCapabilities(body);
+    expect(caps.has("vision")).toBe(true);
+  });
+
+  it("detects vision from Vercel AI SDK / Hermes experimental_attachments", () => {
+    const body = {
+      messages: [
+        {
+          role: "user",
+          content: "Describe this attachment",
+          experimental_attachments: [
+            {
+              contentType: "image/png",
+              url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
+            },
+          ],
+        },
+      ],
+    };
+    const caps = detectRequiredCapabilities(body);
+    expect(caps.has("vision")).toBe(true);
+  });
+
+  it("detects vision from Hermes attachments array", () => {
+    const body = {
+      messages: [
+        {
+          role: "user",
+          content: "Look at this photo",
+          attachments: [
+            {
+              mediaType: "image/jpeg",
+              url: "https://example.com/photo.jpg",
+            },
+          ],
+        },
+      ],
+    };
+    const caps = detectRequiredCapabilities(body);
+    expect(caps.has("vision")).toBe(true);
+  });
+
+  it("detects vision from embedded data:image URI in string content", () => {
+    const body = {
+      messages: [
+        {
+          role: "user",
+          content: "Here is an inline image: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
+        },
+      ],
+    };
+    const caps = detectRequiredCapabilities(body);
+    expect(caps.has("vision")).toBe(true);
+  });
+
+  it("auto-switches non-vision model (deepseek-v4-pro) to Vision Adapter model (Kimi-K3)", () => {
+    const body = {
+      messages: [
+        {
+          role: "user",
+          content: "Analyze image",
+          images: ["base64data..."],
+        },
+      ],
+    };
+    const reqCaps = detectRequiredCapabilities(body);
+    const settings = {
+      capacityAdapter: {
+        vision: {
+          enabled: true,
+          models: ["cmc/moonshotai/Kimi-K3"],
+        },
+      },
+    };
+
+    const augmented = augmentModelsWithCapacityAdapter(["cmc/deepseek/deepseek-v4-pro"], reqCaps, settings);
+    expect(augmented).toEqual(["cmc/moonshotai/Kimi-K3", "cmc/deepseek/deepseek-v4-pro"]);
+  });
+
+  it("strips msg.images and attachments when model does not support vision", () => {
+    const body = {
+      messages: [
+        {
+          role: "user",
+          content: "Test text",
+          images: ["base64..."],
+          experimental_attachments: [{ contentType: "image/png", url: "data:image/png;base64,..." }],
+        },
+      ],
+    };
+    const noVisionCaps = { vision: false, pdf: false, audioInput: false };
+
+    stripUnsupportedModalities(body, FORMATS.OPENAI, noVisionCaps);
+
+    expect(body.messages[0].images).toBeUndefined();
+    expect(body.messages[0].experimental_attachments).toHaveLength(0);
+  });
+});