luulam/9router
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(combo): detect images from Hermes and attachment payloads for Vision Adapter

Browse Source 
Inspect images[], experimental_attachments/attachments, message-level
image/image_url/audio_url, and inline data:image|audio|pdf URIs on trailing
user turns so Vision Adapter auto-switch fires for Hermes/Ollama/Vercel AI
SDK shapes. stripOpenAI now also drops msg.images and image attachments when
the active model lacks vision support.
This commit is contained in:
Bertho Joris
2026-08-13 18:29:51 +07:00
committed by decolua
parent 65197ad11c
commit 345cdcf6a5
3 changed files with 163 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
open-sse/services/combo.js
View File

@@ -138,8 +138,42 @@ export function detectRequiredCapabilities(body) {
if (Array.isArray(content)) for (const b of content) scanBlock(b);
};
const scanMessage = (m) => {
if (!m || typeof m !== "object") return;
// Ollama / Hermes images array (strings or objects)
if (Array.isArray(m.images) && m.images.length > 0) {
required.add("vision");
}
// Vercel AI SDK / Hermes attachments / experimental_attachments
const attachments = m.experimental_attachments || m.attachments;
if (Array.isArray(attachments)) {
for (const att of attachments) {
if (!att) continue;
const mime = att.contentType || att.mediaType || (typeof att.url === "string" && att.url.match(/^data:([^;,]+)/)?.[1]);
if (mime) addByMime(mime);
else if (att.url || att.data) required.add("vision");
}
}
// Direct message-level modality properties
if (m.image_url || m.image) required.add("vision");
if (m.audio_url || m.audio) required.add("audioInput");
// Scan array content blocks
scanContent(m.content);
// Scan string content for embedded data URIs
if (typeof m.content === "string") {
if (m.content.includes("data:image/")) required.add("vision");
else if (m.content.includes("data:audio/")) required.add("audioInput");
else if (m.content.includes("data:application/pdf")) required.add("pdf");
}
};
// Modalities: current user turn only (trailing user run across each known shape).
for (const m of trailingUserItems(body.messages)) scanContent(m.content); // openai / claude
for (const m of trailingUserItems(body.messages)) scanMessage(m); // openai / claude / hermes / ollama
for (const it of trailingUserItems(body.input)) scanContent(it.content); // responses
const contents = body.contents || body.request?.contents; // gemini / antigravity
for (const c of trailingUserItems(contents)) scanContent(c.parts);

13
open-sse/translator/concerns/modality.js
View File

@@ -62,6 +62,19 @@ function stripOpenAI(body, caps) {
if (!Array.isArray(body.messages)) return;
const last = body.messages.length - 1;
body.messages.forEach((msg, i) => {
if (caps.vision === false) {
if (Array.isArray(msg.images)) delete msg.images;
if (Array.isArray(msg.experimental_attachments)) {
msg.experimental_attachments = msg.experimental_attachments.filter(
(a) => !(a?.contentType?.startsWith("image/") || (typeof a?.url === "string" && a.url.startsWith("data:image/")))
);
}
if (Array.isArray(msg.attachments)) {
msg.attachments = msg.attachments.filter(
(a) => !(a?.contentType?.startsWith("image/") || (typeof a?.url === "string" && a.url.startsWith("data:image/")))
);
}
}
if (!Array.isArray(msg.content)) return;
const removed = new Set();
msg.content = filterBlocks(msg.content, capForOpenAIBlock, caps, removed, i === last);