fix(kiro): preserve inline images in OpenAI MITM

Forward Kiro userInputMessage.images as OpenAI-compatible image_url content parts.
This commit is contained in:
Outis
2026-09-03 09:19:38 +07:00
committed by decolua
parent 70f15aa50b
commit 1f190bd00b
3 changed files with 227 additions and 1 deletions

3
.gitignore vendored
View File

@@ -86,3 +86,6 @@ graphify-out/*
.codegraph/
.PR/
.next-analyze/*
# Kiro local workspace state
.kiro/

View File

@@ -180,6 +180,14 @@ function withInitialFrame(state, frames) {
// ─── CodeWhisperer → OpenAI conversion ───────────────────────────────────────
const INLINE_IMAGE_MIME_BY_FORMAT = new Map([
["png", "image/png"],
["jpeg", "image/jpeg"],
["jpg", "image/jpeg"],
["gif", "image/gif"],
["webp", "image/webp"],
]);
/**
* Safely stringify a tool-call input value.
* OpenAI expects `function.arguments` to be a JSON string, never an object.
@@ -214,9 +222,32 @@ function convertUserInputMessage(uim) {
});
}
const images = Array.isArray(uim.images) ? uim.images : [];
const imageParts = images
.filter(image => image !== null
&& typeof image === "object"
&& !Array.isArray(image)
&& image.source !== null
&& typeof image.source === "object"
&& !Array.isArray(image.source)
&& INLINE_IMAGE_MIME_BY_FORMAT.has(image.format)
&& typeof image.source.bytes === "string"
&& image.source.bytes.length > 0)
.map(image => ({
type: "image_url",
image_url: {
url: `data:${INLINE_IMAGE_MIME_BY_FORMAT.get(image.format)};base64,${image.source.bytes}`,
},
}));
// Emit user text only if it exists alongside OR when there are no tool results
const text = (uim.content || "").trim();
if (text || toolResults.length === 0) {
if (imageParts.length > 0) {
const content = [];
if (text) content.push({ type: "text", text });
content.push(...imageParts);
out.push({ role: "user", content });
} else if (text || toolResults.length === 0) {
out.push({ role: "user", content: text });
}

View File

@@ -0,0 +1,192 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { createRequire } from "node:module";
const require = createRequire(import.meta.url);
const { intercept } = require("../../src/mitm/handlers/kiro.js");
const MODEL = "offline-test-model";
function makeResponseCollector() {
const chunks = [];
const response = {
headersSent: false,
statusCode: undefined,
ended: false,
writeHead(statusCode) {
this.statusCode = statusCode;
this.headersSent = true;
return this;
},
write(chunk) {
chunks.push(Buffer.from(chunk));
return true;
},
end(chunk) {
if (chunk !== undefined) chunks.push(Buffer.from(chunk));
this.ended = true;
this.headersSent = true;
return this;
},
};
return { response, chunks };
}
async function captureOpenAIRequest(request) {
const originalFetch = globalThis.fetch;
const { response, chunks } = makeResponseCollector();
let captured;
const fetchMock = vi.fn(async (url, init) => {
captured = { url: String(url), init };
return new Response("data: [DONE]\n\n", {
status: 200,
headers: { "Content-Type": "text/event-stream" },
});
});
globalThis.fetch = fetchMock;
try {
await intercept(
{ headers: { "x-test": "kiro-image-forwarding" } },
response,
Buffer.from(JSON.stringify(request)),
MODEL,
);
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(captured.url.endsWith("/v1/chat/completions")).toBe(true);
expect(captured.init.method).toBe("POST");
expect(response.statusCode).toBe(200);
expect(response.ended).toBe(true);
expect(chunks.length).toBeGreaterThan(0);
return JSON.parse(captured.init.body);
} finally {
if (originalFetch === undefined) delete globalThis.fetch;
else globalThis.fetch = originalFetch;
}
}
function image(format, bytes) {
return { format, source: { bytes } };
}
afterEach(() => {
vi.restoreAllMocks();
vi.unstubAllGlobals();
});
describe("Kiro MITM inline image forwarding", () => {
it("forwards text and inline images as OpenAI image_url content parts", async () => {
const outboundBody = await captureOpenAIRequest({
conversationState: {
history: [],
currentMessage: {
userInputMessage: {
content: " Describe this ",
images: [image("png", "aGVsbG8=")],
},
},
},
});
expect(outboundBody).toMatchObject({
model: MODEL,
stream: true,
messages: [{
role: "user",
content: [
{ type: "text", text: "Describe this" },
{ type: "image_url", image_url: { url: "data:image/png;base64,aGVsbG8=" } },
],
}],
});
});
it("emits an image-only user turn even when tool results are present", async () => {
const outboundBody = await captureOpenAIRequest({
conversationState: {
history: [],
currentMessage: {
userInputMessage: {
content: " ",
images: [image("jpg", "LzlqLzQ=")],
userInputMessageContext: {
toolResults: [
{ toolUseId: "tool-a", content: [{ text: "first" }, { text: "result" }] },
{ toolUseId: "tool-b", content: [{ text: "second" }] },
],
},
},
},
},
});
expect(outboundBody.messages).toEqual([
{ role: "tool", tool_call_id: "tool-a", content: "first\nresult" },
{ role: "tool", tool_call_id: "tool-b", content: "second" },
{
role: "user",
content: [
{ type: "image_url", image_url: { url: "data:image/jpeg;base64,LzlqLzQ=" } },
],
},
]);
});
it("keeps historical images on their original user turn", async () => {
const outboundBody = await captureOpenAIRequest({
conversationState: {
history: [
{
userInputMessage: {
content: " historical evidence ",
images: [image("jpeg", "anBlZw=="), image("webp", "d2VicA==")],
},
},
{ assistantResponseMessage: { content: "assistant reply" } },
],
currentMessage: { userInputMessage: { content: "current question" } },
},
});
expect(outboundBody.messages).toEqual([
{
role: "user",
content: [
{ type: "text", text: "historical evidence" },
{ type: "image_url", image_url: { url: "data:image/jpeg;base64,anBlZw==" } },
{ type: "image_url", image_url: { url: "data:image/webp;base64,d2VicA==" } },
],
},
{ role: "assistant", content: "assistant reply" },
{ role: "user", content: "current question" },
]);
});
it("ignores malformed and unsupported image entries without changing text-only behavior", async () => {
const outboundBody = await captureOpenAIRequest({
conversationState: {
history: [],
currentMessage: {
userInputMessage: {
content: " keep this text ",
images: [
image("svg", "ignored"),
image("PNG", "ignored"),
image("jpeg", ""),
{ format: "gif", source: { bytes: 42 } },
null,
[],
],
},
},
},
});
expect(outboundBody.messages).toEqual([
{ role: "user", content: "keep this text" },
]);
});
});