fix(kiro): preserve inline images in OpenAI MITM
Forward Kiro userInputMessage.images as OpenAI-compatible image_url content parts.
This commit is contained in:
3
.gitignore
vendored
3
.gitignore
vendored
@@ -86,3 +86,6 @@ graphify-out/*
|
||||
.codegraph/
|
||||
.PR/
|
||||
.next-analyze/*
|
||||
|
||||
# Kiro local workspace state
|
||||
.kiro/
|
||||
@@ -180,6 +180,14 @@ function withInitialFrame(state, frames) {
|
||||
|
||||
// ─── CodeWhisperer → OpenAI conversion ───────────────────────────────────────
|
||||
|
||||
const INLINE_IMAGE_MIME_BY_FORMAT = new Map([
|
||||
["png", "image/png"],
|
||||
["jpeg", "image/jpeg"],
|
||||
["jpg", "image/jpeg"],
|
||||
["gif", "image/gif"],
|
||||
["webp", "image/webp"],
|
||||
]);
|
||||
|
||||
/**
|
||||
* Safely stringify a tool-call input value.
|
||||
* OpenAI expects `function.arguments` to be a JSON string, never an object.
|
||||
@@ -214,9 +222,32 @@ function convertUserInputMessage(uim) {
|
||||
});
|
||||
}
|
||||
|
||||
const images = Array.isArray(uim.images) ? uim.images : [];
|
||||
const imageParts = images
|
||||
.filter(image => image !== null
|
||||
&& typeof image === "object"
|
||||
&& !Array.isArray(image)
|
||||
&& image.source !== null
|
||||
&& typeof image.source === "object"
|
||||
&& !Array.isArray(image.source)
|
||||
&& INLINE_IMAGE_MIME_BY_FORMAT.has(image.format)
|
||||
&& typeof image.source.bytes === "string"
|
||||
&& image.source.bytes.length > 0)
|
||||
.map(image => ({
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: `data:${INLINE_IMAGE_MIME_BY_FORMAT.get(image.format)};base64,${image.source.bytes}`,
|
||||
},
|
||||
}));
|
||||
|
||||
// Emit user text only if it exists alongside OR when there are no tool results
|
||||
const text = (uim.content || "").trim();
|
||||
if (text || toolResults.length === 0) {
|
||||
if (imageParts.length > 0) {
|
||||
const content = [];
|
||||
if (text) content.push({ type: "text", text });
|
||||
content.push(...imageParts);
|
||||
out.push({ role: "user", content });
|
||||
} else if (text || toolResults.length === 0) {
|
||||
out.push({ role: "user", content: text });
|
||||
}
|
||||
|
||||
|
||||
192
tests/unit/kiro-image-forwarding.test.js
Normal file
192
tests/unit/kiro-image-forwarding.test.js
Normal file
@@ -0,0 +1,192 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { createRequire } from "node:module";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const { intercept } = require("../../src/mitm/handlers/kiro.js");
|
||||
|
||||
const MODEL = "offline-test-model";
|
||||
|
||||
function makeResponseCollector() {
|
||||
const chunks = [];
|
||||
const response = {
|
||||
headersSent: false,
|
||||
statusCode: undefined,
|
||||
ended: false,
|
||||
writeHead(statusCode) {
|
||||
this.statusCode = statusCode;
|
||||
this.headersSent = true;
|
||||
return this;
|
||||
},
|
||||
write(chunk) {
|
||||
chunks.push(Buffer.from(chunk));
|
||||
return true;
|
||||
},
|
||||
end(chunk) {
|
||||
if (chunk !== undefined) chunks.push(Buffer.from(chunk));
|
||||
this.ended = true;
|
||||
this.headersSent = true;
|
||||
return this;
|
||||
},
|
||||
};
|
||||
|
||||
return { response, chunks };
|
||||
}
|
||||
|
||||
async function captureOpenAIRequest(request) {
|
||||
const originalFetch = globalThis.fetch;
|
||||
const { response, chunks } = makeResponseCollector();
|
||||
let captured;
|
||||
|
||||
const fetchMock = vi.fn(async (url, init) => {
|
||||
captured = { url: String(url), init };
|
||||
return new Response("data: [DONE]\n\n", {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "text/event-stream" },
|
||||
});
|
||||
});
|
||||
globalThis.fetch = fetchMock;
|
||||
|
||||
try {
|
||||
await intercept(
|
||||
{ headers: { "x-test": "kiro-image-forwarding" } },
|
||||
response,
|
||||
Buffer.from(JSON.stringify(request)),
|
||||
MODEL,
|
||||
);
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
expect(captured.url.endsWith("/v1/chat/completions")).toBe(true);
|
||||
expect(captured.init.method).toBe("POST");
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.ended).toBe(true);
|
||||
expect(chunks.length).toBeGreaterThan(0);
|
||||
|
||||
return JSON.parse(captured.init.body);
|
||||
} finally {
|
||||
if (originalFetch === undefined) delete globalThis.fetch;
|
||||
else globalThis.fetch = originalFetch;
|
||||
}
|
||||
}
|
||||
|
||||
function image(format, bytes) {
|
||||
return { format, source: { bytes } };
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe("Kiro MITM inline image forwarding", () => {
|
||||
it("forwards text and inline images as OpenAI image_url content parts", async () => {
|
||||
const outboundBody = await captureOpenAIRequest({
|
||||
conversationState: {
|
||||
history: [],
|
||||
currentMessage: {
|
||||
userInputMessage: {
|
||||
content: " Describe this ",
|
||||
images: [image("png", "aGVsbG8=")],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(outboundBody).toMatchObject({
|
||||
model: MODEL,
|
||||
stream: true,
|
||||
messages: [{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "Describe this" },
|
||||
{ type: "image_url", image_url: { url: "data:image/png;base64,aGVsbG8=" } },
|
||||
],
|
||||
}],
|
||||
});
|
||||
});
|
||||
|
||||
it("emits an image-only user turn even when tool results are present", async () => {
|
||||
const outboundBody = await captureOpenAIRequest({
|
||||
conversationState: {
|
||||
history: [],
|
||||
currentMessage: {
|
||||
userInputMessage: {
|
||||
content: " ",
|
||||
images: [image("jpg", "LzlqLzQ=")],
|
||||
userInputMessageContext: {
|
||||
toolResults: [
|
||||
{ toolUseId: "tool-a", content: [{ text: "first" }, { text: "result" }] },
|
||||
{ toolUseId: "tool-b", content: [{ text: "second" }] },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(outboundBody.messages).toEqual([
|
||||
{ role: "tool", tool_call_id: "tool-a", content: "first\nresult" },
|
||||
{ role: "tool", tool_call_id: "tool-b", content: "second" },
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "image_url", image_url: { url: "data:image/jpeg;base64,LzlqLzQ=" } },
|
||||
],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps historical images on their original user turn", async () => {
|
||||
const outboundBody = await captureOpenAIRequest({
|
||||
conversationState: {
|
||||
history: [
|
||||
{
|
||||
userInputMessage: {
|
||||
content: " historical evidence ",
|
||||
images: [image("jpeg", "anBlZw=="), image("webp", "d2VicA==")],
|
||||
},
|
||||
},
|
||||
{ assistantResponseMessage: { content: "assistant reply" } },
|
||||
],
|
||||
currentMessage: { userInputMessage: { content: "current question" } },
|
||||
},
|
||||
});
|
||||
|
||||
expect(outboundBody.messages).toEqual([
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "historical evidence" },
|
||||
{ type: "image_url", image_url: { url: "data:image/jpeg;base64,anBlZw==" } },
|
||||
{ type: "image_url", image_url: { url: "data:image/webp;base64,d2VicA==" } },
|
||||
],
|
||||
},
|
||||
{ role: "assistant", content: "assistant reply" },
|
||||
{ role: "user", content: "current question" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("ignores malformed and unsupported image entries without changing text-only behavior", async () => {
|
||||
const outboundBody = await captureOpenAIRequest({
|
||||
conversationState: {
|
||||
history: [],
|
||||
currentMessage: {
|
||||
userInputMessage: {
|
||||
content: " keep this text ",
|
||||
images: [
|
||||
image("svg", "ignored"),
|
||||
image("PNG", "ignored"),
|
||||
image("jpeg", ""),
|
||||
{ format: "gif", source: { bytes: 42 } },
|
||||
null,
|
||||
[],
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(outboundBody.messages).toEqual([
|
||||
{ role: "user", content: "keep this text" },
|
||||
]);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user