feat(translator): add vision support for commandcode provider
Map OpenAI image_url / Claude-style image blocks to the {type:"image",
image:"<data URI|url>"} shape the command-code CLI sends to /alpha/generate
instead of dropping them to "[image omitted]". Handles data URIs, raw base64
(with media_type / image/png fallback), and remote URLs.
- Promote bugs-gemini-cursor-commandcode "image content is preserved" from
it.fails to a real assertion (bug fixed)
- Add vision unit tests to openai-to-commandcode.test.js
- Add test-commandcode-vision.sh curl helper for live verification
Co-authored-by: CommandCodeBot <noreply@commandcode.ai>
This commit is contained in:
@@ -13,160 +13,200 @@ import { register } from "../index.js";
|
||||
import { FORMATS } from "../formats.js";
|
||||
import { randomUUID } from "crypto";
|
||||
import { ROLE, OPENAI_BLOCK } from "../schema/index.js";
|
||||
import { DEFAULT_IMAGE_MIME } from "../schema/index.js";
|
||||
import { parseDataUri } from "../concerns/image.js";
|
||||
import { DEFAULT_MAX_TOKENS } from "../../config/runtimeConfig.js";
|
||||
|
||||
function flattenText(content) {
|
||||
if (content == null) return "";
|
||||
if (typeof content === "string") return content;
|
||||
if (Array.isArray(content)) {
|
||||
const parts = [];
|
||||
for (const p of content) {
|
||||
if (typeof p === "string") parts.push(p);
|
||||
else if (p && typeof p === "object" && typeof p.text === "string") parts.push(p.text);
|
||||
}
|
||||
return parts.join("\n");
|
||||
}
|
||||
return String(content);
|
||||
if (content == null) return "";
|
||||
if (typeof content === "string") return content;
|
||||
if (Array.isArray(content)) {
|
||||
const parts = [];
|
||||
for (const p of content) {
|
||||
if (typeof p === "string") parts.push(p);
|
||||
else if (p && typeof p === "object" && typeof p.text === "string")
|
||||
parts.push(p.text);
|
||||
}
|
||||
return parts.join("\n");
|
||||
}
|
||||
return String(content);
|
||||
}
|
||||
|
||||
function toContentBlocks(content) {
|
||||
if (content == null) return [{ type: OPENAI_BLOCK.TEXT, text: "" }];
|
||||
if (typeof content === "string") return [{ type: OPENAI_BLOCK.TEXT, text: content }];
|
||||
if (Array.isArray(content)) {
|
||||
const blocks = [];
|
||||
for (const part of content) {
|
||||
if (typeof part === "string") {
|
||||
blocks.push({ type: OPENAI_BLOCK.TEXT, text: part });
|
||||
} else if (part && typeof part === "object") {
|
||||
if (part.type === OPENAI_BLOCK.TEXT && typeof part.text === "string") {
|
||||
blocks.push({ type: OPENAI_BLOCK.TEXT, text: part.text });
|
||||
} else if (part.type === OPENAI_BLOCK.IMAGE_URL || part.type === OPENAI_BLOCK.IMAGE) {
|
||||
blocks.push({ type: OPENAI_BLOCK.TEXT, text: "[image omitted]" });
|
||||
} else if (typeof part.text === "string") {
|
||||
blocks.push({ type: OPENAI_BLOCK.TEXT, text: part.text });
|
||||
}
|
||||
}
|
||||
}
|
||||
return blocks.length ? blocks : [{ type: OPENAI_BLOCK.TEXT, text: "" }];
|
||||
}
|
||||
return [{ type: OPENAI_BLOCK.TEXT, text: String(content) }];
|
||||
if (content == null) return [{ type: OPENAI_BLOCK.TEXT, text: "" }];
|
||||
if (typeof content === "string")
|
||||
return [{ type: OPENAI_BLOCK.TEXT, text: content }];
|
||||
if (Array.isArray(content)) {
|
||||
const blocks = [];
|
||||
for (const part of content) {
|
||||
if (typeof part === "string") {
|
||||
blocks.push({ type: OPENAI_BLOCK.TEXT, text: part });
|
||||
} else if (part && typeof part === "object") {
|
||||
if (part.type === OPENAI_BLOCK.TEXT && typeof part.text === "string") {
|
||||
blocks.push({ type: OPENAI_BLOCK.TEXT, text: part.text });
|
||||
} else if (
|
||||
part.type === OPENAI_BLOCK.IMAGE_URL ||
|
||||
part.type === OPENAI_BLOCK.IMAGE
|
||||
) {
|
||||
// CommandCode `/alpha/generate` accepts {type:"image", image:"<data URI | url>"} —
|
||||
// same shape the official command-code CLI sends (verified from CLI source).
|
||||
const src = part.source;
|
||||
let raw = part.image_url?.url || src?.data || src?.url || "";
|
||||
let parsed = parseDataUri(raw);
|
||||
if (!parsed && src?.type === "base64" && src?.data) {
|
||||
// Claude-style base64 source without a data-URI prefix → wrap it.
|
||||
raw = `data:${src.media_type || DEFAULT_IMAGE_MIME};base64,${src.data}`;
|
||||
parsed = parseDataUri(raw);
|
||||
}
|
||||
if (parsed) {
|
||||
blocks.push({
|
||||
type: "image",
|
||||
image: `data:${parsed.mimeType};base64,${parsed.base64}`,
|
||||
});
|
||||
} else if (raw) {
|
||||
blocks.push({ type: "image", image: raw });
|
||||
}
|
||||
} else if (typeof part.text === "string") {
|
||||
blocks.push({ type: OPENAI_BLOCK.TEXT, text: part.text });
|
||||
}
|
||||
}
|
||||
}
|
||||
return blocks.length ? blocks : [{ type: OPENAI_BLOCK.TEXT, text: "" }];
|
||||
}
|
||||
return [{ type: OPENAI_BLOCK.TEXT, text: String(content) }];
|
||||
}
|
||||
|
||||
function safeParseJson(s) {
|
||||
if (s == null) return {};
|
||||
if (typeof s !== "string") return s;
|
||||
try { return JSON.parse(s); } catch { return {}; }
|
||||
if (s == null) return {};
|
||||
if (typeof s !== "string") return s;
|
||||
try {
|
||||
return JSON.parse(s);
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function convertMessages(messages = []) {
|
||||
const out = [];
|
||||
const systemTexts = [];
|
||||
const out = [];
|
||||
const systemTexts = [];
|
||||
|
||||
for (const m of messages) {
|
||||
if (!m) continue;
|
||||
const role = m.role;
|
||||
for (const m of messages) {
|
||||
if (!m) continue;
|
||||
const role = m.role;
|
||||
|
||||
if (role === ROLE.SYSTEM) {
|
||||
const t = flattenText(m.content);
|
||||
if (t) systemTexts.push(t);
|
||||
continue;
|
||||
}
|
||||
if (role === ROLE.SYSTEM) {
|
||||
const t = flattenText(m.content);
|
||||
if (t) systemTexts.push(t);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (role === ROLE.TOOL) {
|
||||
const value = typeof m.content === "string" ? m.content : flattenText(m.content);
|
||||
out.push({
|
||||
role: ROLE.TOOL,
|
||||
content: [{
|
||||
type: "tool-result",
|
||||
toolCallId: m.tool_call_id || "",
|
||||
toolName: m.name || "",
|
||||
output: { type: "text", value },
|
||||
}],
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (role === ROLE.TOOL) {
|
||||
const value =
|
||||
typeof m.content === "string" ? m.content : flattenText(m.content);
|
||||
out.push({
|
||||
role: ROLE.TOOL,
|
||||
content: [
|
||||
{
|
||||
type: "tool-result",
|
||||
toolCallId: m.tool_call_id || "",
|
||||
toolName: m.name || "",
|
||||
output: { type: "text", value },
|
||||
},
|
||||
],
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (role === ROLE.ASSISTANT) {
|
||||
const blocks = [];
|
||||
const text = flattenText(m.content);
|
||||
if (text) blocks.push({ type: OPENAI_BLOCK.TEXT, text });
|
||||
if (Array.isArray(m.tool_calls)) {
|
||||
for (const tc of m.tool_calls) {
|
||||
const fn = tc.function || {};
|
||||
blocks.push({
|
||||
type: "tool-call",
|
||||
toolCallId: tc.id || "",
|
||||
toolName: fn.name || "",
|
||||
input: safeParseJson(fn.arguments),
|
||||
});
|
||||
}
|
||||
}
|
||||
out.push({ role: ROLE.ASSISTANT, content: blocks.length ? blocks : [{ type: OPENAI_BLOCK.TEXT, text: "" }] });
|
||||
continue;
|
||||
}
|
||||
if (role === ROLE.ASSISTANT) {
|
||||
const blocks = [];
|
||||
const text = flattenText(m.content);
|
||||
if (text) blocks.push({ type: OPENAI_BLOCK.TEXT, text });
|
||||
if (Array.isArray(m.tool_calls)) {
|
||||
for (const tc of m.tool_calls) {
|
||||
const fn = tc.function || {};
|
||||
blocks.push({
|
||||
type: "tool-call",
|
||||
toolCallId: tc.id || "",
|
||||
toolName: fn.name || "",
|
||||
input: safeParseJson(fn.arguments),
|
||||
});
|
||||
}
|
||||
}
|
||||
out.push({
|
||||
role: ROLE.ASSISTANT,
|
||||
content: blocks.length
|
||||
? blocks
|
||||
: [{ type: OPENAI_BLOCK.TEXT, text: "" }],
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
out.push({ role: ROLE.USER, content: toContentBlocks(m.content) });
|
||||
}
|
||||
out.push({ role: ROLE.USER, content: toContentBlocks(m.content) });
|
||||
}
|
||||
|
||||
return { messages: out, system: systemTexts.join("\n\n") };
|
||||
return { messages: out, system: systemTexts.join("\n\n") };
|
||||
}
|
||||
|
||||
function convertTools(tools) {
|
||||
if (!Array.isArray(tools) || tools.length === 0) return undefined;
|
||||
const result = [];
|
||||
for (const t of tools) {
|
||||
if (!t) continue;
|
||||
if (t.type === OPENAI_BLOCK.FUNCTION && t.function) {
|
||||
result.push({
|
||||
name: t.function.name,
|
||||
description: t.function.description,
|
||||
input_schema: t.function.parameters || { type: "object" },
|
||||
});
|
||||
} else if (t.name && (t.input_schema || t.parameters)) {
|
||||
result.push({
|
||||
name: t.name,
|
||||
description: t.description,
|
||||
input_schema: t.input_schema || t.parameters,
|
||||
});
|
||||
}
|
||||
}
|
||||
return result.length ? result : undefined;
|
||||
if (!Array.isArray(tools) || tools.length === 0) return undefined;
|
||||
const result = [];
|
||||
for (const t of tools) {
|
||||
if (!t) continue;
|
||||
if (t.type === OPENAI_BLOCK.FUNCTION && t.function) {
|
||||
result.push({
|
||||
name: t.function.name,
|
||||
description: t.function.description,
|
||||
input_schema: t.function.parameters || { type: "object" },
|
||||
});
|
||||
} else if (t.name && (t.input_schema || t.parameters)) {
|
||||
result.push({
|
||||
name: t.name,
|
||||
description: t.description,
|
||||
input_schema: t.input_schema || t.parameters,
|
||||
});
|
||||
}
|
||||
}
|
||||
return result.length ? result : undefined;
|
||||
}
|
||||
|
||||
export function openaiToCommandCodeRequest(model, body, stream /* , credentials */) {
|
||||
const { messages, system } = convertMessages(body.messages);
|
||||
const params = {
|
||||
model,
|
||||
messages,
|
||||
stream: stream !== false,
|
||||
max_tokens: body.max_tokens ?? body.max_output_tokens ?? DEFAULT_MAX_TOKENS,
|
||||
temperature: body.temperature ?? 0.3,
|
||||
};
|
||||
export function openaiToCommandCodeRequest(
|
||||
model,
|
||||
body,
|
||||
stream /* , credentials */,
|
||||
) {
|
||||
const { messages, system } = convertMessages(body.messages);
|
||||
const params = {
|
||||
model,
|
||||
messages,
|
||||
stream: stream !== false,
|
||||
max_tokens: body.max_tokens ?? body.max_output_tokens ?? DEFAULT_MAX_TOKENS,
|
||||
temperature: body.temperature ?? 0.3,
|
||||
};
|
||||
|
||||
if (system) params.system = system;
|
||||
if (system) params.system = system;
|
||||
|
||||
const tools = convertTools(body.tools);
|
||||
if (tools) params.tools = tools;
|
||||
if (body.top_p != null) params.top_p = body.top_p;
|
||||
const tools = convertTools(body.tools);
|
||||
if (tools) params.tools = tools;
|
||||
if (body.top_p != null) params.top_p = body.top_p;
|
||||
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
|
||||
return {
|
||||
threadId: randomUUID(),
|
||||
memory: "",
|
||||
config: {
|
||||
workingDir: process.cwd(),
|
||||
date: today,
|
||||
environment: process.platform,
|
||||
structure: [],
|
||||
isGitRepo: false,
|
||||
currentBranch: "",
|
||||
mainBranch: "",
|
||||
gitStatus: "",
|
||||
recentCommits: [],
|
||||
},
|
||||
params,
|
||||
};
|
||||
return {
|
||||
threadId: randomUUID(),
|
||||
memory: "",
|
||||
config: {
|
||||
workingDir: process.cwd(),
|
||||
date: today,
|
||||
environment: process.platform,
|
||||
structure: [],
|
||||
isGitRepo: false,
|
||||
currentBranch: "",
|
||||
mainBranch: "",
|
||||
gitStatus: "",
|
||||
recentCommits: [],
|
||||
},
|
||||
params,
|
||||
};
|
||||
}
|
||||
|
||||
register(FORMATS.OPENAI, FORMATS.COMMANDCODE, openaiToCommandCodeRequest, null);
|
||||
|
||||
Reference in New Issue
Block a user