diff --git a/open-sse/executors/qoder.js b/open-sse/executors/qoder.js index 86055aba..8ee21ac5 100644 --- a/open-sse/executors/qoder.js +++ b/open-sse/executors/qoder.js @@ -31,14 +31,16 @@ import { proxyAwareFetch } from "../utils/proxyFetch.js"; import { SSE_DONE } from "../utils/sseConstants.js"; import { FETCH_CONNECT_TIMEOUT_MS } from "../config/runtimeConfig.js"; import { - QODER_CHAT_URL_ENCODED, - QODER_CHAT_BASE_ALT, QODER_CHAT_SIG_PATH, - QODER_MODEL_MAP, + QODER_CONTEXT_TIER_ENV, + qoderInferenceBase, } from "../shared/qoder/constants.js"; import { getQoderModelConfig, resolveQoderModels, isQoderPat, resolveQoderCredentials } from "../services/qoderModels.js"; import { OPENAI_BLOCK, CLAUDE_BLOCK } from "../translator/schema/blocks.js"; import { encodeDataUri } from "../translator/concerns/image.js"; +import { createQoderSseCoalescer } from "../shared/qoder/sse.js"; +import { rewriteQoderMessageAttachments } from "../shared/qoder/attachments.js"; +import { resolveQoderContextTier, applyQoderContextTier } from "../shared/qoder/contextTier.js"; /** * Hoist role:"system" messages out of the messages array (Qoder rejects @@ -70,15 +72,16 @@ function normalizeMessages(messages) { * * Text-only content is flattened to a plain string (Qoder's historical * shape). When images are present the content stays an array and image - * blocks are kept as OpenAI-style `image_url` parts — verified against the - * upstream: it accepts both http(s) URLs and inline base64 data: URIs - * directly, no pre-upload to the /image/upload OSS flow required (that is - * a qodercli client-side choice, not a protocol requirement). The legacy + * blocks are kept as OpenAI-style `image_url` parts. Native qodercli + * uploads inlined bytes to `/api/v2/image/upload` first and then sends + * the OSS URL — `buildQoderRequestBody` does that rewrite before this + * runs. Tiny leftover data URIs are still accepted. The legacy * top-level `image_urls` / `chat_context.imageUrls` slots stay null — * qodercli leaves them null too. * * Claude-style `{type:"image", source:{...}}` blocks are converted to - * `image_url` so claude-format clients also round-trip. + * `image_url`. File/document blocks that survived rewrite become short + * stubs so 30MB PDFs never land in agent_chat_generation. */ function normalizeContent(content) { if (typeof content === "string") return content; @@ -88,10 +91,24 @@ function normalizeContent(content) { const blocks = []; const textParts = []; let hasImage = false; + + const pushText = (text) => { + if (!text) return; + if (hasImage || blocks.length) blocks.push({ type: OPENAI_BLOCK.TEXT, text }); + else textParts.push(text); + }; + + const imageUrlOf = (item) => { + if (typeof item.image_url === "string" && item.image_url) return item.image_url; + if (typeof item.image_url?.url === "string" && item.image_url.url) return item.image_url.url; + return null; + }; + for (const item of content) { if (!item || typeof item !== "object") continue; - if (item.type === OPENAI_BLOCK.IMAGE_URL && typeof item.image_url?.url === "string" && item.image_url.url) { - blocks.push({ type: OPENAI_BLOCK.IMAGE_URL, image_url: { url: item.image_url.url } }); + const imageUrl = item.type === OPENAI_BLOCK.IMAGE_URL ? imageUrlOf(item) : null; + if (imageUrl) { + blocks.push({ type: OPENAI_BLOCK.IMAGE_URL, image_url: { url: imageUrl } }); hasImage = true; } else if (item.type === CLAUDE_BLOCK.IMAGE && item.source) { // Claude base64/url image → OpenAI image_url equivalent. @@ -103,13 +120,14 @@ function normalizeContent(content) { blocks.push({ type: OPENAI_BLOCK.IMAGE_URL, image_url: { url } }); hasImage = true; } + } else if (item.type === OPENAI_BLOCK.FILE) { + const name = item.file?.filename || item.file?.name || "file"; + pushText(`[file omitted: ${name} — Qoder reads documents via its file API, not inlined bytes]`); + } else if (item.type === CLAUDE_BLOCK.DOCUMENT) { + const name = item.title || "document"; + pushText(`[file omitted: ${name} — Qoder reads documents via its file API, not inlined bytes]`); } else if (typeof item.text === "string" && item.text) { - if (hasImage || blocks.length) { - // Keep ordering faithful once images are in play. - blocks.push({ type: OPENAI_BLOCK.TEXT, text: item.text }); - } else { - textParts.push(item.text); - } + pushText(item.text); } } @@ -189,7 +207,7 @@ function truncate(s, n) { /** * Map the OpenAI-style request body into the exact shape Qoder expects. */ -async function buildQoderRequestBody({ model, body, credentials, log, proxyOptions, signal }) { +async function buildQoderRequestBody({ model, body, credentials, log, proxyOptions, signal, uploadFn = null }) { const qoderKey = String(model || "").replace(/^qoder\//, ""); // Fetch model config from dynamic API instead of relying on static QODER_MODEL_MAP. @@ -208,7 +226,30 @@ async function buildQoderRequestBody({ model, body, credentials, log, proxyOptio modelConfig = { ...retried, key: qoderKey }; } - const { messages, systemText } = normalizeMessages(body.messages || []); + const incoming = Array.isArray(body.messages) + ? body.messages.map((m) => { + if (!m || typeof m !== "object") return m; + return { + ...m, + content: Array.isArray(m.content) + ? m.content.map((b) => (b && typeof b === "object" ? { ...b } : b)) + : m.content, + }; + }) + : []; + try { + await rewriteQoderMessageAttachments(incoming, { + credentials, + log, + proxyOptions, + signal, + uploadFn, + }); + } catch (err) { + log?.warn?.("QODER", `attachment rewrite failed: ${err.message}`); + } + + const { messages, systemText } = normalizeMessages(incoming); const tools = body.tools; const isReasoning = !!modelConfig.is_reasoning; const maxOutputTokens = Number(modelConfig.max_output_tokens) || 0; @@ -227,7 +268,21 @@ async function buildQoderRequestBody({ model, body, credentials, log, proxyOptio const sessionId = stableHash("qoder-session", psd.userId, qoderKey); const recordId = stableChatRecordId(qoderKey, messages, tools, maxTokens); - return { + // Context-window tier (200K/400K/1M): the IDE picks one from model_config.context_config; + // qodercli-style requests default to the smallest. Escalate when the prompt no longer fits. + const tierChoice = resolveQoderContextTier( + modelConfig, + { system: systemText, messages, tools }, + { preference: process.env[QODER_CONTEXT_TIER_ENV] }, + ); + if (tierChoice) { + log?.info?.( + "QODER", + `context tier ${tierChoice.tier.name} (${tierChoice.tier.tokenCount} tokens, ${tierChoice.reason}) for ~${tierChoice.estimatedTokens} prompt tokens`, + ); + } + + const built = { qoderKey, payload: { request_id: uuidv4(), @@ -275,6 +330,8 @@ async function buildQoderRequestBody({ model, body, credentials, log, proxyOptio }, modelConfig, }; + if (tierChoice) applyQoderContextTier(built.payload, tierChoice.tier); + return built; } /** @@ -338,6 +395,11 @@ async function peekFirstQoderFrame(reader, decoder) { * response.text() which hangs until the socket closes — so on terminal * events we cancel the upstream reader and close our stream immediately. * + * Usage: Qoder puts finish_reason on `delta` and sends token counts on a + * later `choices: []` frame. Downstream OpenAI/Claude clients only read + * usage from the finish chunk, so we coalesce those two frames (see + * createQoderSseCoalescer) before forwarding. + * * NEW: Peek first frame to detect billing blocks (code 112/10605/pricingUrl). * If detected, return 403 response so chatCore marks connection unavailable * and triggers combo fallback instead of leaking error text into chat. @@ -364,6 +426,11 @@ async function wrapQoderSSE(response, model) { const upstreamDrained = peek.upstreamDone === true; const encoder = new TextEncoder(); let doneEmitted = false; + const coalescer = createQoderSseCoalescer({ model, encoder, sseDone: SSE_DONE }); + + const syncDone = () => { + if (coalescer.doneEmitted) doneEmitted = true; + }; // Process one already-extracted SSE line (no trailing newline). const processLine = (line, controller) => { @@ -374,15 +441,17 @@ async function wrapQoderSSE(response, model) { const data = trimmed.slice(5).trimStart(); if (data === "[DONE]") { - controller.enqueue(encoder.encode(SSE_DONE)); - doneEmitted = true; + coalescer.flush(controller); + syncDone(); return; } let envelope; try { envelope = JSON.parse(data); } catch { return; } const statusVal = typeof envelope.statusCodeValue === "number" ? envelope.statusCodeValue : 200; - const inner = typeof envelope.body === "string" ? envelope.body : ""; + const inner = typeof envelope.body === "string" + ? envelope.body + : envelope.body != null ? JSON.stringify(envelope.body) : ""; if (statusVal !== 200) { const msg = inner || `upstream status ${statusVal}`; const errChunk = JSON.stringify({ @@ -398,14 +467,8 @@ async function wrapQoderSSE(response, model) { return; } if (!inner) return; - if (inner === "[DONE]") { - controller.enqueue(encoder.encode(SSE_DONE)); - doneEmitted = true; - return; - } - // Strip embedded newlines so the SSE frame stays a single event. - const sanitized = inner.replace(/\r?\n/g, ""); - controller.enqueue(encoder.encode(`data: ${sanitized}\n\n`)); + coalescer.handleInner(inner, controller); + syncDone(); }; const stream = new ReadableStream({ @@ -464,7 +527,7 @@ async function wrapQoderSSE(response, model) { } finally { if (!doneEmitted) { try { - controller.enqueue(encoder.encode(SSE_DONE)); + coalescer.flush(controller); doneEmitted = true; } catch { /* already closed */ } } @@ -493,13 +556,7 @@ export class QoderExecutor extends BaseExecutor { } buildUrl(credentials) { - // Job-token (jt-...) traffic must hit api2.qoder.sh — api3 rejects jt- - // with "Login expired" (403). Device tokens (dt-...) stay on api3. - const raw = credentials?.apiKey || credentials?.accessToken; - if (typeof raw === "string" && !raw.startsWith("pt-") && (raw.startsWith("jt-") || (credentials?.accessToken || "").startsWith("jt-"))) { - return `${QODER_CHAT_BASE_ALT}/algo${QODER_CHAT_SIG_PATH}?FetchKeys=llm_model_result&AgentId=agent_common&Encode=1`; - } - return QODER_CHAT_URL_ENCODED; + return `${qoderInferenceBase(credentials)}/algo${QODER_CHAT_SIG_PATH}?FetchKeys=llm_model_result&AgentId=agent_common&Encode=1`; } // Override execute entirely — Qoder needs: diff --git a/open-sse/handlers/chatCore/nonStreamingHandler.js b/open-sse/handlers/chatCore/nonStreamingHandler.js index becc0842..3b6cfc6d 100644 --- a/open-sse/handlers/chatCore/nonStreamingHandler.js +++ b/open-sse/handlers/chatCore/nonStreamingHandler.js @@ -10,6 +10,7 @@ import { buildRequestDetail, extractRequestConfig, extractUsageFromResponse, sav import { appendRequestLog, saveRequestDetail } from "@/lib/usageDb.js"; import { decloakToolNames } from "../../utils/claudeCloaking.js"; import { ROLE, RESPONSES_ITEM } from "../../translator/schema/index.js"; +import { toResponsesUsage } from "../../translator/concerns/usage.js"; function parseToolArguments(value) { if (!value) return {}; @@ -130,11 +131,8 @@ function openAICompletionToResponses(responseBody, customToolNames = null) { background: false, error: null, output, - usage: { - input_tokens: usage.prompt_tokens || usage.input_tokens || 0, - output_tokens: usage.completion_tokens || usage.output_tokens || 0, - total_tokens: usage.total_tokens || (usage.prompt_tokens || 0) + (usage.completion_tokens || 0), - }, + // Keep cached/reasoning details (input_tokens_details) — proxies bill cache hits from them + usage: toResponsesUsage(usage) || { input_tokens: 0, output_tokens: 0, total_tokens: 0 }, }; } diff --git a/open-sse/handlers/chatCore/sseToJsonHandler.js b/open-sse/handlers/chatCore/sseToJsonHandler.js index 6801be89..0da5c69f 100644 --- a/open-sse/handlers/chatCore/sseToJsonHandler.js +++ b/open-sse/handlers/chatCore/sseToJsonHandler.js @@ -5,6 +5,7 @@ import { FORMATS } from "../../translator/formats.js"; import { PROVIDERS } from "../../config/providers.js"; import { buildRequestDetail, extractRequestConfig, saveUsageStats, formatDoneLine } from "./requestDetail.js"; import { ROLE, RESPONSES_ITEM } from "../../translator/schema/index.js"; +import { toResponsesUsage } from "../../translator/concerns/usage.js"; // Responses-API providers (e.g. codex) may emit SSE without content-type + use Responses output shape const isResponsesProvider = (p) => PROVIDERS[p]?.format === FORMATS.OPENAI_RESPONSES; @@ -97,11 +98,8 @@ function chatCompletionToResponses(responseBody, customToolNames = null) { background: false, error: null, output, - usage: { - input_tokens: usage.prompt_tokens || usage.input_tokens || 0, - output_tokens: usage.completion_tokens || usage.output_tokens || 0, - total_tokens: usage.total_tokens || (usage.prompt_tokens || 0) + (usage.completion_tokens || 0), - }, + // Keep cached/reasoning details (input_tokens_details) — proxies bill cache hits from them + usage: toResponsesUsage(usage) || { input_tokens: 0, output_tokens: 0, total_tokens: 0 }, }; } diff --git a/open-sse/providers/capabilities.js b/open-sse/providers/capabilities.js index dd9fa381..e011b788 100644 --- a/open-sse/providers/capabilities.js +++ b/open-sse/providers/capabilities.js @@ -222,9 +222,9 @@ export const PROVIDER_CAPABILITIES = { // windows (GLM-5.3 / Kimi-K3 / Qwen3.8-Max claim 180K but accept more). // max_output_tokens arrives as 0 for every model, so outputs are // best-guess from the real model family. Vision tags below follow the - // upstream is_vl flag per explicit request, even though the executor - // currently sends image_urls:null (image pass-through over the agent_chat - // SSE protocol is unverified). reasoning:true on all of them — every model can + // upstream is_vl flag. The executor uploads inlined images to + // /api/v2/image/upload and leaves image_urls/chat_context.imageUrls null + // (same as qodercli). reasoning:true on all of them — every model can // reason; the upstream is_reasoning flag only drives model_config selection. // thinkingFormat keeps the true-model family for documentation/UI, but // thinkingCanDisable:false everywhere: the executor only forwards diff --git a/open-sse/services/qoderModels.js b/open-sse/services/qoderModels.js index 572931e5..e9a7879b 100644 --- a/open-sse/services/qoderModels.js +++ b/open-sse/services/qoderModels.js @@ -343,6 +343,30 @@ export async function resolveQoderModels(credentials, options = {}) { } } +/** + * Every model key the chat endpoint accepts for this credential: the IDE-visible + * models first, then catalog entries flagged `enable:false` (hidden in the IDE + * picker, e.g. by an account policy, but still served by agent_chat_generation — + * see fetchQoderCatalogRaw). /v1/models uses this so the advertised list matches + * what the router will actually route instead of collapsing to one or two keys. + */ +export function routableQoderModels(catalog) { + if (!catalog) return []; + const out = []; + const seen = new Set(); + for (const m of catalog.models || []) { + if (!m?.id || seen.has(m.id)) continue; + seen.add(m.id); + out.push({ id: m.id, name: m.name || m.id, hidden: false }); + } + for (const [key, cfg] of catalog.rawConfigs || []) { + if (!key || seen.has(key)) continue; + seen.add(key); + out.push({ id: key, name: cfg?.display_name || key, hidden: true }); + } + return out; +} + export function invalidateQoderCatalog(credentials) { if (!credentials) return; catalogCache.delete(cacheKey(credentials)); diff --git a/open-sse/shared/qoder/attachments.js b/open-sse/shared/qoder/attachments.js new file mode 100644 index 00000000..d2024529 --- /dev/null +++ b/open-sse/shared/qoder/attachments.js @@ -0,0 +1,341 @@ +/** + * Native qodercli does NOT stuff image/PDF bytes into agent_chat_generation. + * It PUTs them to /algo/api/v2/image/upload (COSY-signed multipart) and then + * sends the returned OSS URL. Agents like Claude Code send OpenAI/Claude + * data-URIs instead, which 9router previously forwarded verbatim — 10MB + * images become 30MB+ JSON and upstream 413s even though the model window + * is ~200k tokens. + * + * This module: + * 1. Uploads inlined images to Qoder's file API (cached by sha256). + * 2. Replaces huge non-image file blocks with a short stub. + * 3. Caps leftover data-URIs so the chat JSON stays small. + */ + +import { createHash } from "crypto"; +import { v4 as uuidv4 } from "uuid"; + +import { proxyAwareFetch } from "../../utils/proxyFetch.js"; +import { parseDataUri } from "../../translator/concerns/image.js"; +import { OPENAI_BLOCK, CLAUDE_BLOCK } from "../../translator/schema/blocks.js"; +import { MAX_IMAGE_BYTES } from "../../config/mediaConfig.js"; +import { buildCosyHeaders } from "./cosy.js"; +import { + QODER_IMAGE_UPLOAD_SIG_PATH, + QODER_INLINE_FALLBACK_MAX_BYTES, + QODER_MAX_PAYLOAD_BYTES, + qoderInferenceBase, +} from "./constants.js"; + +const IMAGE_MIME_RE = /^image\//i; +const DATA_URI_RE = /data:[^;]+;base64,[A-Za-z0-9+/=\s]+/g; + +function mimeExt(mime) { + const m = String(mime || "").toLowerCase(); + if (m.includes("png")) return "png"; + if (m.includes("jpeg") || m.includes("jpg")) return "jpg"; + if (m.includes("gif")) return "gif"; + if (m.includes("webp")) return "webp"; + if (m.includes("bmp")) return "bmp"; + if (m.includes("pdf")) return "pdf"; + return "bin"; +} + +function decodedBytes(b64) { + if (typeof b64 !== "string" || !b64) return 0; + const compact = b64.replace(/\s/g, ""); + return Math.floor(compact.length * 3 / 4); +} + +function stubText({ name, mime, bytes, reason }) { + const label = name || mime || "attachment"; + const size = bytes ? `, ${bytes} bytes` : ""; + return `[file omitted: ${label}${size} — ${reason}]`; +} + +export function buildMultipartFile(buffer, { fieldName = "file", fileName, mediaType } = {}) { + const boundary = `----9routerQoder${Date.now().toString(16)}${Math.random().toString(16).slice(2)}`; + const filename = fileName || `upload.${mimeExt(mediaType)}`; + const head = Buffer.from( + `--${boundary}\r\nContent-Disposition: form-data; name="${fieldName}"; filename="${filename}"\r\nContent-Type: ${mediaType || "application/octet-stream"}\r\n\r\n`, + ); + const tail = Buffer.from(`\r\n--${boundary}--\r\n`); + const body = Buffer.concat([head, buffer, tail]); + return { boundary, body }; +} + +function extractUrlFromUploadResponse(json) { + if (!json || typeof json !== "object") return null; + const result = json.result && typeof json.result === "object" ? json.result : json; + const arrays = [result.imageUrls, result.image_urls, json.imageUrls, json.image_urls]; + for (const arr of arrays) { + if (Array.isArray(arr) && typeof arr[0] === "string" && arr[0]) return arr[0]; + } + const keys = ["imageUrl", "image_url", "url", "ossUrl", "oss_url", "originalUrl", "originUrl", "link", "image"]; + for (const key of keys) { + const v = result[key] ?? json[key]; + if (typeof v === "string" && v) return v; + } + if (typeof json.body === "string") { + try { return extractUrlFromUploadResponse(JSON.parse(json.body)); } catch { /* ignore */ } + } + return null; +} + +async function defaultUploadImage({ buffer, mediaType, credentials, proxyOptions, signal }) { + const requestId = uuidv4(); + const url = `${qoderInferenceBase(credentials)}${`/algo${QODER_IMAGE_UPLOAD_SIG_PATH}`}?request_id=${requestId}`; + const { boundary, body } = buildMultipartFile(buffer, { + fileName: `image.${mimeExt(mediaType)}`, + mediaType: mediaType || "application/octet-stream", + }); + const psd = credentials?.providerSpecificData || {}; + const cosyHeaders = buildCosyHeaders(body, url, { + userId: psd.userId, + authToken: credentials.accessToken, + name: credentials.displayName || "", + email: credentials.email || "", + machineId: psd.machineId || "", + }); + const headers = { + ...cosyHeaders, + Accept: "application/json", + "Content-Type": `multipart/form-data; boundary=${boundary}`, + "Content-Length": String(body.length), + "AI-CLIENT-TIMESTAMP": String(Math.floor(Date.now() / 1000)), + "Accept-Encoding": "identity", + }; + const res = await proxyAwareFetch( + url, + { method: "PUT", headers, body, signal }, + proxyOptions, + ); + if (!res.ok) { + const text = await res.text().catch(() => ""); + throw new Error(`HTTP ${res.status}${text ? `: ${text.slice(0, 180)}` : ""}`); + } + const json = await res.json().catch(() => null); + const uploaded = extractUrlFromUploadResponse(json); + if (!uploaded) throw new Error("upload response missing url"); + return uploaded; +} + +async function uploadImageData({ base64, mediaType, credentials, proxyOptions, signal, log, uploadFn, cache }) { + const compact = String(base64 || "").replace(/\s/g, ""); + if (!compact) return null; + const bytes = decodedBytes(compact); + if (bytes > MAX_IMAGE_BYTES) { + log?.warn?.("QODER", `image ${bytes} bytes exceeds upload cap, stubbing`); + return { stub: true, bytes, mime: mediaType }; + } + let buffer; + try { + buffer = Buffer.from(compact, "base64"); + } catch { + return { stub: true, bytes, mime: mediaType }; + } + const digest = createHash("sha256").update(buffer).digest("hex"); + if (cache?.has(digest)) return { url: cache.get(digest), bytes, mime: mediaType }; + + const doUpload = uploadFn || defaultUploadImage; + try { + const url = await doUpload({ buffer, mediaType, credentials, proxyOptions, signal }); + if (typeof url === "string" && url) { + cache?.set(digest, url); + return { url, bytes, mime: mediaType }; + } + } catch (err) { + log?.warn?.("QODER", `image upload failed (${err.message}); ${bytes <= QODER_INLINE_FALLBACK_MAX_BYTES ? "keeping inline" : "stubbing"}`); + } + if (bytes <= QODER_INLINE_FALLBACK_MAX_BYTES) return { keep: true, bytes, mime: mediaType }; + return { stub: true, bytes, mime: mediaType }; +} + +function imageUrlBlock(url) { + return { type: OPENAI_BLOCK.IMAGE_URL, image_url: { url } }; +} + +async function rewriteBlock(block, ctx) { + if (!block || typeof block !== "object") return block; + + if (block.type === OPENAI_BLOCK.IMAGE_URL) { + const raw = typeof block.image_url === "string" ? block.image_url : block.image_url?.url; + if (typeof raw !== "string" || !raw) return null; + if (raw.startsWith("http://") || raw.startsWith("https://")) return imageUrlBlock(raw); + const parsed = parseDataUri(raw); + if (!parsed) return { type: OPENAI_BLOCK.TEXT, text: stubText({ name: "attachment", reason: "unreadable data URI" }) }; + if (!IMAGE_MIME_RE.test(parsed.mimeType)) { + return { type: OPENAI_BLOCK.TEXT, text: stubText({ name: "file", mime: parsed.mimeType, bytes: decodedBytes(parsed.base64), reason: "non-image bytes are not inlined into Qoder context" }) }; + } + const up = await uploadImageData({ ...ctx, base64: parsed.base64, mediaType: parsed.mimeType }); + if (up?.url) return imageUrlBlock(up.url); + if (up?.keep) return imageUrlBlock(raw); + return { type: OPENAI_BLOCK.TEXT, text: stubText({ name: "image", mime: parsed.mimeType, bytes: up?.bytes, reason: "upload failed; not inlined" }) }; + } + + if (block.type === OPENAI_BLOCK.IMAGE || block.type === CLAUDE_BLOCK.IMAGE) { + const src = block.source || {}; + if (src.type === "url" && typeof src.url === "string") return imageUrlBlock(src.url); + if (src.type === "base64" && src.data) { + const mime = src.media_type || "image/png"; + const up = await uploadImageData({ ...ctx, base64: src.data, mediaType: mime }); + if (up?.url) return imageUrlBlock(up.url); + if (up?.keep) return imageUrlBlock(`data:${mime};base64,${src.data}`); + return { type: OPENAI_BLOCK.TEXT, text: stubText({ name: "image", mime, bytes: up?.bytes, reason: "upload failed; not inlined" }) }; + } + } + + if (block.type === OPENAI_BLOCK.FILE && block.file) { + const file = block.file; + const name = file.filename || file.name || "file"; + const dataUri = typeof file.file_data === "string" ? file.file_data : null; + const parsed = dataUri ? parseDataUri(dataUri) : null; + const b64 = parsed?.base64 || (typeof file.file_data === "string" && !file.file_data.startsWith("data:") ? file.file_data : null); + const mime = parsed?.mimeType || file.format || "application/octet-stream"; + if (b64 && IMAGE_MIME_RE.test(mime)) { + const up = await uploadImageData({ ...ctx, base64: b64, mediaType: mime }); + if (up?.url) return imageUrlBlock(up.url); + } + return { type: OPENAI_BLOCK.TEXT, text: stubText({ name, mime, bytes: decodedBytes(b64 || ""), reason: "Qoder reads documents via its file API, not inlined bytes" }) }; + } + + if (block.type === CLAUDE_BLOCK.DOCUMENT && block.source) { + const src = block.source; + const name = block.title || "document"; + if (src.type === "base64" && src.data) { + const mime = src.media_type || "application/pdf"; + if (IMAGE_MIME_RE.test(mime)) { + const up = await uploadImageData({ ...ctx, base64: src.data, mediaType: mime }); + if (up?.url) return imageUrlBlock(up.url); + } + return { type: OPENAI_BLOCK.TEXT, text: stubText({ name, mime, bytes: decodedBytes(src.data), reason: "Qoder reads documents via its file API, not inlined bytes" }) }; + } + } + + if (typeof block.text === "string" && block.text.includes("data:") && block.text.length > 8192) { + const next = block.text.replace(DATA_URI_RE, (m) => { + const parsed = parseDataUri(m.trim()); + const bytes = parsed ? decodedBytes(parsed.base64) : m.length; + if (bytes <= QODER_INLINE_FALLBACK_MAX_BYTES) return m; + return stubText({ mime: parsed?.mimeType, bytes, reason: "inlined data URI stripped from Qoder context" }); + }); + return { ...block, text: next }; + } + + return block; +} + +async function rewriteContent(content, ctx) { + if (typeof content === "string") { + if (content.includes("data:") && content.length > 8192) { + return content.replace(DATA_URI_RE, (m) => { + const parsed = parseDataUri(m.trim()); + const bytes = parsed ? decodedBytes(parsed.base64) : m.length; + if (bytes <= QODER_INLINE_FALLBACK_MAX_BYTES) return m; + return stubText({ mime: parsed?.mimeType, bytes, reason: "inlined data URI stripped from Qoder context" }); + }); + } + return content; + } + if (!Array.isArray(content)) return content; + const out = []; + for (const block of content) { + const next = await rewriteBlock(block, ctx); + if (next == null) continue; + out.push(next); + } + return out.length ? out : ""; +} + +function payloadBytes(messages) { + try { + return Buffer.byteLength(JSON.stringify(messages), "utf8"); + } catch { + return 0; + } +} + +function stripRemainingDataUris(messages) { + for (const msg of messages || []) { + if (typeof msg?.content === "string" && msg.content.includes("data:")) { + msg.content = msg.content.replace(DATA_URI_RE, (m) => + stubText({ bytes: m.length, reason: "payload over Qoder size budget" }), + ); + } else if (Array.isArray(msg?.content)) { + msg.content = msg.content.map((block) => { + if (block?.type === OPENAI_BLOCK.IMAGE_URL) { + const raw = typeof block.image_url === "string" ? block.image_url : block.image_url?.url; + if (typeof raw === "string" && raw.startsWith("data:")) { + return { type: OPENAI_BLOCK.TEXT, text: stubText({ name: "image", reason: "payload over Qoder size budget" }) }; + } + } + if (typeof block?.text === "string" && block.text.includes("data:")) { + return { ...block, text: block.text.replace(DATA_URI_RE, (m) => + stubText({ bytes: m.length, reason: "payload over Qoder size budget" }), + ) }; + } + return block; + }); + } + } +} + +/** + * Rewrite OpenAI-shaped messages in place: upload images, stub huge files. + * @returns {Promise<{imageUrls: string[], uploaded: number, stubbed: number}>} + */ +export async function rewriteQoderMessageAttachments(messages, { + credentials, + log, + proxyOptions = null, + signal = null, + uploadFn = null, +} = {}) { + const stats = { imageUrls: [], uploaded: 0, stubbed: 0 }; + if (!Array.isArray(messages) || messages.length === 0) return stats; + + const ctx = { credentials, log, proxyOptions, signal, uploadFn, cache: new Map() }; + + for (const msg of messages) { + if (!msg || typeof msg !== "object") continue; + if (Array.isArray(msg.images)) { + // Ollama-style sidecar; fold into content so normalizeMessages can see them. + const extras = msg.images.map((url) => imageUrlBlock(String(url))); + msg.content = Array.isArray(msg.content) + ? [...msg.content, ...extras] + : [{ type: OPENAI_BLOCK.TEXT, text: typeof msg.content === "string" ? msg.content : "" }, ...extras]; + delete msg.images; + } + msg.content = await rewriteContent(msg.content, ctx); + } + + // Collect surviving http(s) image URLs for callers that want image_urls. + for (const msg of messages) { + if (!Array.isArray(msg?.content)) continue; + for (const block of msg.content) { + const url = block?.type === OPENAI_BLOCK.IMAGE_URL + ? (typeof block.image_url === "string" ? block.image_url : block.image_url?.url) + : null; + if (typeof url === "string" && /^https?:\/\//i.test(url)) stats.imageUrls.push(url); + if (block?.type === OPENAI_BLOCK.TEXT && typeof block.text === "string" && block.text.startsWith("[file omitted:")) stats.stubbed += 1; + } + } + stats.uploaded = stats.imageUrls.length; + + if (payloadBytes(messages) > QODER_MAX_PAYLOAD_BYTES) { + log?.warn?.("QODER", `request still ${payloadBytes(messages)} bytes after rewrite; stripping leftover data URIs`); + stripRemainingDataUris(messages); + } + + return stats; +} + +/** Test helper kept for callers; upload memo is now per-request. */ +export function clearQoderUploadCache() {} + +export const __test__ = { + extractUrlFromUploadResponse, + decodedBytes, + stubText, + payloadBytes, +}; diff --git a/open-sse/shared/qoder/constants.js b/open-sse/shared/qoder/constants.js index 861e8f30..849f67ed 100644 --- a/open-sse/shared/qoder/constants.js +++ b/open-sse/shared/qoder/constants.js @@ -33,6 +33,39 @@ export const QODER_CHAT_SIG_PATH = "/api/v2/service/pro/sse/agent_chat_generatio export const QODER_CHAT_URL = `${QODER_CHAT_BASE}/algo${QODER_CHAT_SIG_PATH}?FetchKeys=llm_model_result&AgentId=agent_common`; export const QODER_CHAT_URL_ENCODED = `${QODER_CHAT_URL}&Encode=1`; export const QODER_MODEL_LIST_URL = `${QODER_CHAT_BASE}/algo/api/v2/model/list`; +// Official qodercli uploads images here (COSY-signed PUT multipart, field "file") +// instead of inlining base64 into agent_chat_generation. +export const QODER_IMAGE_UPLOAD_SIG_PATH = "/api/v2/image/upload"; + +// Drop remaining inlined binaries if the Qoder JSON body would still exceed this. +// 30MB+ payloads are what blow past Claude-Code's ~200k context on the wire. +export const QODER_MAX_PAYLOAD_BYTES = 6 * 1024 * 1024; +// If OSS upload fails, keep tiny data-URIs; anything larger is stubbed. +export const QODER_INLINE_FALLBACK_MAX_BYTES = 512 * 1024; + +// Context-window tier selection (see shared/qoder/contextTier.js). The IDE exposes the +// model's context_config tiers (200K/400K/1M); we auto-escalate when the estimated prompt +// (+ headroom, tokenizer variance) no longer fits the current max_input_tokens. +export const QODER_CONTEXT_TIER_HEADROOM = 0.15; +export const QODER_CONTEXT_TIER_ENV = "QODER_CONTEXT_TIER"; +export const QODER_CONTEXT_TIER_MODES = Object.freeze({ AUTO: "auto", MAX: "max", DEFAULT: "default" }); + +/** + * Job-token (jt-...) traffic must hit api2.qoder.sh — api3 rejects jt- with + * "Login expired" (403). Device tokens (dt-...) stay on api3. PATs (pt-...) + * are exchanged for jt- before this is consulted. + */ +export function qoderInferenceBase(credentials) { + const raw = credentials?.apiKey || credentials?.accessToken; + if ( + typeof raw === "string" && + !raw.startsWith("pt-") && + (raw.startsWith("jt-") || (credentials?.accessToken || "").startsWith("jt-")) + ) { + return QODER_CHAT_BASE_ALT; + } + return QODER_CHAT_BASE; +} // COSY header constants. These are not arbitrary — the upstream signature // validation matches them against the values used at signing time. diff --git a/open-sse/shared/qoder/contextTier.js b/open-sse/shared/qoder/contextTier.js new file mode 100644 index 00000000..523c2fe5 --- /dev/null +++ b/open-sse/shared/qoder/contextTier.js @@ -0,0 +1,160 @@ +/** + * Qoder context-window tiers. + * + * Each Qoder model_config ships a `context_config` list (e.g. 200K / 400K / 1M for + * qmodel_38max) while `max_input_tokens` only carries the tier the IDE currently has + * selected (~180K by default). The Qoder IDE lets the user switch tiers from the model + * picker; a qodercli-style client (which is what 9router impersonates) has no picker, + * so a long Claude-Code / Codex session that grew past the default tier is rejected + * upstream even though the model itself supports 1M. + * + * This module emulates the IDE: estimate the prompt size, pick the smallest advertised + * tier that fits (never below the model's current default), and mirror the choice into + * the same three places the IDE writes: + * parameters.context_length + * chat_context.extra.ideModelConfigOverride.max_input_tokens + * model_config.max_input_tokens + * + * Override with QODER_CONTEXT_TIER = auto (default) | max | default | . + * Pure functions, no I/O — the executor wires them into buildQoderRequestBody. + */ + +import { QODER_CONTEXT_TIER_HEADROOM, QODER_CONTEXT_TIER_MODES } from "./constants.js"; + +const UNIT = { K: 1_000, M: 1_000_000 }; + +/** "200K" | "1M" | "204800" | 204800 → integer token count (0 when unparseable). */ +export function parseTierTokenCount(value) { + if (typeof value === "number") return Number.isFinite(value) && value > 0 ? Math.floor(value) : 0; + if (typeof value !== "string") return 0; + const m = value.trim().toUpperCase().match(/^(\d+(?:\.\d+)?)\s*([KM])?$/); + if (!m) return 0; + const n = Number(m[1]) * (UNIT[m[2]] || 1); + return Number.isFinite(n) && n > 0 ? Math.floor(n) : 0; +} + +function tierName(entry, tokenCount) { + const raw = entry.name ?? entry.label ?? entry.display_name ?? entry.displayName ?? entry.key ?? entry.id; + if (typeof raw === "string" && raw.trim()) return raw.trim(); + if (tokenCount >= UNIT.M && tokenCount % UNIT.M === 0) return `${tokenCount / UNIT.M}M`; + if (tokenCount >= UNIT.K && tokenCount % UNIT.K === 0) return `${tokenCount / UNIT.K}K`; + return String(tokenCount); +} + +/** + * Normalize a model_config into sorted tiers: [{ name, tokenCount, isDefault }] ascending. + * Accepts snake_case and camelCase shapes; returns [] when the model has no tiers. + */ +export function getQoderContextTiers(modelConfig) { + const list = modelConfig?.context_config ?? modelConfig?.contextConfig; + if (!Array.isArray(list)) return []; + const byCount = new Map(); + for (const entry of list) { + if (!entry || typeof entry !== "object") continue; + const tokenCount = parseTierTokenCount( + entry.tokenCount ?? entry.token_count ?? entry.max_input_tokens ?? entry.maxInputTokens ?? entry.contextLength ?? entry.context_length, + ); + if (!tokenCount) continue; + const isDefault = entry.isDefault === true || entry.is_default === true || entry.default === true; + const prev = byCount.get(tokenCount); + byCount.set(tokenCount, { + name: tierName(entry, tokenCount), + tokenCount, + isDefault: (prev?.isDefault || false) || isDefault, + }); + } + return [...byCount.values()].sort((a, b) => a.tokenCount - b.tokenCount); +} + +const CJK_RE = /[\u1100-\u11ff\u2e80-\u9fff\uac00-\ud7af\uf900-\ufaff\uff00-\uffef]/g; + +/** + * Rough prompt-size estimate in tokens. CJK characters count ~1 token each, everything + * else ~4 chars/token — the plain chars/4 rule underestimates Chinese/Japanese by up to + * 4x, which is exactly when a tier decision matters. + */ +export function estimateQoderPromptTokens({ system, messages, tools } = {}) { + let text = ""; + try { + text = JSON.stringify({ system: system || "", messages: messages || [], tools: tools || [] }) || ""; + } catch { + return 0; + } + const cjk = (text.match(CJK_RE) || []).length; + return Math.ceil(cjk + (text.length - cjk) / 4); +} + +function normalizeMode(preference) { + const p = String(preference ?? "").trim(); + return p ? p : QODER_CONTEXT_TIER_MODES.AUTO; +} + +function findNamedTier(tiers, name) { + const wanted = name.replace(/\s+/g, "").toUpperCase(); + const asCount = parseTierTokenCount(wanted); + return tiers.find((t) => t.name.replace(/\s+/g, "").toUpperCase() === wanted || (asCount && t.tokenCount === asCount)) || null; +} + +/** + * Decide which tier a request should run under. + * + * @param {object} modelConfig raw Qoder model_config (has context_config + max_input_tokens) + * @param {{system?: string, messages?: any[], tools?: any[]}} prompt what will be sent + * @param {{preference?: string, headroom?: number}} [options] + * @returns {{ tier: {name, tokenCount, isDefault}, estimatedTokens: number, reason: string } | null} + * null → leave the payload exactly as before (no tiers, or the default already fits). + */ +export function resolveQoderContextTier(modelConfig, prompt, options = {}) { + const tiers = getQoderContextTiers(modelConfig); + if (!tiers.length) return null; + + const mode = normalizeMode(options.preference); + const largest = tiers[tiers.length - 1]; + const defaultTier = tiers.find((t) => t.isDefault) || tiers[0]; + const estimatedTokens = estimateQoderPromptTokens(prompt); + const headroom = typeof options.headroom === "number" ? options.headroom : QODER_CONTEXT_TIER_HEADROOM; + const need = Math.ceil(estimatedTokens * (1 + headroom)); + + if (mode.toLowerCase() === QODER_CONTEXT_TIER_MODES.MAX) { + return { tier: largest, estimatedTokens, reason: "forced:max" }; + } + if (mode.toLowerCase() === QODER_CONTEXT_TIER_MODES.DEFAULT) { + return { tier: defaultTier, estimatedTokens, reason: "forced:default" }; + } + if (mode.toLowerCase() !== QODER_CONTEXT_TIER_MODES.AUTO) { + const named = findNamedTier(tiers, mode); + if (named) return { tier: named, estimatedTokens, reason: `forced:${named.name}` }; + // Unknown tier name → fall through to auto rather than silently breaking requests. + } + + // auto: keep the upstream default (current behaviour) while the prompt fits in it. + const currentMax = parseTierTokenCount(modelConfig?.max_input_tokens ?? modelConfig?.maxInputTokens); + const currentLimit = currentMax || defaultTier.tokenCount; + if (need <= currentLimit) return null; + + const fits = tiers.find((t) => t.tokenCount >= need && t.tokenCount > currentLimit); + const tier = fits || largest; + if (tier.tokenCount <= currentLimit) return null; // nothing bigger to escalate to + return { tier, estimatedTokens, reason: fits ? "auto:fits" : "auto:largest" }; +} + +/** + * Write the chosen tier into a Qoder chat payload (mutates + returns it). + * Mirrors the IDE: parameters.context_length, ideModelConfigOverride, model_config. + */ +export function applyQoderContextTier(payload, tier) { + if (!payload || !tier?.tokenCount) return payload; + payload.parameters = { ...(payload.parameters || {}), context_length: tier.tokenCount }; + payload.chat_context = payload.chat_context || {}; + payload.chat_context.extra = { + ...(payload.chat_context.extra || {}), + ideModelConfigOverride: { + ...(payload.chat_context.extra?.ideModelConfigOverride || {}), + max_input_tokens: tier.tokenCount, + }, + }; + if (payload.model_config && typeof payload.model_config === "object") { + payload.model_config = { ...payload.model_config, max_input_tokens: tier.tokenCount }; + } + return payload; +} diff --git a/open-sse/shared/qoder/sse.js b/open-sse/shared/qoder/sse.js new file mode 100644 index 00000000..ad30785e --- /dev/null +++ b/open-sse/shared/qoder/sse.js @@ -0,0 +1,208 @@ +/** + * Qoder SSE is OpenAI-shaped inside `{statusCodeValue, body}` envelopes, but + * usage arrives on a later `choices: []` frame — after finish_reason, which + * itself often lives on `delta.finish_reason` rather than the choice. + * + * Downstream (Claude translator, OpenAI clients, Claude Code) look for usage + * on the finish chunk or drop `choices: []` entirely. 9router's own dashboard + * still sees tokens because extractUsage runs on every forwarded frame. + * + * Coalesce: hold empty finish + usage-only frames, then emit one OpenAI + * include_usage-style chunk: `{choices:[{delta:{}, finish_reason}], usage}`. + */ + +function num(v) { + const n = Number(v); + return Number.isFinite(n) ? n : null; +} + +/** + * Normalize Qoder/OpenAI usage into the shape stream.js + Claude translation + * already understand (prompt_tokens + prompt_tokens_details.cached_tokens). + */ +export function canonicalizeQoderUsage(usage) { + if (!usage || typeof usage !== "object" || Array.isArray(usage)) return null; + + const prompt = num(usage.prompt_tokens ?? usage.input_tokens); + const completion = num(usage.completion_tokens ?? usage.output_tokens); + if (prompt == null && completion == null) return null; + + const details = (usage.prompt_tokens_details && typeof usage.prompt_tokens_details === "object") + ? { ...usage.prompt_tokens_details } + : {}; + const cached = num( + details.cached_tokens ?? + usage.cached_tokens ?? + usage.prompt_cache_hit_tokens ?? + usage.cache_read_input_tokens, + ); + const cacheCreation = num( + details.cache_creation_tokens ?? + usage.cache_creation_input_tokens, + ); + + const promptTokens = prompt || 0; + const completionTokens = completion || 0; + const out = { + prompt_tokens: promptTokens, + completion_tokens: completionTokens, + total_tokens: num(usage.total_tokens) ?? (promptTokens + completionTokens), + }; + + if (cached != null) { + out.cached_tokens = cached; + details.cached_tokens = cached; + } + if (cacheCreation != null) { + details.cache_creation_tokens = cacheCreation; + } + if (Object.keys(details).length) out.prompt_tokens_details = details; + + if (usage.completion_tokens_details && typeof usage.completion_tokens_details === "object") { + out.completion_tokens_details = usage.completion_tokens_details; + } + const reasoning = num(usage.reasoning_tokens ?? usage.completion_tokens_details?.reasoning_tokens); + if (reasoning != null) out.reasoning_tokens = reasoning; + + return out; +} + +function finishReasonOf(parsed) { + const choice = parsed?.choices?.[0]; + return choice?.finish_reason || choice?.delta?.finish_reason || parsed?.finish_reason || null; +} + +function hasValuableDelta(parsed) { + const delta = parsed?.choices?.[0]?.delta; + if (!delta || typeof delta !== "object") return false; + if (typeof delta.content === "string" && delta.content.length > 0) return true; + if (typeof delta.reasoning_content === "string" && delta.reasoning_content.length > 0) return true; + if (Array.isArray(delta.tool_calls) && delta.tool_calls.length > 0) return true; + if (delta.role) return true; + return false; +} + +function parseInner(inner) { + if (inner == null || inner === "") return { raw: false, parsed: null }; + if (inner === "[DONE]") return { done: true }; + if (typeof inner !== "string") { + if (typeof inner === "object") return { parsed: inner }; + return { raw: true, text: String(inner) }; + } + try { + return { parsed: JSON.parse(inner) }; + } catch { + return { raw: true, text: inner }; + } +} + +/** + * @param {object} opts + * @param {string} opts.model + * @param {TextEncoder} opts.encoder + * @param {string} opts.sseDone "data: [DONE]\\n\\n" + */ +export function createQoderSseCoalescer({ model, encoder, sseDone }) { + let pendingFinish = null; + let pendingUsage = null; + let lastMeta = { id: null, created: null, model }; + let doneEmitted = false; + let finishAlreadyForwarded = false; + + const emitJson = (controller, obj) => { + const sanitized = JSON.stringify(obj).replace(/\r?\n/g, ""); + controller.enqueue(encoder.encode(`data: ${sanitized}\n\n`)); + }; + + const emitRaw = (controller, text) => { + controller.enqueue(encoder.encode(`data: ${String(text).replace(/\r?\n/g, "")}\n\n`)); + }; + + const emitDone = (controller) => { + if (doneEmitted) return; + controller.enqueue(encoder.encode(sseDone)); + doneEmitted = true; + }; + + const emitTerminal = (controller) => { + if (!pendingFinish && !pendingUsage) return; + emitJson(controller, { + id: lastMeta.id || `qoder-${Date.now()}`, + object: "chat.completion.chunk", + created: lastMeta.created || Math.floor(Date.now() / 1000), + model: lastMeta.model || model, + choices: [{ index: 0, delta: {}, finish_reason: pendingFinish || "stop" }], + ...(pendingUsage ? { usage: pendingUsage } : {}), + }); + pendingFinish = null; + pendingUsage = null; + }; + + const flush = (controller) => { + if (doneEmitted) return; + if (pendingUsage || (pendingFinish && !finishAlreadyForwarded)) { + emitTerminal(controller); + } + emitDone(controller); + }; + + const handleInner = (inner, controller) => { + if (doneEmitted) return { terminal: true }; + + const parsedInner = parseInner(inner); + if (parsedInner.done) { + flush(controller); + return { terminal: true }; + } + if (parsedInner.raw) { + emitRaw(controller, parsedInner.text); + return {}; + } + const parsed = parsedInner.parsed; + if (!parsed || typeof parsed !== "object") return {}; + + if (typeof parsed.id === "string" && parsed.id) lastMeta.id = parsed.id; + if (typeof parsed.created === "number") lastMeta.created = parsed.created; + if (typeof parsed.model === "string" && parsed.model) lastMeta.model = parsed.model; + + const usage = canonicalizeQoderUsage(parsed.usage); + if (usage) pendingUsage = usage; + + const finish = finishReasonOf(parsed); + if (hasValuableDelta(parsed)) { + // Stream content as-is (preserves upstream JSON for tests/clients). + emitRaw(controller, typeof inner === "string" ? inner : JSON.stringify(parsed)); + if (finish) { + finishAlreadyForwarded = true; + // Keep finish around only if we still need a usage trailer. + pendingFinish = pendingUsage ? finish : null; + } + if (pendingFinish && pendingUsage) { + emitTerminal(controller); + emitDone(controller); + return { terminal: true }; + } + return {}; + } + + if (finish) pendingFinish = finish; + + // Empty finish and/or usage-only: emit as soon as we have both (Qoder + // order is finish then usage). Don't wait for the later [DONE]/keepalive. + if ((pendingFinish || finishAlreadyForwarded) && pendingUsage) { + if (!pendingFinish) pendingFinish = "stop"; + emitTerminal(controller); + emitDone(controller); + return { terminal: true }; + } + return {}; + }; + + return { + handleInner, + flush, + get doneEmitted() { + return doneEmitted; + }, + }; +} diff --git a/open-sse/transformer/responsesTransformer.js b/open-sse/transformer/responsesTransformer.js index ac84db20..6558a3f1 100644 --- a/open-sse/transformer/responsesTransformer.js +++ b/open-sse/transformer/responsesTransformer.js @@ -6,6 +6,7 @@ import fs from "fs"; import path from "path"; +import { toResponsesUsage } from "../translator/concerns/usage.js"; // Create log directory for responses (Node.js only) export function createResponsesLogger(model, logsDir = null) { @@ -73,6 +74,7 @@ export function createResponsesApiTransformStream(logger = null) { funcArgsDone: {}, funcItemDone: {}, buffer: "", + usage: null, completedSent: false }; @@ -225,17 +227,17 @@ export function createResponsesApiTransformStream(logger = null) { const sendCompleted = (controller) => { if (!state.completedSent) { state.completedSent = true; - emit(controller, "response.completed", { - type: "response.completed", - response: { - id: state.responseId, - object: "response", - created_at: state.created, - status: "completed", - background: false, - error: null - } - }); + const response = { + id: state.responseId, + object: "response", + created_at: state.created, + status: "completed", + background: false, + error: null + }; + const usage = toResponsesUsage(state.usage); + if (usage) response.usage = usage; + emit(controller, "response.completed", { type: "response.completed", response }); } }; @@ -264,6 +266,9 @@ export function createResponsesApiTransformStream(logger = null) { continue; } + // Remember usage (finish chunk or trailing include_usage frame) for response.completed + if (parsed.usage && typeof parsed.usage === "object") state.usage = parsed.usage; + if (!parsed.choices?.length) continue; const choice = parsed.choices[0]; diff --git a/open-sse/translator/concerns/usage.js b/open-sse/translator/concerns/usage.js index 3ace3062..1ee28489 100644 --- a/open-sse/translator/concerns/usage.js +++ b/open-sse/translator/concerns/usage.js @@ -67,3 +67,34 @@ export function toOpenAIUsage(raw, kind) { if (!extract || !raw || typeof raw !== "object") return null; return buildUsage(extract(raw)); } + +// Convert an OpenAI-shaped (or already-canonical / Claude-shaped) usage object into the +// Responses API shape emitted by `response.completed`. Details objects are always present +// (like the real API) so proxies that read `input_tokens_details.cached_tokens` never see undefined. +// Returns null when there is nothing countable. +export function toResponsesUsage(usage) { + if (!usage || typeof usage !== "object") return null; + const input = n(usage.prompt_tokens ?? usage.input_tokens); + const output = n(usage.completion_tokens ?? usage.output_tokens); + if (input === 0 && output === 0) return null; + const cached = n( + usage.input_tokens_details?.cached_tokens ?? + usage.prompt_tokens_details?.cached_tokens ?? + usage.cached_tokens ?? + usage.cache_read_input_tokens + ); + const reasoning = n( + usage.output_tokens_details?.reasoning_tokens ?? + usage.completion_tokens_details?.reasoning_tokens ?? + usage.reasoning_tokens + ); + const out = { + input_tokens: input, + output_tokens: output, + total_tokens: typeof usage.total_tokens === "number" ? usage.total_tokens : input + output, + input_tokens_details: { cached_tokens: cached }, + output_tokens_details: { reasoning_tokens: reasoning }, + }; + if (usage.estimated) out.estimated = true; + return out; +} diff --git a/open-sse/translator/response/openai-responses.js b/open-sse/translator/response/openai-responses.js index bd435f9c..1d9a7dec 100644 --- a/open-sse/translator/response/openai-responses.js +++ b/open-sse/translator/response/openai-responses.js @@ -5,7 +5,7 @@ import { register } from "../index.js"; import { FORMATS } from "../formats.js"; import { buildChunk } from "../concerns/chunk.js"; -import { buildUsage } from "../concerns/usage.js"; +import { buildUsage, toResponsesUsage } from "../concerns/usage.js"; import { fallbackToolCallId } from "../concerns/toolCall.js"; import { reasoningDelta, extractReasoningText } from "../concerns/reasoning.js"; import { ROLE, OPENAI_BLOCK, RESPONSES_ITEM, OPENAI_FINISH, MODEL_FALLBACK } from "../schema/index.js"; @@ -18,7 +18,13 @@ export function openaiToOpenAIResponsesResponse(chunk, state) { if (!chunk) { return flushEvents(state); } - + + // Usage riding on the finish chunk (include_usage style, e.g. coalesced Qoder frames): + // remember it so response.completed can report tokens even outside stream.js. + if (chunk.usage && typeof chunk.usage === "object" && !state.usage) { + state.usage = chunk.usage; + } + if (!chunk.choices?.length) return []; const events = []; @@ -368,17 +374,19 @@ function closeToolCall(state, emit, idx) { function sendCompleted(state, emit) { if (!state.completedSent) { state.completedSent = true; - emit("response.completed", { - type: "response.completed", - response: { - id: state.responseId, - object: "response", - created_at: state.created, - status: "completed", - background: false, - error: null - } - }); + const response = { + id: state.responseId, + object: "response", + created_at: state.created, + status: "completed", + background: false, + error: null + }; + // Carry provider usage (recorded by stream.js or from the finish chunk itself) in the + // Responses shape; proxies such as sub2api/Codex read tokens only from here. + const usage = toResponsesUsage(state.usage); + if (usage) response.usage = usage; + emit("response.completed", { type: "response.completed", response }); } } diff --git a/open-sse/translator/response/openai-to-claude.js b/open-sse/translator/response/openai-to-claude.js index 3998cc84..f291e83a 100644 --- a/open-sse/translator/response/openai-to-claude.js +++ b/open-sse/translator/response/openai-to-claude.js @@ -67,48 +67,50 @@ function stopTextBlock(state, results) { state.textBlockStarted = false; } +function recordOpenAIUsage(chunk, state) { + if (!chunk?.usage || typeof chunk.usage !== "object") return; + + const promptTokens = typeof chunk.usage.prompt_tokens === "number" ? chunk.usage.prompt_tokens : 0; + const outputTokens = typeof chunk.usage.completion_tokens === "number" ? chunk.usage.completion_tokens : 0; + + // Extract cache tokens from prompt_tokens_details + const cachedTokens = chunk.usage.prompt_tokens_details?.cached_tokens; + const cacheCreationTokens = chunk.usage.prompt_tokens_details?.cache_creation_tokens; + const cacheReadTokens = typeof cachedTokens === "number" ? cachedTokens : 0; + const cacheCreateTokens = typeof cacheCreationTokens === "number" ? cacheCreationTokens : 0; + + // input_tokens = prompt_tokens - cached_tokens - cache_creation_tokens + // Because OpenAI's prompt_tokens includes all prompt-side tokens + const inputTokens = promptTokens - cacheReadTokens - cacheCreateTokens; + + state.usage = { + input_tokens: inputTokens, + output_tokens: outputTokens + }; + + if (cacheReadTokens > 0) { + state.usage.cache_read_input_tokens = cacheReadTokens; + } + if (cacheCreateTokens > 0) { + state.usage.cache_creation_input_tokens = cacheCreateTokens; + } +} + // Convert OpenAI stream chunk to Claude format export function openaiToClaudeResponse(chunk, state) { - if (!chunk || !chunk.choices?.[0]) return null; + if (!chunk) return null; + + // Track usage from OpenAI chunk if available + if (chunk.usage && typeof chunk.usage === "object") { + recordOpenAIUsage(chunk, state); + } + + if (!chunk.choices?.[0]) return null; const results = []; const choice = chunk.choices[0]; const delta = choice.delta; - // Track usage from OpenAI chunk if available - if (chunk.usage && typeof chunk.usage === "object") { - const promptTokens = typeof chunk.usage.prompt_tokens === "number" ? chunk.usage.prompt_tokens : 0; - const outputTokens = typeof chunk.usage.completion_tokens === "number" ? chunk.usage.completion_tokens : 0; - - // Extract cache tokens from prompt_tokens_details - const cachedTokens = chunk.usage.prompt_tokens_details?.cached_tokens; - const cacheCreationTokens = chunk.usage.prompt_tokens_details?.cache_creation_tokens; - const cacheReadTokens = typeof cachedTokens === "number" ? cachedTokens : 0; - const cacheCreateTokens = typeof cacheCreationTokens === "number" ? cacheCreationTokens : 0; - - // input_tokens = prompt_tokens - cached_tokens - cache_creation_tokens - // Because OpenAI's prompt_tokens includes all prompt-side tokens - const inputTokens = promptTokens - cacheReadTokens - cacheCreateTokens; - - state.usage = { - input_tokens: inputTokens, - output_tokens: outputTokens - }; - - // Add cache_read_input_tokens if present - if (cacheReadTokens > 0) { - state.usage.cache_read_input_tokens = cacheReadTokens; - } - - // Add cache_creation_input_tokens if present - if (cacheCreateTokens > 0) { - state.usage.cache_creation_input_tokens = cacheCreateTokens; - } - - // Note: completion_tokens_details.reasoning_tokens is already included in output_tokens - // No need to add separately as Claude expects total output_tokens - } - // First chunk - ALWAYS send message_start first if (!state.messageStartSent) { state.messageStartSent = true; @@ -221,8 +223,9 @@ export function openaiToClaudeResponse(chunk, state) { } } - // Finish - if (choice.finish_reason) { + // Finish (OpenAI puts this on the choice; Qoder often puts it on delta) + const finishReason = choice.finish_reason || delta?.finish_reason; + if (finishReason) { stopThinkingBlock(state, results); stopTextBlock(state, results); @@ -244,13 +247,13 @@ export function openaiToClaudeResponse(chunk, state) { } // Mark finish for later usage injection in stream.js - state.finishReason = choice.finish_reason; + state.finishReason = finishReason; // Use tracked usage (will be estimated in stream.js if not valid) const finalUsage = state.usage || { input_tokens: 0, output_tokens: 0 }; results.push({ type: "message_delta", - delta: { stop_reason: convertFinishReason(choice.finish_reason) }, + delta: { stop_reason: convertFinishReason(finishReason) }, usage: finalUsage }); results.push({ type: "message_stop" }); diff --git a/open-sse/utils/stream.js b/open-sse/utils/stream.js index 15ee37d8..f61fd193 100644 --- a/open-sse/utils/stream.js +++ b/open-sse/utils/stream.js @@ -3,6 +3,7 @@ import { FORMATS } from "../translator/formats.js"; import { trackPendingRequest, appendRequestLog } from "@/lib/usageDb.js"; import { extractUsage, mergeUsage, hasValidUsage, estimateUsage, logUsage, addBufferToUsage, filterUsageForFormat, COLORS } from "./usageTracking.js"; import { parseSSELine, hasValuableContent, fixInvalidId, formatSSE } from "./streamHelpers.js"; +import { toResponsesUsage } from "../translator/concerns/usage.js"; import { getOpenAIResponsesEventName, isOpenAIResponsesTerminalEvent, formatIncompleteOpenAIResponsesStreamFailure } from "./responsesStreamHelpers.js"; import { dbg, isDebugEnabled } from "./debugLog.js"; @@ -201,7 +202,8 @@ export function createSSEStream(options = {}) { responsesTerminal = isOpenAIResponsesTerminalEvent(currentOpenAIResponsesEvent, parsed); - const isFinishChunk = parsed.choices?.[0]?.finish_reason; + const isFinishChunk = parsed.choices?.[0]?.finish_reason + || parsed.choices?.[0]?.delta?.finish_reason; if (isFinishChunk && !hasValidUsage(parsed.usage)) { const estimated = estimateUsage(body, totalContentLength, FORMATS.OPENAI); parsed.usage = filterUsageForFormat(estimated, FORMATS.OPENAI); @@ -365,6 +367,19 @@ export function createSSEStream(options = {}) { item.usage = filterUsageForFormat(buffered, sourceFormat); } + // Responses API clients (Codex, sub2api /v1/responses): usage lives on + // response.completed → response.usage. Same buffer/estimate policy as above. + const completedResponse = item.event === "response.completed" ? item.data?.response : null; + if (completedResponse && typeof completedResponse === "object") { + if (state.usage) { + completedResponse.usage = toResponsesUsage(addBufferToUsage(state.usage)) ?? completedResponse.usage; + } else if (!completedResponse.usage && totalContentLength > 0) { + const estimated = estimateUsage(body, totalContentLength, FORMATS.OPENAI); + completedResponse.usage = toResponsesUsage(estimated); + state.usage = estimated; + } + } + const output = formatSSE(item, sourceFormat); reqLogger?.appendConvertedChunk?.(output); controller.enqueue(sharedEncoder.encode(output)); diff --git a/src/app/api/v1/models/route.js b/src/app/api/v1/models/route.js index 40adc696..173d91c2 100644 --- a/src/app/api/v1/models/route.js +++ b/src/app/api/v1/models/route.js @@ -9,7 +9,7 @@ import { getProviderConnections, getCombos, getCustomModels, getModelAliases } f import { getDisabledModels } from "@/lib/disabledModelsDb"; import { resolveKiroModels } from "open-sse/services/kiroModels.js"; import { resolveKimchiModels } from "open-sse/services/kimchiModels.js"; -import { resolveQoderModels } from "open-sse/services/qoderModels.js"; +import { resolveQoderModels, routableQoderModels } from "open-sse/services/qoderModels.js"; import { resolveCopilotModels } from "open-sse/services/copilotModels.js"; import { resolveClinepassModels } from "open-sse/services/clinepassModels.js"; import { resolveGrokCliModels } from "open-sse/services/grokCliModels.js"; @@ -34,15 +34,18 @@ const LIVE_MODEL_RESOLVERS = { qoder: async (conn) => { const result = await resolveQoderModels({ accessToken: conn.accessToken, + // PAT (pt-...) connections keep the token in apiKey; without it the live + // catalog silently fails and /v1/models falls back to the static list. + apiKey: conn.apiKey, refreshToken: conn.refreshToken, email: conn.email, displayName: conn.displayName, providerSpecificData: conn.providerSpecificData || {} }); - if (!result?.models?.length) return null; - return { - models: result.models.map((m) => ({ id: m.id, name: m.name })), - }; + // Visible + hidden (enable:false) catalog keys — chat routes all of them. + const models = routableQoderModels(result); + if (!models.length) return null; + return { models: models.map((m) => ({ id: m.id, name: m.name })) }; }, kimchi: async (conn) => { const result = await resolveKimchiModels({ diff --git a/tests/unit/openai-responses-usage.test.js b/tests/unit/openai-responses-usage.test.js new file mode 100644 index 00000000..38ec581d --- /dev/null +++ b/tests/unit/openai-responses-usage.test.js @@ -0,0 +1,182 @@ +/** + * Responses API clients (Codex, sub2api /v1/responses) read token usage only from + * `response.completed → response.usage`. For chat-native upstreams (Qoder, most + * OpenAI-compatible providers) the translator used to emit that event without usage, + * so proxies logged 0 input / 0 output / 0 cached tokens. + */ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/lib/usageDb.js", () => ({ + appendRequestLog: vi.fn(async () => {}), + saveRequestDetail: vi.fn(async () => {}), + saveRequestUsage: vi.fn(async () => {}), + trackPendingRequest: vi.fn(() => {}), +})); + +const { FORMATS } = await import("../../open-sse/translator/formats.js"); +const { initState } = await import("../../open-sse/translator/index.js"); +const { toResponsesUsage } = await import("../../open-sse/translator/concerns/usage.js"); +const { openaiToOpenAIResponsesResponse } = await import("../../open-sse/translator/response/openai-responses.js"); +const { createSSETransformStreamWithLogger } = await import("../../open-sse/utils/stream.js"); +const { createResponsesApiTransformStream } = await import("../../open-sse/transformer/responsesTransformer.js"); +const { addBufferToUsage } = await import("../../open-sse/utils/usageTracking.js"); +// stream.js adds the same context-safety buffer it applies to chat/claude clients +const BUFFER_TOKENS = addBufferToUsage({ prompt_tokens: 0 }).prompt_tokens; + +const QODER_FINISH_CHUNK = { + id: "chatcmpl-qoder-1", + object: "chat.completion.chunk", + created: 1_700_000_000, + model: "qmodel_38max", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { + prompt_tokens: 27_339, + completion_tokens: 437, + total_tokens: 27_776, + prompt_tokens_details: { cached_tokens: 27_200 }, + }, +}; + +function sse(chunks) { + return chunks.map((c) => `data: ${typeof c === "string" ? c : JSON.stringify(c)}\n\n`).join(""); +} + +async function pipe(input, transform) { + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(input)); + controller.close(); + }, + }); + const reader = stream.pipeThrough(transform).getReader(); + const decoder = new TextDecoder(); + let text = ""; + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + text += decoder.decode(value, { stream: true }); + } + return text + decoder.decode(); +} + +function completedEvent(text) { + const m = text.match(/event: response\.completed\ndata: (.+)\n/); + return m ? JSON.parse(m[1]) : null; +} + +describe("toResponsesUsage", () => { + it("maps OpenAI usage (nested cached_tokens) to the Responses shape", () => { + expect(toResponsesUsage(QODER_FINISH_CHUNK.usage)).toEqual({ + input_tokens: 27_339, + output_tokens: 437, + total_tokens: 27_776, + input_tokens_details: { cached_tokens: 27_200 }, + output_tokens_details: { reasoning_tokens: 0 }, + }); + }); + + it("accepts canonical flat fields and Claude-style cache fields", () => { + expect(toResponsesUsage({ prompt_tokens: 10, completion_tokens: 2, cached_tokens: 4, reasoning_tokens: 1 })).toMatchObject({ + input_tokens: 10, + output_tokens: 2, + total_tokens: 12, + input_tokens_details: { cached_tokens: 4 }, + output_tokens_details: { reasoning_tokens: 1 }, + }); + expect(toResponsesUsage({ input_tokens: 5, output_tokens: 1, cache_read_input_tokens: 3 }).input_tokens_details.cached_tokens).toBe(3); + }); + + it("keeps the estimated marker and returns null for empty usage", () => { + expect(toResponsesUsage({ prompt_tokens: 1, completion_tokens: 1, estimated: true }).estimated).toBe(true); + expect(toResponsesUsage({})).toBeNull(); + expect(toResponsesUsage(null)).toBeNull(); + }); +}); + +describe("openai → openai-responses translator", () => { + it("puts usage from the finish chunk on response.completed", () => { + const state = initState(FORMATS.OPENAI_RESPONSES); + const events = openaiToOpenAIResponsesResponse(QODER_FINISH_CHUNK, state); + const completed = events.find((e) => e.event === "response.completed"); + expect(completed).toBeTruthy(); + expect(completed.data.response.usage).toEqual({ + input_tokens: 27_339, + output_tokens: 437, + total_tokens: 27_776, + input_tokens_details: { cached_tokens: 27_200 }, + output_tokens_details: { reasoning_tokens: 0 }, + }); + }); + + it("omits usage when the upstream never reported any", () => { + const state = initState(FORMATS.OPENAI_RESPONSES); + const events = openaiToOpenAIResponsesResponse({ ...QODER_FINISH_CHUNK, usage: undefined }, state); + const completed = events.find((e) => e.event === "response.completed"); + expect(completed.data.response.usage).toBeUndefined(); + }); +}); + +describe("stream.js translate mode: chat upstream → Responses client", () => { + const transform = () => createSSETransformStreamWithLogger( + FORMATS.OPENAI, // provider (Qoder executor emits OpenAI chunks) + FORMATS.OPENAI_RESPONSES, // client + "qoder", + null, + null, + "qmodel_38max", + null, + { model: "qd/qmodel_38max", messages: [{ role: "user", content: "hi" }] }, + ); + + it("emits provider usage (+buffer) with cached tokens on response.completed", async () => { + const out = await pipe(sse([ + { ...QODER_FINISH_CHUNK, choices: [{ index: 0, delta: { role: "assistant", content: "Hello" }, finish_reason: null }], usage: undefined }, + QODER_FINISH_CHUNK, + "[DONE]", + ]), transform()); + + const completed = completedEvent(out); + expect(completed).toBeTruthy(); + expect(completed.response.usage).toEqual({ + input_tokens: 27_339 + BUFFER_TOKENS, + output_tokens: 437, + total_tokens: 27_776 + BUFFER_TOKENS, + input_tokens_details: { cached_tokens: 27_200 }, + output_tokens_details: { reasoning_tokens: 0 }, + }); + // Responses clients terminate on response.completed (no [DONE] sentinel in translate mode) + expect(out.indexOf("event: response.completed")).toBeGreaterThan(out.indexOf("event: response.output_item.done")); + }); + + it("injects estimated usage when the upstream reports none", async () => { + const out = await pipe(sse([ + { ...QODER_FINISH_CHUNK, choices: [{ index: 0, delta: { role: "assistant", content: "Hello world" }, finish_reason: null }], usage: undefined }, + { ...QODER_FINISH_CHUNK, usage: undefined }, + "[DONE]", + ]), transform()); + + const completed = completedEvent(out); + expect(completed.response.usage).toBeTruthy(); + expect(completed.response.usage.estimated).toBe(true); + expect(completed.response.usage.input_tokens).toBeGreaterThan(0); + expect(completed.response.usage.output_tokens).toBeGreaterThan(0); + }); +}); + +describe("responsesTransformer (Chat SSE → Codex Responses SSE)", () => { + it("forwards finish-chunk usage on response.completed", async () => { + const out = await pipe(sse([ + { ...QODER_FINISH_CHUNK, choices: [{ index: 0, delta: { role: "assistant", content: "Hello" }, finish_reason: null }], usage: undefined }, + QODER_FINISH_CHUNK, + "[DONE]", + ]), createResponsesApiTransformStream()); + + const completed = completedEvent(out); + expect(completed.response.usage).toMatchObject({ + input_tokens: 27_339, + output_tokens: 437, + input_tokens_details: { cached_tokens: 27_200 }, + }); + }); +}); diff --git a/tests/unit/openai-to-claude.test.js b/tests/unit/openai-to-claude.test.js index 45b67fb2..0c5768d7 100644 --- a/tests/unit/openai-to-claude.test.js +++ b/tests/unit/openai-to-claude.test.js @@ -203,4 +203,31 @@ describe("openaiToClaudeResponse", () => { limit: 120 }); }); + + it("records usage from a choices:[] frame so the finish chunk can emit it", () => { + const state = { toolCalls: new Map() }; + expect(openaiToClaudeResponse({ + usage: { + prompt_tokens: 90, + completion_tokens: 7, + prompt_tokens_details: { cached_tokens: 30 }, + }, + choices: [], + }, state)).toBeNull(); + expect(state.usage).toEqual({ + input_tokens: 60, + output_tokens: 7, + cache_read_input_tokens: 30, + }); + + const events = openaiToClaudeResponse({ + id: "chatcmpl-qoder-finish", + model: "qoder/auto", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + }, state); + const delta = events.find((e) => e.type === "message_delta"); + expect(delta.usage.input_tokens).toBe(60); + expect(delta.usage.output_tokens).toBe(7); + expect(delta.usage.cache_read_input_tokens).toBe(30); + }); }); diff --git a/tests/unit/qoder-context-tier.test.js b/tests/unit/qoder-context-tier.test.js new file mode 100644 index 00000000..8850fd0d --- /dev/null +++ b/tests/unit/qoder-context-tier.test.js @@ -0,0 +1,193 @@ +/** + * Qoder context-window tiers + routable model listing. + * + * The Qoder IDE lets a user pick 200K / 400K / 1M for a model; qodercli-style + * requests (what 9router sends) only carry the default max_input_tokens. These + * tests pin the escalation policy and the payload fields the IDE writes. + */ +import { describe, it, expect } from "vitest"; + +import { + parseTierTokenCount, + getQoderContextTiers, + estimateQoderPromptTokens, + resolveQoderContextTier, + applyQoderContextTier, +} from "../../open-sse/shared/qoder/contextTier.js"; +import { routableQoderModels } from "../../open-sse/services/qoderModels.js"; + +// Shape mirrors the live /algo/api/v2/model/list entry for qmodel_38max. +const MODEL_CONFIG = { + key: "qmodel_38max", + display_name: "Qwen3.8-Max", + is_reasoning: true, + max_input_tokens: 180_000, + max_output_tokens: 32_768, + context_config: [ + { name: "200K", tokenCount: 200_000, isDefault: true }, + { name: "400K", tokenCount: 400_000, isDefault: false }, + { name: "1M", tokenCount: 1_000_000, isDefault: false }, + ], +}; + +function promptOfTokens(n) { + // ~4 ASCII chars per token + return { system: "", messages: [{ role: "user", content: "abcd".repeat(n) }], tools: [] }; +} + +describe("parseTierTokenCount", () => { + it("accepts numbers and K/M suffixed strings", () => { + expect(parseTierTokenCount(204800)).toBe(204800); + expect(parseTierTokenCount("200K")).toBe(200_000); + expect(parseTierTokenCount("1M")).toBe(1_000_000); + expect(parseTierTokenCount("1.5m")).toBe(1_500_000); + expect(parseTierTokenCount("131072")).toBe(131072); + }); + + it("returns 0 for garbage", () => { + expect(parseTierTokenCount(null)).toBe(0); + expect(parseTierTokenCount("big")).toBe(0); + expect(parseTierTokenCount(-5)).toBe(0); + }); +}); + +describe("getQoderContextTiers", () => { + it("sorts tiers ascending and keeps the default flag", () => { + const tiers = getQoderContextTiers({ + context_config: [ + { name: "1M", tokenCount: 1_000_000 }, + { name: "200K", tokenCount: 200_000, isDefault: true }, + ], + }); + expect(tiers.map((t) => t.tokenCount)).toEqual([200_000, 1_000_000]); + expect(tiers[0].isDefault).toBe(true); + expect(tiers[1].isDefault).toBe(false); + }); + + it("understands camelCase / snake_case variants and derives names", () => { + const tiers = getQoderContextTiers({ + contextConfig: [{ token_count: "400K", is_default: true }, { max_input_tokens: 1_000_000 }], + }); + expect(tiers).toEqual([ + { name: "400K", tokenCount: 400_000, isDefault: true }, + { name: "1M", tokenCount: 1_000_000, isDefault: false }, + ]); + }); + + it("returns [] when the model has no tiers", () => { + expect(getQoderContextTiers({ max_input_tokens: 131072 })).toEqual([]); + expect(getQoderContextTiers(null)).toEqual([]); + }); +}); + +describe("estimateQoderPromptTokens", () => { + it("counts CJK characters as ~1 token each instead of chars/4", () => { + const ascii = estimateQoderPromptTokens({ messages: [{ role: "user", content: "a".repeat(4000) }] }); + const cjk = estimateQoderPromptTokens({ messages: [{ role: "user", content: "中".repeat(4000) }] }); + expect(ascii).toBeLessThan(1_200); + expect(cjk).toBeGreaterThan(4_000); + }); +}); + +describe("resolveQoderContextTier (auto)", () => { + it("leaves the payload untouched while the prompt fits the current max_input_tokens", () => { + expect(resolveQoderContextTier(MODEL_CONFIG, promptOfTokens(50_000))).toBeNull(); + }); + + it("escalates to the smallest tier that fits once the prompt outgrows the default", () => { + const choice = resolveQoderContextTier(MODEL_CONFIG, promptOfTokens(250_000)); + expect(choice).not.toBeNull(); + expect(choice.tier.name).toBe("400K"); + expect(choice.reason).toBe("auto:fits"); + expect(choice.estimatedTokens).toBeGreaterThan(240_000); + }); + + it("falls back to the largest tier when nothing fits (upstream decides)", () => { + const choice = resolveQoderContextTier(MODEL_CONFIG, promptOfTokens(1_200_000)); + expect(choice.tier.name).toBe("1M"); + expect(choice.reason).toBe("auto:largest"); + }); + + it("applies headroom so a prompt just under the limit still escalates", () => { + // 170K estimated * 1.15 = 195.5K > 180K current → smallest tier above the current limit (200K) + expect(resolveQoderContextTier(MODEL_CONFIG, promptOfTokens(170_000))?.tier.name).toBe("200K"); + // 190K * 1.15 = 218.5K → 200K no longer fits → 400K + expect(resolveQoderContextTier(MODEL_CONFIG, promptOfTokens(190_000))?.tier.name).toBe("400K"); + }); + + it("returns null for models without context_config", () => { + expect(resolveQoderContextTier({ max_input_tokens: 131072 }, promptOfTokens(500_000))).toBeNull(); + }); + + it("never escalates when the current limit is already the largest tier", () => { + const cfg = { ...MODEL_CONFIG, max_input_tokens: 1_000_000 }; + expect(resolveQoderContextTier(cfg, promptOfTokens(1_500_000))).toBeNull(); + }); +}); + +describe("resolveQoderContextTier (forced via QODER_CONTEXT_TIER)", () => { + it("max picks the largest tier regardless of prompt size", () => { + const choice = resolveQoderContextTier(MODEL_CONFIG, promptOfTokens(10), { preference: "max" }); + expect(choice.tier.name).toBe("1M"); + expect(choice.reason).toBe("forced:max"); + }); + + it("default picks the isDefault tier", () => { + const choice = resolveQoderContextTier(MODEL_CONFIG, promptOfTokens(10), { preference: "default" }); + expect(choice.tier.name).toBe("200K"); + }); + + it("a tier name or token count selects that tier", () => { + expect(resolveQoderContextTier(MODEL_CONFIG, promptOfTokens(10), { preference: "400k" }).tier.tokenCount).toBe(400_000); + expect(resolveQoderContextTier(MODEL_CONFIG, promptOfTokens(10), { preference: "1000000" }).tier.name).toBe("1M"); + }); + + it("an unknown tier name falls back to auto", () => { + expect(resolveQoderContextTier(MODEL_CONFIG, promptOfTokens(10), { preference: "9M" })).toBeNull(); + expect(resolveQoderContextTier(MODEL_CONFIG, promptOfTokens(250_000), { preference: "9M" }).tier.name).toBe("400K"); + }); +}); + +describe("applyQoderContextTier", () => { + it("mirrors the tier into the three places the IDE writes", () => { + const payload = { + parameters: { max_tokens: 32_768 }, + chat_context: { extra: { context: [], modelConfig: { key: "qmodel_38max" } } }, + model_config: { ...MODEL_CONFIG }, + }; + applyQoderContextTier(payload, { name: "1M", tokenCount: 1_000_000 }); + expect(payload.parameters).toEqual({ max_tokens: 32_768, context_length: 1_000_000 }); + expect(payload.chat_context.extra.ideModelConfigOverride).toEqual({ max_input_tokens: 1_000_000 }); + expect(payload.chat_context.extra.modelConfig).toEqual({ key: "qmodel_38max" }); + expect(payload.model_config.max_input_tokens).toBe(1_000_000); + expect(payload.model_config.context_config).toHaveLength(3); + }); + + it("is a no-op without a tier", () => { + const payload = { parameters: { max_tokens: 1 } }; + expect(applyQoderContextTier(payload, null)).toBe(payload); + expect(payload).toEqual({ parameters: { max_tokens: 1 } }); + }); +}); + +describe("routableQoderModels", () => { + it("lists visible models first, then hidden (enable:false) catalog keys", () => { + const catalog = { + models: [{ id: "qmodel_38max", name: "Qwen3.8-Max" }], + rawConfigs: new Map([ + ["qmodel_38max", { key: "qmodel_38max", enable: true }], + ["qfmodel", { key: "qfmodel", enable: false, display_name: "Qwen Fast" }], + ["dmodel", { key: "dmodel", enable: false }], + ]), + }; + expect(routableQoderModels(catalog)).toEqual([ + { id: "qmodel_38max", name: "Qwen3.8-Max", hidden: false }, + { id: "qfmodel", name: "Qwen Fast", hidden: true }, + { id: "dmodel", name: "dmodel", hidden: true }, + ]); + }); + + it("returns [] for a failed catalog fetch", () => { + expect(routableQoderModels(null)).toEqual([]); + }); +}); diff --git a/tests/unit/qoder.test.js b/tests/unit/qoder.test.js index 04ce1c45..ea45ac61 100644 --- a/tests/unit/qoder.test.js +++ b/tests/unit/qoder.test.js @@ -9,7 +9,7 @@ * - device flow URL construction */ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, beforeEach } from "vitest"; import crypto from "crypto"; import { qoderEncodeBody } from "../../src/lib/qoder/encoding.js"; @@ -22,6 +22,13 @@ import { } from "../../src/lib/qoder/constants.js"; import { PROVIDER_MODELS } from "../../open-sse/config/providerModels.js"; import { __test__ as qoderExecutorInternals } from "../../open-sse/executors/qoder.js"; +import { canonicalizeQoderUsage } from "../../open-sse/shared/qoder/sse.js"; +import { + rewriteQoderMessageAttachments, + clearQoderUploadCache, + buildMultipartFile, +} from "../../open-sse/shared/qoder/attachments.js"; +import { qoderInferenceBase } from "../../open-sse/shared/qoder/constants.js"; // Convenience aliases — tests were originally written against module-level // helpers; the QoderService class wraps them so each test creates its own @@ -431,6 +438,21 @@ describe("normalizeMessages", () => { ]); expect(result.messages[0].content).toBe("hi"); }); + + it("turns leftover file/document blocks into short stubs instead of dropping them", () => { + const result = normalizeMessages([ + { + role: "user", + content: [ + { type: "text", text: "see" }, + { type: "file", file: { filename: "big.pdf", file_data: "data:application/pdf;base64,AAA" } }, + ], + }, + ]); + expect(result.messages[0].content).toContain("see"); + expect(result.messages[0].content).toContain("big.pdf"); + expect(result.messages[0].content).not.toContain("AAA"); + }); }); describe("wrapQoderSSE", () => { @@ -530,4 +552,190 @@ describe("wrapQoderSSE", () => { const wrapped = await wrapQoderSSE(r, "qoder/auto"); expect(wrapped).toBe(r); }); + + function envelope(body) { + return `data: ${JSON.stringify({ statusCodeValue: 200, body })}\n\n`; + } + + function parseForwardedChunks(out) { + return out + .split("\n\n") + .map((block) => block.trim()) + .filter((block) => block.startsWith("data:") && !block.includes("[DONE]")) + .map((block) => JSON.parse(block.slice("data:".length).trim())); + } + + it("coalesces empty finish-in-delta + usage-only into one OpenAI usage chunk", async () => { + const content = JSON.stringify({ + id: "chatcmpl-qoder-1", + created: 1700000000, + model: "auto", + choices: [{ index: 0, delta: { content: "hi" } }], + }); + const finish = JSON.stringify({ + id: "chatcmpl-qoder-1", + choices: [{ index: 0, delta: { content: "", finish_reason: "stop" } }], + }); + const usage = JSON.stringify({ + id: "chatcmpl-qoder-1", + choices: [], + usage: { + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + prompt_tokens_details: { cached_tokens: 40 }, + }, + }); + const wrapped = await wrapQoderSSE( + makeResponse([envelope(content) + envelope(finish) + envelope(usage) + envelope("[DONE]")]), + "qoder/auto", + ); + const out = await drain(wrapped); + expect(out).toContain(`data: ${content}\n\n`); + const chunks = parseForwardedChunks(out); + const usageChunk = chunks.find((c) => c.usage); + expect(usageChunk).toBeDefined(); + expect(usageChunk.choices[0].finish_reason).toBe("stop"); + expect(usageChunk.usage.prompt_tokens).toBe(100); + expect(usageChunk.usage.completion_tokens).toBe(20); + expect(usageChunk.usage.prompt_tokens_details.cached_tokens).toBe(40); + expect(chunks.some((c) => Array.isArray(c.choices) && c.choices.length === 0)).toBe(false); + expect((out.match(/data: \[DONE\]/g) || []).length).toBe(1); + }); + + it("maps Qoder input_tokens aliases onto prompt_tokens in the coalesced usage chunk", async () => { + const finish = JSON.stringify({ + choices: [{ index: 0, delta: { finish_reason: "stop" } }], + }); + const usage = JSON.stringify({ + choices: [], + usage: { + input_tokens: 80, + output_tokens: 10, + cache_read_input_tokens: 25, + }, + }); + const wrapped = await wrapQoderSSE( + makeResponse([envelope(finish) + envelope(usage)]), + "qoder/lite", + ); + const chunks = parseForwardedChunks(await drain(wrapped)); + const usageChunk = chunks.find((c) => c.usage); + expect(usageChunk.usage.prompt_tokens).toBe(80); + expect(usageChunk.usage.completion_tokens).toBe(10); + expect(usageChunk.usage.prompt_tokens_details.cached_tokens).toBe(25); + }); +}); + +describe("canonicalizeQoderUsage", () => { + it("returns null for missing or empty usage", () => { + expect(canonicalizeQoderUsage(null)).toBeNull(); + expect(canonicalizeQoderUsage({})).toBeNull(); + }); + + it("copies prompt_tokens_details.cached_tokens through", () => { + const out = canonicalizeQoderUsage({ + prompt_tokens: 50, + completion_tokens: 5, + prompt_tokens_details: { cached_tokens: 12 }, + }); + expect(out.prompt_tokens).toBe(50); + expect(out.cached_tokens).toBe(12); + expect(out.prompt_tokens_details.cached_tokens).toBe(12); + expect(out.total_tokens).toBe(55); + }); +}); + +describe("qoderInferenceBase", () => { + it("sends job tokens to api2 and device tokens to api3", () => { + expect(qoderInferenceBase({ accessToken: "jt-abc" })).toContain("api2.qoder.sh"); + expect(qoderInferenceBase({ accessToken: "dt-abc" })).toContain("api3.qoder.sh"); + }); +}); + +describe("rewriteQoderMessageAttachments", () => { + beforeEach(() => clearQoderUploadCache()); + + it("uploads data-URI images and keeps only the OSS URL in the message", async () => { + const messages = [{ + role: "user", + content: [ + { type: "text", text: "see this" }, + { type: "image_url", image_url: { url: "data:image/png;base64,AAAA" } }, + ], + }]; + const stats = await rewriteQoderMessageAttachments(messages, { + uploadFn: async ({ buffer, mediaType }) => { + expect(Buffer.isBuffer(buffer)).toBe(true); + expect(mediaType).toBe("image/png"); + return "https://cdn.qoder.example/img.png"; + }, + }); + expect(messages[0].content).toEqual([ + { type: "text", text: "see this" }, + { type: "image_url", image_url: { url: "https://cdn.qoder.example/img.png" } }, + ]); + expect(JSON.stringify(messages)).not.toContain("AAAA"); + expect(stats.imageUrls).toEqual(["https://cdn.qoder.example/img.png"]); + }); + + it("does not re-upload already-hosted http(s) image URLs", async () => { + const messages = [{ + role: "user", + content: [{ type: "image_url", image_url: { url: "https://example.com/a.png" } }], + }]; + await rewriteQoderMessageAttachments(messages, { + uploadFn: async () => { + throw new Error("should not upload remote URLs"); + }, + }); + expect(messages[0].content[0].image_url.url).toBe("https://example.com/a.png"); + }); + + it("stubs non-image file blocks instead of inlining bytes", async () => { + const pdfB64 = "A".repeat(200); + const messages = [{ + role: "user", + content: [ + { type: "text", text: "read this" }, + { type: "file", file: { filename: "big.pdf", file_data: `data:application/pdf;base64,${pdfB64}` } }, + ], + }]; + await rewriteQoderMessageAttachments(messages, { + uploadFn: async () => { + throw new Error("should not upload PDFs as images"); + }, + }); + const wire = JSON.stringify(messages); + expect(wire).not.toContain(pdfB64); + expect(wire).toContain("[file omitted: big.pdf"); + }); + + it("stubs oversized images when OSS upload fails instead of keeping a huge data URI", async () => { + const big = "A".repeat(700_000); + const messages = [{ + role: "user", + content: [{ type: "image_url", image_url: { url: `data:image/png;base64,${big}` } }], + }]; + await rewriteQoderMessageAttachments(messages, { + uploadFn: async () => { + throw new Error("upstream 413"); + }, + }); + const wire = JSON.stringify(messages); + expect(wire).not.toContain(big); + expect(wire).toContain("[file omitted:"); + expect(Buffer.byteLength(wire, "utf8")).toBeLessThan(4096); + }); + + it("buildMultipartFile uses the file field name qodercli sends", () => { + const { boundary, body } = buildMultipartFile(Buffer.from("hi"), { + fileName: "image.png", + mediaType: "image/png", + }); + const text = body.toString("latin1"); + expect(text).toContain(`name="file"`); + expect(text).toContain("filename=\"image.png\""); + expect(text).toContain(`--${boundary}`); + }); });