feat(open-sse): early-peek stream error detection + fix UTF-8 loss in CommandCode peek
- new open-sse/utils/streamErrorPeek.js: bounded peek of the first bytes of a 200 stream; configured pattern match → 502 so account/combo fallback can run before any byte reaches the client (streaming included). Re-emits RAW bytes so split multi-byte UTF-8 sequences survive the peek (never re-encode decoded text — TextDecoder flush corrupts a lone leading byte to U+FFFD). - chatCore: run the peek after executor.execute when the provider has streamErrorPatterns configured; chat.js passes the settings through. - commandcode executor: same raw-bytes fix in peekForUpstreamError + regression test that fails against the old flush-based re-encode.
This commit is contained in:
@@ -27,7 +27,7 @@ export class CommandCodeExecutor extends BaseExecutor {
|
||||
super("commandcode", PROVIDERS.commandcode);
|
||||
}
|
||||
|
||||
transformRequest(model, body, stream, credentials) {
|
||||
transformRequest(_model, body, _stream, _credentials) {
|
||||
body.stream = true;
|
||||
return body;
|
||||
}
|
||||
@@ -131,13 +131,17 @@ export async function peekForUpstreamError(
|
||||
) {
|
||||
const reader = originalResponse.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
const encoder = new TextEncoder();
|
||||
const abortController = new AbortController();
|
||||
const forwardAbort = () => abortController.abort(signal?.reason);
|
||||
if (signal?.aborted) abortController.abort(signal?.reason);
|
||||
else if (signal)
|
||||
signal.addEventListener("abort", forwardAbort, { once: true });
|
||||
|
||||
// Raw bytes for lossless re-emission; decoded text is only used for line
|
||||
// parsing / error detection. Never re-encode decoded text: TextDecoder
|
||||
// holds a split multi-byte char internally and flush() would replace it
|
||||
// with U+FFFD, corrupting the stream.
|
||||
const rawChunks = [];
|
||||
let peeked = "";
|
||||
let errorEvent = null;
|
||||
let committed = false;
|
||||
@@ -167,6 +171,7 @@ export async function peekForUpstreamError(
|
||||
Math.max(deadline - Date.now(), 1),
|
||||
);
|
||||
if (done) break;
|
||||
rawChunks.push(value);
|
||||
peeked += decoder.decode(value, { stream: true });
|
||||
const lines = peeked.split("\n");
|
||||
// The last segment may be a partial line — only parse complete ones.
|
||||
@@ -188,10 +193,6 @@ export async function peekForUpstreamError(
|
||||
// the downstream stream pipeline (stall detection, abort handling) takes over.
|
||||
}
|
||||
|
||||
// Flush any partial multi-byte UTF-8 sequence held by the decoder so the
|
||||
// re-encoded peeked bytes round-trip losslessly.
|
||||
peeked += decoder.decode();
|
||||
|
||||
if (signal) signal.removeEventListener("abort", forwardAbort);
|
||||
|
||||
if (errorEvent) {
|
||||
@@ -210,7 +211,9 @@ export async function peekForUpstreamError(
|
||||
start(controller) {
|
||||
(async () => {
|
||||
try {
|
||||
if (peeked) controller.enqueue(encoder.encode(peeked));
|
||||
// Re-emit RAW bytes (never re-encoded decoded text) so split
|
||||
// multi-byte UTF-8 sequences survive the peek untouched.
|
||||
for (const c of rawChunks) controller.enqueue(c);
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
136
open-sse/utils/streamErrorPeek.js
Normal file
136
open-sse/utils/streamErrorPeek.js
Normal file
@@ -0,0 +1,136 @@
|
||||
import { matchStreamErrorPatterns } from "./streamErrorPatterns.js";
|
||||
import { HTTP_STATUS } from "../config/runtimeConfig.js";
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = (() => {
|
||||
const raw = process.env.STREAM_ERROR_PEEK_TIMEOUT_MS;
|
||||
const n = raw ? parseInt(raw, 10) : NaN;
|
||||
return Number.isFinite(n) && n > 0 ? n : 3000;
|
||||
})();
|
||||
|
||||
const DEFAULT_MAX_BYTES = (() => {
|
||||
const raw = process.env.STREAM_ERROR_PEEK_MAX_BYTES;
|
||||
const n = raw ? parseInt(raw, 10) : NaN;
|
||||
return Number.isFinite(n) && n > 0 ? n : 8192;
|
||||
})();
|
||||
|
||||
function makeAbortError(reason) {
|
||||
const error = new Error(reason?.message || reason || "Request aborted");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the first bytes of a 200 response and reject it with a 502 when a
|
||||
* configured stream-error pattern matches — BEFORE any byte reaches the
|
||||
* client, so account/model fallback can still kick in for streaming too.
|
||||
* On no-match/timeout/abort the response is re-emitted (buffered bytes +
|
||||
* rest of stream) unchanged in spirit. Fail-open: never throws.
|
||||
*/
|
||||
export async function maybeRejectEarlyStreamError(
|
||||
response,
|
||||
patterns,
|
||||
{
|
||||
signal = null,
|
||||
timeoutMs = DEFAULT_TIMEOUT_MS,
|
||||
maxBytes = DEFAULT_MAX_BYTES,
|
||||
} = {},
|
||||
) {
|
||||
if (!Array.isArray(patterns) || patterns.length === 0) return response;
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
const abortController = new AbortController();
|
||||
const forwardAbort = () => abortController.abort(signal?.reason);
|
||||
if (signal?.aborted) abortController.abort(signal?.reason);
|
||||
else if (signal)
|
||||
signal.addEventListener("abort", forwardAbort, { once: true });
|
||||
|
||||
// Raw bytes for lossless re-emission; decoded text ONLY for pattern matching.
|
||||
// Never re-encode decoded text: TextDecoder holds a split multi-byte char
|
||||
// internally and flush() would replace it with U+FFFD, corrupting the stream.
|
||||
const rawChunks = [];
|
||||
let peekedText = "";
|
||||
let total = 0;
|
||||
|
||||
const readWithTimeout = (ms) => {
|
||||
if (abortController.signal.aborted)
|
||||
return Promise.reject(makeAbortError(abortController.signal.reason));
|
||||
const timeoutPromise = new Promise((_, reject) => {
|
||||
const t = setTimeout(
|
||||
() => reject(new Error("stream error peek timeout")),
|
||||
ms,
|
||||
);
|
||||
t.unref?.();
|
||||
});
|
||||
const abortPromise = new Promise((_, reject) => {
|
||||
abortController.signal.addEventListener(
|
||||
"abort",
|
||||
() => reject(makeAbortError(abortController.signal.reason)),
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
return Promise.race([reader.read(), timeoutPromise, abortPromise]);
|
||||
};
|
||||
|
||||
try {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (total < maxBytes && Date.now() < deadline) {
|
||||
const { done, value } = await readWithTimeout(
|
||||
Math.max(deadline - Date.now(), 1),
|
||||
);
|
||||
if (done) break;
|
||||
rawChunks.push(value);
|
||||
peekedText += decoder.decode(value, { stream: true });
|
||||
total += value.byteLength;
|
||||
const matched = matchStreamErrorPatterns(patterns, peekedText);
|
||||
if (matched) {
|
||||
await reader.cancel("stream error pattern matched").catch(() => {});
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
error: {
|
||||
message: `Stream error pattern matched: ${matched}`,
|
||||
type: "upstream_error",
|
||||
},
|
||||
}),
|
||||
{
|
||||
status: HTTP_STATUS.BAD_GATEWAY,
|
||||
statusText: String(matched).slice(0, 200),
|
||||
headers: { "Content-Type": "application/json" },
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// timeout / abort / read failure → commit; downstream stall/abort handling takes over.
|
||||
}
|
||||
|
||||
if (signal) signal.removeEventListener("abort", forwardAbort);
|
||||
|
||||
const remaining = new ReadableStream({
|
||||
start(controller) {
|
||||
(async () => {
|
||||
try {
|
||||
// Re-emit RAW bytes (never re-encoded decoded text) so split
|
||||
// multi-byte UTF-8 sequences survive the peek untouched.
|
||||
for (const c of rawChunks) controller.enqueue(c);
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
controller.enqueue(value);
|
||||
}
|
||||
controller.close();
|
||||
} catch (err) {
|
||||
controller.error(err);
|
||||
}
|
||||
})();
|
||||
},
|
||||
cancel() {
|
||||
reader.cancel("stream cancelled during peek commit").catch(() => {});
|
||||
},
|
||||
});
|
||||
|
||||
return new Response(remaining, {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
headers: response.headers,
|
||||
});
|
||||
}
|
||||
@@ -1,11 +1,11 @@
|
||||
import "open-sse/index.js";
|
||||
|
||||
import {
|
||||
getProviderCredentials,
|
||||
markAccountUnavailable,
|
||||
clearAccountError,
|
||||
extractApiKey,
|
||||
isValidApiKey,
|
||||
getProviderCredentials,
|
||||
markAccountUnavailable,
|
||||
clearAccountError,
|
||||
extractApiKey,
|
||||
isValidApiKey,
|
||||
} from "../services/auth.js";
|
||||
import { cacheClaudeHeaders } from "open-sse/utils/claudeHeaderCache.js";
|
||||
import { getSettings } from "@/lib/localDb";
|
||||
@@ -20,7 +20,10 @@ import { handleBypassRequest } from "open-sse/utils/bypassHandler.js";
|
||||
import { HTTP_STATUS } from "open-sse/config/runtimeConfig.js";
|
||||
import { detectFormatByEndpoint } from "open-sse/translator/formats.js";
|
||||
import * as log from "../utils/logger.js";
|
||||
import { updateProviderCredentials, checkAndRefreshToken } from "../services/tokenRefresh.js";
|
||||
import {
|
||||
updateProviderCredentials,
|
||||
checkAndRefreshToken,
|
||||
} from "../services/tokenRefresh.js";
|
||||
import { getProjectIdForConnection } from "open-sse/services/projectId.js";
|
||||
|
||||
/**
|
||||
@@ -29,266 +32,355 @@ import { getProjectIdForConnection } from "open-sse/services/projectId.js";
|
||||
* Format detection and translation handled by translator
|
||||
*/
|
||||
export async function handleChat(request, clientRawRequest = null) {
|
||||
let body;
|
||||
try {
|
||||
body = await request.json();
|
||||
} catch {
|
||||
log.warn("CHAT", "Invalid JSON body");
|
||||
return errorResponse(HTTP_STATUS.BAD_REQUEST, "Invalid JSON body");
|
||||
}
|
||||
let body;
|
||||
try {
|
||||
body = await request.json();
|
||||
} catch {
|
||||
log.warn("CHAT", "Invalid JSON body");
|
||||
return errorResponse(HTTP_STATUS.BAD_REQUEST, "Invalid JSON body");
|
||||
}
|
||||
|
||||
// Build clientRawRequest for logging (if not provided)
|
||||
if (!clientRawRequest) {
|
||||
const url = new URL(request.url);
|
||||
clientRawRequest = {
|
||||
endpoint: url.pathname,
|
||||
body,
|
||||
headers: Object.fromEntries(request.headers.entries())
|
||||
};
|
||||
}
|
||||
cacheClaudeHeaders(clientRawRequest.headers);
|
||||
// Build clientRawRequest for logging (if not provided)
|
||||
if (!clientRawRequest) {
|
||||
const url = new URL(request.url);
|
||||
clientRawRequest = {
|
||||
endpoint: url.pathname,
|
||||
body,
|
||||
headers: Object.fromEntries(request.headers.entries()),
|
||||
};
|
||||
}
|
||||
cacheClaudeHeaders(clientRawRequest.headers);
|
||||
|
||||
const modelStr = body.model;
|
||||
const modelStr = body.model;
|
||||
|
||||
// Request summary is emitted as the unified "▶" line in chatCore (has fmt/thinking/account)
|
||||
// Request summary is emitted as the unified "▶" line in chatCore (has fmt/thinking/account)
|
||||
|
||||
// Log API key (masked)
|
||||
const authHeader = request.headers.get("Authorization");
|
||||
const apiKey = extractApiKey(request);
|
||||
if (authHeader && apiKey) {
|
||||
const masked = log.maskKey(apiKey);
|
||||
log.debug("AUTH", `API Key: ${masked}`);
|
||||
} else {
|
||||
log.debug("AUTH", "No API key provided (local mode)");
|
||||
}
|
||||
// Log API key (masked)
|
||||
const authHeader = request.headers.get("Authorization");
|
||||
const apiKey = extractApiKey(request);
|
||||
if (authHeader && apiKey) {
|
||||
const masked = log.maskKey(apiKey);
|
||||
log.debug("AUTH", `API Key: ${masked}`);
|
||||
} else {
|
||||
log.debug("AUTH", "No API key provided (local mode)");
|
||||
}
|
||||
|
||||
// Enforce API key if enabled in settings
|
||||
const settings = await getSettings();
|
||||
if (settings.requireApiKey) {
|
||||
if (!apiKey) {
|
||||
log.warn("AUTH", "Missing API key (requireApiKey=true)");
|
||||
return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Missing API key");
|
||||
}
|
||||
const valid = await isValidApiKey(apiKey);
|
||||
if (!valid) {
|
||||
log.warn("AUTH", "Invalid API key (requireApiKey=true)");
|
||||
return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Invalid API key");
|
||||
}
|
||||
}
|
||||
// Enforce API key if enabled in settings
|
||||
const settings = await getSettings();
|
||||
if (settings.requireApiKey) {
|
||||
if (!apiKey) {
|
||||
log.warn("AUTH", "Missing API key (requireApiKey=true)");
|
||||
return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Missing API key");
|
||||
}
|
||||
const valid = await isValidApiKey(apiKey);
|
||||
if (!valid) {
|
||||
log.warn("AUTH", "Invalid API key (requireApiKey=true)");
|
||||
return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Invalid API key");
|
||||
}
|
||||
}
|
||||
|
||||
if (!modelStr) {
|
||||
log.warn("CHAT", "Missing model");
|
||||
return errorResponse(HTTP_STATUS.BAD_REQUEST, "Missing model");
|
||||
}
|
||||
if (!modelStr) {
|
||||
log.warn("CHAT", "Missing model");
|
||||
return errorResponse(HTTP_STATUS.BAD_REQUEST, "Missing model");
|
||||
}
|
||||
|
||||
// Bypass naming/warmup requests before combo rotation to avoid wasting rotation slots
|
||||
const userAgent = request?.headers?.get("user-agent") || "";
|
||||
const bypassResponse = handleBypassRequest(body, modelStr, userAgent, !!settings.ccFilterNaming);
|
||||
if (bypassResponse) return bypassResponse.response || bypassResponse;
|
||||
// Bypass naming/warmup requests before combo rotation to avoid wasting rotation slots
|
||||
const userAgent = request?.headers?.get("user-agent") || "";
|
||||
const bypassResponse = handleBypassRequest(
|
||||
body,
|
||||
modelStr,
|
||||
userAgent,
|
||||
!!settings.ccFilterNaming,
|
||||
);
|
||||
if (bypassResponse) return bypassResponse.response || bypassResponse;
|
||||
|
||||
// Check if model is a combo (has multiple models with fallback)
|
||||
const comboModels = await getComboModels(modelStr);
|
||||
if (comboModels) {
|
||||
// Check for combo-specific strategy first, fallback to global
|
||||
const comboStrategies = settings.comboStrategies || {};
|
||||
const comboSpecificStrategy = comboStrategies[modelStr]?.fallbackStrategy;
|
||||
const comboStrategy = comboSpecificStrategy || settings.comboStrategy || "fallback";
|
||||
// Check if model is a combo (has multiple models with fallback)
|
||||
const comboModels = await getComboModels(modelStr);
|
||||
if (comboModels) {
|
||||
// Check for combo-specific strategy first, fallback to global
|
||||
const comboStrategies = settings.comboStrategies || {};
|
||||
const comboSpecificStrategy = comboStrategies[modelStr]?.fallbackStrategy;
|
||||
const comboStrategy =
|
||||
comboSpecificStrategy || settings.comboStrategy || "fallback";
|
||||
|
||||
if (comboStrategy === "fusion") {
|
||||
log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models (strategy: fusion)`);
|
||||
return handleFusionChat({
|
||||
body,
|
||||
models: comboModels,
|
||||
handleSingleModel: (b, m, isPanel) => {
|
||||
let cleanRawReq = clientRawRequest;
|
||||
if (isPanel && clientRawRequest) {
|
||||
const { tools, tool_choice, ...cleanBody } = clientRawRequest.body || {};
|
||||
cleanRawReq = { ...clientRawRequest, body: cleanBody };
|
||||
}
|
||||
return handleSingleModelChat(b, m, cleanRawReq, request, apiKey);
|
||||
},
|
||||
log,
|
||||
comboName: modelStr,
|
||||
judgeModel: comboStrategies[modelStr]?.judgeModel,
|
||||
tuning: comboStrategies[modelStr]?.fusionTuning,
|
||||
});
|
||||
}
|
||||
if (comboStrategy === "fusion") {
|
||||
log.info(
|
||||
"CHAT",
|
||||
`Combo "${modelStr}" with ${comboModels.length} models (strategy: fusion)`,
|
||||
);
|
||||
return handleFusionChat({
|
||||
body,
|
||||
models: comboModels,
|
||||
handleSingleModel: (b, m, isPanel) => {
|
||||
let cleanRawReq = clientRawRequest;
|
||||
if (isPanel && clientRawRequest) {
|
||||
const { tools, tool_choice, ...cleanBody } =
|
||||
clientRawRequest.body || {};
|
||||
cleanRawReq = { ...clientRawRequest, body: cleanBody };
|
||||
}
|
||||
return handleSingleModelChat(b, m, cleanRawReq, request, apiKey);
|
||||
},
|
||||
log,
|
||||
comboName: modelStr,
|
||||
judgeModel: comboStrategies[modelStr]?.judgeModel,
|
||||
tuning: comboStrategies[modelStr]?.fusionTuning,
|
||||
});
|
||||
}
|
||||
|
||||
const comboStickyLimit = settings.comboStickyRoundRobinLimit;
|
||||
log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models (strategy: ${comboStrategy}, sticky: ${comboStickyLimit})`);
|
||||
return handleComboChat({
|
||||
body,
|
||||
models: comboModels,
|
||||
handleSingleModel: (b, m) => handleSingleModelChat(b, m, clientRawRequest, request, apiKey),
|
||||
log,
|
||||
comboName: modelStr,
|
||||
comboStrategy,
|
||||
comboStickyLimit
|
||||
});
|
||||
}
|
||||
const comboStickyLimit = settings.comboStickyRoundRobinLimit;
|
||||
log.info(
|
||||
"CHAT",
|
||||
`Combo "${modelStr}" with ${comboModels.length} models (strategy: ${comboStrategy}, sticky: ${comboStickyLimit})`,
|
||||
);
|
||||
return handleComboChat({
|
||||
body,
|
||||
models: comboModels,
|
||||
handleSingleModel: (b, m) =>
|
||||
handleSingleModelChat(b, m, clientRawRequest, request, apiKey),
|
||||
log,
|
||||
comboName: modelStr,
|
||||
comboStrategy,
|
||||
comboStickyLimit,
|
||||
});
|
||||
}
|
||||
|
||||
// Single model request
|
||||
return handleSingleModelChat(body, modelStr, clientRawRequest, request, apiKey);
|
||||
// Single model request
|
||||
return handleSingleModelChat(
|
||||
body,
|
||||
modelStr,
|
||||
clientRawRequest,
|
||||
request,
|
||||
apiKey,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle single model chat request
|
||||
*/
|
||||
async function handleSingleModelChat(body, modelStr, clientRawRequest = null, request = null, apiKey = null) {
|
||||
const modelInfo = await getModelInfo(modelStr);
|
||||
async function handleSingleModelChat(
|
||||
body,
|
||||
modelStr,
|
||||
clientRawRequest = null,
|
||||
request = null,
|
||||
apiKey = null,
|
||||
) {
|
||||
const modelInfo = await getModelInfo(modelStr);
|
||||
|
||||
// If provider is null, this might be a combo name - check and handle
|
||||
if (!modelInfo.provider) {
|
||||
const comboModels = await getComboModels(modelStr);
|
||||
if (comboModels) {
|
||||
const chatSettings = await getSettings();
|
||||
// Check for combo-specific strategy first, fallback to global
|
||||
const comboStrategies = chatSettings.comboStrategies || {};
|
||||
const comboSpecificStrategy = comboStrategies[modelStr]?.fallbackStrategy;
|
||||
const comboStrategy = comboSpecificStrategy || chatSettings.comboStrategy || "fallback";
|
||||
// If provider is null, this might be a combo name - check and handle
|
||||
if (!modelInfo.provider) {
|
||||
const comboModels = await getComboModels(modelStr);
|
||||
if (comboModels) {
|
||||
const chatSettings = await getSettings();
|
||||
// Check for combo-specific strategy first, fallback to global
|
||||
const comboStrategies = chatSettings.comboStrategies || {};
|
||||
const comboSpecificStrategy = comboStrategies[modelStr]?.fallbackStrategy;
|
||||
const comboStrategy =
|
||||
comboSpecificStrategy || chatSettings.comboStrategy || "fallback";
|
||||
|
||||
if (comboStrategy === "fusion") {
|
||||
log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models (strategy: fusion)`);
|
||||
return handleFusionChat({
|
||||
body,
|
||||
models: comboModels,
|
||||
handleSingleModel: (b, m, isPanel) => {
|
||||
let cleanRawReq = clientRawRequest;
|
||||
if (isPanel && clientRawRequest) {
|
||||
const { tools, tool_choice, ...cleanBody } = clientRawRequest.body || {};
|
||||
cleanRawReq = { ...clientRawRequest, body: cleanBody };
|
||||
}
|
||||
return handleSingleModelChat(b, m, cleanRawReq, request, apiKey);
|
||||
},
|
||||
log,
|
||||
comboName: modelStr,
|
||||
judgeModel: comboStrategies[modelStr]?.judgeModel,
|
||||
tuning: comboStrategies[modelStr]?.fusionTuning,
|
||||
});
|
||||
}
|
||||
if (comboStrategy === "fusion") {
|
||||
log.info(
|
||||
"CHAT",
|
||||
`Combo "${modelStr}" with ${comboModels.length} models (strategy: fusion)`,
|
||||
);
|
||||
return handleFusionChat({
|
||||
body,
|
||||
models: comboModels,
|
||||
handleSingleModel: (b, m, isPanel) => {
|
||||
let cleanRawReq = clientRawRequest;
|
||||
if (isPanel && clientRawRequest) {
|
||||
const { tools, tool_choice, ...cleanBody } =
|
||||
clientRawRequest.body || {};
|
||||
cleanRawReq = { ...clientRawRequest, body: cleanBody };
|
||||
}
|
||||
return handleSingleModelChat(b, m, cleanRawReq, request, apiKey);
|
||||
},
|
||||
log,
|
||||
comboName: modelStr,
|
||||
judgeModel: comboStrategies[modelStr]?.judgeModel,
|
||||
tuning: comboStrategies[modelStr]?.fusionTuning,
|
||||
});
|
||||
}
|
||||
|
||||
const comboStickyLimit = chatSettings.comboStickyRoundRobinLimit;
|
||||
log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models (strategy: ${comboStrategy}, sticky: ${comboStickyLimit})`);
|
||||
return handleComboChat({
|
||||
body,
|
||||
models: comboModels,
|
||||
handleSingleModel: (b, m) => handleSingleModelChat(b, m, clientRawRequest, request, apiKey),
|
||||
log,
|
||||
comboName: modelStr,
|
||||
comboStrategy,
|
||||
comboStickyLimit
|
||||
});
|
||||
}
|
||||
log.warn("CHAT", "Invalid model format", { model: modelStr });
|
||||
return errorResponse(HTTP_STATUS.BAD_REQUEST, "Invalid model format");
|
||||
}
|
||||
const comboStickyLimit = chatSettings.comboStickyRoundRobinLimit;
|
||||
log.info(
|
||||
"CHAT",
|
||||
`Combo "${modelStr}" with ${comboModels.length} models (strategy: ${comboStrategy}, sticky: ${comboStickyLimit})`,
|
||||
);
|
||||
return handleComboChat({
|
||||
body,
|
||||
models: comboModels,
|
||||
handleSingleModel: (b, m) =>
|
||||
handleSingleModelChat(b, m, clientRawRequest, request, apiKey),
|
||||
log,
|
||||
comboName: modelStr,
|
||||
comboStrategy,
|
||||
comboStickyLimit,
|
||||
});
|
||||
}
|
||||
log.warn("CHAT", "Invalid model format", { model: modelStr });
|
||||
return errorResponse(HTTP_STATUS.BAD_REQUEST, "Invalid model format");
|
||||
}
|
||||
|
||||
const { provider, model } = modelInfo;
|
||||
const { provider, model } = modelInfo;
|
||||
|
||||
// Routing shown in the unified "▶" line (client model → provider/model)
|
||||
// Routing shown in the unified "▶" line (client model → provider/model)
|
||||
|
||||
// Extract userAgent from request
|
||||
const userAgent = request?.headers?.get("user-agent") || "";
|
||||
// Optional pin to a specific connection (dashboard test / client override)
|
||||
const preferredConnectionId = request?.headers?.get("x-connection-id") || null;
|
||||
// Extract userAgent from request
|
||||
const userAgent = request?.headers?.get("user-agent") || "";
|
||||
// Optional pin to a specific connection (dashboard test / client override)
|
||||
const preferredConnectionId =
|
||||
request?.headers?.get("x-connection-id") || null;
|
||||
|
||||
// Try with available accounts (fallback on errors unless pinned)
|
||||
const excludeConnectionIds = new Set();
|
||||
let lastError = null;
|
||||
let lastStatus = null;
|
||||
// Try with available accounts (fallback on errors unless pinned)
|
||||
const excludeConnectionIds = new Set();
|
||||
let lastError = null;
|
||||
let lastStatus = null;
|
||||
|
||||
while (true) {
|
||||
const credentials = await getProviderCredentials(provider, excludeConnectionIds, model, { preferredConnectionId });
|
||||
while (true) {
|
||||
const credentials = await getProviderCredentials(
|
||||
provider,
|
||||
excludeConnectionIds,
|
||||
model,
|
||||
{ preferredConnectionId },
|
||||
);
|
||||
|
||||
// All accounts unavailable
|
||||
if (!credentials || credentials.allRateLimited) {
|
||||
if (credentials?.allRateLimited) {
|
||||
const errorMsg = lastError || credentials.lastError || "Unavailable";
|
||||
const status = lastStatus || Number(credentials.lastErrorCode) || HTTP_STATUS.SERVICE_UNAVAILABLE;
|
||||
log.warn("CHAT", `[${provider}/${model}] ${errorMsg} (${credentials.retryAfterHuman})`);
|
||||
return unavailableResponse(status, `[${provider}/${model}] ${errorMsg}`, credentials.retryAfter, credentials.retryAfterHuman);
|
||||
}
|
||||
if (excludeConnectionIds.size === 0) {
|
||||
log.warn("AUTH", `No active credentials for provider: ${provider}`);
|
||||
return errorResponse(HTTP_STATUS.NOT_FOUND, `No active credentials for provider: ${provider}`);
|
||||
}
|
||||
log.warn("CHAT", "No more accounts available", { provider });
|
||||
return errorResponse(lastStatus || HTTP_STATUS.SERVICE_UNAVAILABLE, lastError || "All accounts unavailable");
|
||||
}
|
||||
// All accounts unavailable
|
||||
if (!credentials || credentials.allRateLimited) {
|
||||
if (credentials?.allRateLimited) {
|
||||
const errorMsg = lastError || credentials.lastError || "Unavailable";
|
||||
const status =
|
||||
lastStatus ||
|
||||
Number(credentials.lastErrorCode) ||
|
||||
HTTP_STATUS.SERVICE_UNAVAILABLE;
|
||||
log.warn(
|
||||
"CHAT",
|
||||
`[${provider}/${model}] ${errorMsg} (${credentials.retryAfterHuman})`,
|
||||
);
|
||||
return unavailableResponse(
|
||||
status,
|
||||
`[${provider}/${model}] ${errorMsg}`,
|
||||
credentials.retryAfter,
|
||||
credentials.retryAfterHuman,
|
||||
);
|
||||
}
|
||||
if (excludeConnectionIds.size === 0) {
|
||||
log.warn("AUTH", `No active credentials for provider: ${provider}`);
|
||||
return errorResponse(
|
||||
HTTP_STATUS.NOT_FOUND,
|
||||
`No active credentials for provider: ${provider}`,
|
||||
);
|
||||
}
|
||||
log.warn("CHAT", "No more accounts available", { provider });
|
||||
return errorResponse(
|
||||
lastStatus || HTTP_STATUS.SERVICE_UNAVAILABLE,
|
||||
lastError || "All accounts unavailable",
|
||||
);
|
||||
}
|
||||
|
||||
// Account selection shown in the unified "▶" line (acc:...)
|
||||
const refreshedCredentials = await checkAndRefreshToken(provider, credentials);
|
||||
// Account selection shown in the unified "▶" line (acc:...)
|
||||
const refreshedCredentials = await checkAndRefreshToken(
|
||||
provider,
|
||||
credentials,
|
||||
);
|
||||
|
||||
// Ensure real project ID is available for providers that need it (P0 fix: cold miss)
|
||||
if ((provider === "antigravity" || provider === "gemini-cli") && !refreshedCredentials.projectId) {
|
||||
const pid = await getProjectIdForConnection(credentials.connectionId, refreshedCredentials.accessToken, provider);
|
||||
if (pid) {
|
||||
refreshedCredentials.projectId = pid;
|
||||
// Persist to DB in background so subsequent requests have it immediately
|
||||
updateProviderCredentials(credentials.connectionId, { projectId: pid }).catch(() => { });
|
||||
}
|
||||
}
|
||||
// Ensure real project ID is available for providers that need it (P0 fix: cold miss)
|
||||
if (
|
||||
(provider === "antigravity" || provider === "gemini-cli") &&
|
||||
!refreshedCredentials.projectId
|
||||
) {
|
||||
const pid = await getProjectIdForConnection(
|
||||
credentials.connectionId,
|
||||
refreshedCredentials.accessToken,
|
||||
provider,
|
||||
);
|
||||
if (pid) {
|
||||
refreshedCredentials.projectId = pid;
|
||||
// Persist to DB in background so subsequent requests have it immediately
|
||||
updateProviderCredentials(credentials.connectionId, {
|
||||
projectId: pid,
|
||||
}).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
// Use shared chatCore
|
||||
const chatSettings = await getSettings();
|
||||
const providerThinking = (chatSettings.providerThinking || {})[provider] || null;
|
||||
const result = await handleChatCore({
|
||||
body: { ...body, model: `${provider}/${model}` },
|
||||
modelInfo: { provider, model },
|
||||
credentials: refreshedCredentials,
|
||||
log,
|
||||
clientRawRequest,
|
||||
connectionId: credentials.connectionId,
|
||||
userAgent,
|
||||
apiKey,
|
||||
ccFilterNaming: !!chatSettings.ccFilterNaming,
|
||||
rtkEnabled: !!chatSettings.rtkEnabled,
|
||||
headroomEnabled: !!chatSettings.headroomEnabled,
|
||||
headroomUrl: chatSettings.headroomUrl || DEFAULT_HEADROOM_URL,
|
||||
headroomCompressUserMessages: !!chatSettings.headroomCompressUserMessages,
|
||||
cavemanEnabled: !!chatSettings.cavemanEnabled,
|
||||
cavemanLevel: chatSettings.cavemanLevel || "full",
|
||||
ponytailEnabled: !!chatSettings.ponytailEnabled,
|
||||
ponytailLevel: chatSettings.ponytailLevel || "full",
|
||||
pxpipeEnabled: !!chatSettings.pxpipeEnabled,
|
||||
pxpipeMinChars: chatSettings.pxpipeMinChars,
|
||||
pxpipeTimeoutMs: chatSettings.pxpipeTimeoutMs,
|
||||
// Lazily warms the in-process module on first use; null when not installed (fail-open)
|
||||
pxpipeTransform: chatSettings.pxpipeEnabled ? await getPxpipeTransform() : null,
|
||||
onPxpipeEvent: appendPxpipeEvent,
|
||||
providerThinking,
|
||||
// Detect source format by endpoint + body
|
||||
sourceFormatOverride: request?.url ? detectFormatByEndpoint(new URL(request.url).pathname, body) : null,
|
||||
onCredentialsRefreshed: async (newCreds) => {
|
||||
await updateProviderCredentials(credentials.connectionId, {
|
||||
...newCreds,
|
||||
existingProviderSpecificData: credentials.providerSpecificData,
|
||||
testStatus: "active"
|
||||
});
|
||||
},
|
||||
onRequestSuccess: async () => {
|
||||
await clearAccountError(credentials.connectionId, credentials, model);
|
||||
}
|
||||
});
|
||||
// Use shared chatCore
|
||||
const chatSettings = await getSettings();
|
||||
const providerThinking =
|
||||
(chatSettings.providerThinking || {})[provider] || null;
|
||||
const result = await handleChatCore({
|
||||
body: { ...body, model: `${provider}/${model}` },
|
||||
modelInfo: { provider, model },
|
||||
credentials: refreshedCredentials,
|
||||
log,
|
||||
clientRawRequest,
|
||||
connectionId: credentials.connectionId,
|
||||
userAgent,
|
||||
apiKey,
|
||||
ccFilterNaming: !!chatSettings.ccFilterNaming,
|
||||
rtkEnabled: !!chatSettings.rtkEnabled,
|
||||
headroomEnabled: !!chatSettings.headroomEnabled,
|
||||
headroomUrl: chatSettings.headroomUrl || DEFAULT_HEADROOM_URL,
|
||||
headroomCompressUserMessages: !!chatSettings.headroomCompressUserMessages,
|
||||
cavemanEnabled: !!chatSettings.cavemanEnabled,
|
||||
cavemanLevel: chatSettings.cavemanLevel || "full",
|
||||
ponytailEnabled: !!chatSettings.ponytailEnabled,
|
||||
ponytailLevel: chatSettings.ponytailLevel || "full",
|
||||
pxpipeEnabled: !!chatSettings.pxpipeEnabled,
|
||||
pxpipeMinChars: chatSettings.pxpipeMinChars,
|
||||
pxpipeTimeoutMs: chatSettings.pxpipeTimeoutMs,
|
||||
// Lazily warms the in-process module on first use; null when not installed (fail-open)
|
||||
pxpipeTransform: chatSettings.pxpipeEnabled
|
||||
? await getPxpipeTransform()
|
||||
: null,
|
||||
onPxpipeEvent: appendPxpipeEvent,
|
||||
providerThinking,
|
||||
streamErrorPatterns: chatSettings.streamErrorPatterns || {},
|
||||
// Detect source format by endpoint + body
|
||||
sourceFormatOverride: request?.url
|
||||
? detectFormatByEndpoint(new URL(request.url).pathname, body)
|
||||
: null,
|
||||
onCredentialsRefreshed: async (newCreds) => {
|
||||
await updateProviderCredentials(credentials.connectionId, {
|
||||
...newCreds,
|
||||
existingProviderSpecificData: credentials.providerSpecificData,
|
||||
testStatus: "active",
|
||||
});
|
||||
},
|
||||
onRequestSuccess: async () => {
|
||||
await clearAccountError(credentials.connectionId, credentials, model);
|
||||
},
|
||||
});
|
||||
|
||||
if (result.success) return result.response;
|
||||
if (result.success) return result.response;
|
||||
|
||||
// Mark account unavailable (auto-calculates cooldown with exponential backoff, or precise resetsAtMs)
|
||||
const { shouldFallback } = await markAccountUnavailable(credentials.connectionId, result.status, result.error, provider, model, result.resetsAtMs);
|
||||
// Mark account unavailable (auto-calculates cooldown with exponential backoff, or precise resetsAtMs)
|
||||
const { shouldFallback } = await markAccountUnavailable(
|
||||
credentials.connectionId,
|
||||
result.status,
|
||||
result.error,
|
||||
provider,
|
||||
model,
|
||||
result.resetsAtMs,
|
||||
);
|
||||
|
||||
if (shouldFallback) {
|
||||
// When a connection is explicitly pinned, never rotate to another account.
|
||||
if (preferredConnectionId) {
|
||||
log.warn("AUTH", `Pinned account ${credentials.connectionName} unavailable (${result.status}), no fallback`);
|
||||
return result.response;
|
||||
}
|
||||
log.warn("FALLBACK", `⇄ ACC:${credentials.connectionName} UNAVAILABLE (${result.status}) → NEXT ACCOUNT`);
|
||||
excludeConnectionIds.add(credentials.connectionId);
|
||||
lastError = result.error;
|
||||
lastStatus = result.status;
|
||||
continue;
|
||||
}
|
||||
if (shouldFallback) {
|
||||
// When a connection is explicitly pinned, never rotate to another account.
|
||||
if (preferredConnectionId) {
|
||||
log.warn(
|
||||
"AUTH",
|
||||
`Pinned account ${credentials.connectionName} unavailable (${result.status}), no fallback`,
|
||||
);
|
||||
return result.response;
|
||||
}
|
||||
log.warn(
|
||||
"FALLBACK",
|
||||
`⇄ ACC:${credentials.connectionName} UNAVAILABLE (${result.status}) → NEXT ACCOUNT`,
|
||||
);
|
||||
excludeConnectionIds.add(credentials.connectionId);
|
||||
lastError = result.error;
|
||||
lastStatus = result.status;
|
||||
continue;
|
||||
}
|
||||
|
||||
return result.response;
|
||||
}
|
||||
return result.response;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,4 +98,26 @@ describe("commandcode executor — early-error peek", () => {
|
||||
expect(res.status).toBe(200);
|
||||
await res.body.cancel();
|
||||
});
|
||||
|
||||
it("re-emits raw bytes so a multi-byte char split across the peek boundary survives", async () => {
|
||||
// chunk1 ends mid-é (0xC3); the peek commits on the first complete line
|
||||
// (text-delta "hi") while the decoder still holds 0xC3. Re-emission must
|
||||
// use RAW bytes — re-encoding decoded text would replace 0xC3 with U+FFFD.
|
||||
const first = Buffer.from(
|
||||
'{"type":"text-delta","text":"hi"}\n{"type":"text-delta","text":"caf',
|
||||
);
|
||||
const chunk1 = new Uint8Array([...first, 0xc3]);
|
||||
const rest = new Uint8Array([0xa9, 0x22, 0x7d, 0x0a]); // é"}\n
|
||||
const body = new ReadableStream({
|
||||
start(c) {
|
||||
c.enqueue(chunk1);
|
||||
c.enqueue(rest);
|
||||
c.close();
|
||||
},
|
||||
});
|
||||
const res = await peekForUpstreamError(new Response(body, { status: 200 }), "m");
|
||||
const text = await res.text();
|
||||
expect(text).toContain("café");
|
||||
expect(text).not.toContain("\uFFFD");
|
||||
});
|
||||
});
|
||||
|
||||
99
tests/unit/stream-error-peek.test.js
Normal file
99
tests/unit/stream-error-peek.test.js
Normal file
@@ -0,0 +1,99 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { maybeRejectEarlyStreamError } from "../../open-sse/utils/streamErrorPeek.js";
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
const sseResp = (lines) =>
|
||||
new Response(
|
||||
new ReadableStream({
|
||||
start(c) {
|
||||
for (const l of lines) c.enqueue(encoder.encode(l));
|
||||
c.close();
|
||||
},
|
||||
}),
|
||||
{ status: 200, headers: { "content-type": "text/event-stream" } },
|
||||
);
|
||||
|
||||
describe("maybeRejectEarlyStreamError", () => {
|
||||
it("returns 502 when a pattern matches early stream text", async () => {
|
||||
const res = await maybeRejectEarlyStreamError(
|
||||
sseResp([
|
||||
'{"type":"start"}\n{"type":"error","error":{"type":"server_error","message":"Network connection lost."}}\n',
|
||||
]),
|
||||
["server_error"],
|
||||
);
|
||||
expect(res.status).toBe(502);
|
||||
const body = await res.json();
|
||||
expect(body.error.message).toContain("server_error");
|
||||
});
|
||||
|
||||
it("passes the stream through unchanged when nothing matches", async () => {
|
||||
const res = await maybeRejectEarlyStreamError(
|
||||
sseResp([
|
||||
'data: {"choices":[{"delta":{"content":"hello"},"finish_reason":null}]}\n\ndata: [DONE]\n\n',
|
||||
]),
|
||||
["server_error"],
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
const text = await res.text();
|
||||
expect(text).toContain("hello");
|
||||
expect(text).toContain("[DONE]");
|
||||
});
|
||||
|
||||
it("commits on timeout without hanging", async () => {
|
||||
const stalled = new Response(new ReadableStream({ start() {} }), {
|
||||
status: 200,
|
||||
});
|
||||
const res = await maybeRejectEarlyStreamError(stalled, ["x"], {
|
||||
timeoutMs: 50,
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
await res.body.cancel();
|
||||
});
|
||||
|
||||
it("commits on abort without hanging", async () => {
|
||||
const ctrl = new AbortController();
|
||||
const stalled = new Response(new ReadableStream({ start() {} }), {
|
||||
status: 200,
|
||||
});
|
||||
setTimeout(() => ctrl.abort(new Error("gone")), 10);
|
||||
const res = await maybeRejectEarlyStreamError(stalled, ["x"], {
|
||||
signal: ctrl.signal,
|
||||
timeoutMs: 2000,
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
await res.body.cancel();
|
||||
});
|
||||
|
||||
it("commits (passthrough) when patterns are empty", async () => {
|
||||
const res = await maybeRejectEarlyStreamError(
|
||||
sseResp(["data: hi\n\n"]),
|
||||
[],
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
expect(await res.text()).toBe("data: hi\n\n");
|
||||
});
|
||||
|
||||
it("multi-byte UTF-8 split across peek boundary round-trips losslessly", async () => {
|
||||
// "café" split mid-é: the peek consumes bytes up to and including 0xC3
|
||||
// (the first half of é); the re-emitted stream must contain the RAW bytes
|
||||
// (never a re-encoded decoded string — TextDecoder flush would replace the
|
||||
// lone 0xC3 with U+FFFD and corrupt the output).
|
||||
const bytes = [
|
||||
0x64, 0x61, 0x74, 0x61, 0x3a, 0x20, 0x22, 0x63, 0x61, 0x66, 0xc3,
|
||||
];
|
||||
const rest = new Uint8Array([0xa9, 0x22, 0x0a, 0x0a]);
|
||||
const body = new ReadableStream({
|
||||
start(c) {
|
||||
c.enqueue(new Uint8Array(bytes));
|
||||
c.enqueue(rest);
|
||||
c.close();
|
||||
},
|
||||
});
|
||||
const res = await maybeRejectEarlyStreamError(
|
||||
new Response(body, { status: 200 }),
|
||||
["nomatch"],
|
||||
{ maxBytes: 11 },
|
||||
);
|
||||
expect(await res.text()).toBe('data: "café"\n\n');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user