feat(open-sse): early-peek stream error detection + fix UTF-8 loss in CommandCode peek

- new open-sse/utils/streamErrorPeek.js: bounded peek of the first bytes of a
  200 stream; configured pattern match → 502 so account/combo fallback can run
  before any byte reaches the client (streaming included). Re-emits RAW bytes
  so split multi-byte UTF-8 sequences survive the peek (never re-encode
  decoded text — TextDecoder flush corrupts a lone leading byte to U+FFFD).
- chatCore: run the peek after executor.execute when the provider has
  streamErrorPatterns configured; chat.js passes the settings through.
- commandcode executor: same raw-bytes fix in peekForUpstreamError + regression
  test that fails against the old flush-based re-encode.
This commit is contained in:
2026-08-04 23:33:12 +07:00
parent 008e0ef311
commit e438a03f96
6 changed files with 1301 additions and 583 deletions

View File

@@ -1,11 +1,11 @@
import "open-sse/index.js";
import {
getProviderCredentials,
markAccountUnavailable,
clearAccountError,
extractApiKey,
isValidApiKey,
getProviderCredentials,
markAccountUnavailable,
clearAccountError,
extractApiKey,
isValidApiKey,
} from "../services/auth.js";
import { cacheClaudeHeaders } from "open-sse/utils/claudeHeaderCache.js";
import { getSettings } from "@/lib/localDb";
@@ -20,7 +20,10 @@ import { handleBypassRequest } from "open-sse/utils/bypassHandler.js";
import { HTTP_STATUS } from "open-sse/config/runtimeConfig.js";
import { detectFormatByEndpoint } from "open-sse/translator/formats.js";
import * as log from "../utils/logger.js";
import { updateProviderCredentials, checkAndRefreshToken } from "../services/tokenRefresh.js";
import {
updateProviderCredentials,
checkAndRefreshToken,
} from "../services/tokenRefresh.js";
import { getProjectIdForConnection } from "open-sse/services/projectId.js";
/**
@@ -29,266 +32,355 @@ import { getProjectIdForConnection } from "open-sse/services/projectId.js";
* Format detection and translation handled by translator
*/
export async function handleChat(request, clientRawRequest = null) {
let body;
try {
body = await request.json();
} catch {
log.warn("CHAT", "Invalid JSON body");
return errorResponse(HTTP_STATUS.BAD_REQUEST, "Invalid JSON body");
}
let body;
try {
body = await request.json();
} catch {
log.warn("CHAT", "Invalid JSON body");
return errorResponse(HTTP_STATUS.BAD_REQUEST, "Invalid JSON body");
}
// Build clientRawRequest for logging (if not provided)
if (!clientRawRequest) {
const url = new URL(request.url);
clientRawRequest = {
endpoint: url.pathname,
body,
headers: Object.fromEntries(request.headers.entries())
};
}
cacheClaudeHeaders(clientRawRequest.headers);
// Build clientRawRequest for logging (if not provided)
if (!clientRawRequest) {
const url = new URL(request.url);
clientRawRequest = {
endpoint: url.pathname,
body,
headers: Object.fromEntries(request.headers.entries()),
};
}
cacheClaudeHeaders(clientRawRequest.headers);
const modelStr = body.model;
const modelStr = body.model;
// Request summary is emitted as the unified "▶" line in chatCore (has fmt/thinking/account)
// Request summary is emitted as the unified "▶" line in chatCore (has fmt/thinking/account)
// Log API key (masked)
const authHeader = request.headers.get("Authorization");
const apiKey = extractApiKey(request);
if (authHeader && apiKey) {
const masked = log.maskKey(apiKey);
log.debug("AUTH", `API Key: ${masked}`);
} else {
log.debug("AUTH", "No API key provided (local mode)");
}
// Log API key (masked)
const authHeader = request.headers.get("Authorization");
const apiKey = extractApiKey(request);
if (authHeader && apiKey) {
const masked = log.maskKey(apiKey);
log.debug("AUTH", `API Key: ${masked}`);
} else {
log.debug("AUTH", "No API key provided (local mode)");
}
// Enforce API key if enabled in settings
const settings = await getSettings();
if (settings.requireApiKey) {
if (!apiKey) {
log.warn("AUTH", "Missing API key (requireApiKey=true)");
return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Missing API key");
}
const valid = await isValidApiKey(apiKey);
if (!valid) {
log.warn("AUTH", "Invalid API key (requireApiKey=true)");
return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Invalid API key");
}
}
// Enforce API key if enabled in settings
const settings = await getSettings();
if (settings.requireApiKey) {
if (!apiKey) {
log.warn("AUTH", "Missing API key (requireApiKey=true)");
return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Missing API key");
}
const valid = await isValidApiKey(apiKey);
if (!valid) {
log.warn("AUTH", "Invalid API key (requireApiKey=true)");
return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Invalid API key");
}
}
if (!modelStr) {
log.warn("CHAT", "Missing model");
return errorResponse(HTTP_STATUS.BAD_REQUEST, "Missing model");
}
if (!modelStr) {
log.warn("CHAT", "Missing model");
return errorResponse(HTTP_STATUS.BAD_REQUEST, "Missing model");
}
// Bypass naming/warmup requests before combo rotation to avoid wasting rotation slots
const userAgent = request?.headers?.get("user-agent") || "";
const bypassResponse = handleBypassRequest(body, modelStr, userAgent, !!settings.ccFilterNaming);
if (bypassResponse) return bypassResponse.response || bypassResponse;
// Bypass naming/warmup requests before combo rotation to avoid wasting rotation slots
const userAgent = request?.headers?.get("user-agent") || "";
const bypassResponse = handleBypassRequest(
body,
modelStr,
userAgent,
!!settings.ccFilterNaming,
);
if (bypassResponse) return bypassResponse.response || bypassResponse;
// Check if model is a combo (has multiple models with fallback)
const comboModels = await getComboModels(modelStr);
if (comboModels) {
// Check for combo-specific strategy first, fallback to global
const comboStrategies = settings.comboStrategies || {};
const comboSpecificStrategy = comboStrategies[modelStr]?.fallbackStrategy;
const comboStrategy = comboSpecificStrategy || settings.comboStrategy || "fallback";
// Check if model is a combo (has multiple models with fallback)
const comboModels = await getComboModels(modelStr);
if (comboModels) {
// Check for combo-specific strategy first, fallback to global
const comboStrategies = settings.comboStrategies || {};
const comboSpecificStrategy = comboStrategies[modelStr]?.fallbackStrategy;
const comboStrategy =
comboSpecificStrategy || settings.comboStrategy || "fallback";
if (comboStrategy === "fusion") {
log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models (strategy: fusion)`);
return handleFusionChat({
body,
models: comboModels,
handleSingleModel: (b, m, isPanel) => {
let cleanRawReq = clientRawRequest;
if (isPanel && clientRawRequest) {
const { tools, tool_choice, ...cleanBody } = clientRawRequest.body || {};
cleanRawReq = { ...clientRawRequest, body: cleanBody };
}
return handleSingleModelChat(b, m, cleanRawReq, request, apiKey);
},
log,
comboName: modelStr,
judgeModel: comboStrategies[modelStr]?.judgeModel,
tuning: comboStrategies[modelStr]?.fusionTuning,
});
}
if (comboStrategy === "fusion") {
log.info(
"CHAT",
`Combo "${modelStr}" with ${comboModels.length} models (strategy: fusion)`,
);
return handleFusionChat({
body,
models: comboModels,
handleSingleModel: (b, m, isPanel) => {
let cleanRawReq = clientRawRequest;
if (isPanel && clientRawRequest) {
const { tools, tool_choice, ...cleanBody } =
clientRawRequest.body || {};
cleanRawReq = { ...clientRawRequest, body: cleanBody };
}
return handleSingleModelChat(b, m, cleanRawReq, request, apiKey);
},
log,
comboName: modelStr,
judgeModel: comboStrategies[modelStr]?.judgeModel,
tuning: comboStrategies[modelStr]?.fusionTuning,
});
}
const comboStickyLimit = settings.comboStickyRoundRobinLimit;
log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models (strategy: ${comboStrategy}, sticky: ${comboStickyLimit})`);
return handleComboChat({
body,
models: comboModels,
handleSingleModel: (b, m) => handleSingleModelChat(b, m, clientRawRequest, request, apiKey),
log,
comboName: modelStr,
comboStrategy,
comboStickyLimit
});
}
const comboStickyLimit = settings.comboStickyRoundRobinLimit;
log.info(
"CHAT",
`Combo "${modelStr}" with ${comboModels.length} models (strategy: ${comboStrategy}, sticky: ${comboStickyLimit})`,
);
return handleComboChat({
body,
models: comboModels,
handleSingleModel: (b, m) =>
handleSingleModelChat(b, m, clientRawRequest, request, apiKey),
log,
comboName: modelStr,
comboStrategy,
comboStickyLimit,
});
}
// Single model request
return handleSingleModelChat(body, modelStr, clientRawRequest, request, apiKey);
// Single model request
return handleSingleModelChat(
body,
modelStr,
clientRawRequest,
request,
apiKey,
);
}
/**
* Handle single model chat request
*/
async function handleSingleModelChat(body, modelStr, clientRawRequest = null, request = null, apiKey = null) {
const modelInfo = await getModelInfo(modelStr);
async function handleSingleModelChat(
body,
modelStr,
clientRawRequest = null,
request = null,
apiKey = null,
) {
const modelInfo = await getModelInfo(modelStr);
// If provider is null, this might be a combo name - check and handle
if (!modelInfo.provider) {
const comboModels = await getComboModels(modelStr);
if (comboModels) {
const chatSettings = await getSettings();
// Check for combo-specific strategy first, fallback to global
const comboStrategies = chatSettings.comboStrategies || {};
const comboSpecificStrategy = comboStrategies[modelStr]?.fallbackStrategy;
const comboStrategy = comboSpecificStrategy || chatSettings.comboStrategy || "fallback";
// If provider is null, this might be a combo name - check and handle
if (!modelInfo.provider) {
const comboModels = await getComboModels(modelStr);
if (comboModels) {
const chatSettings = await getSettings();
// Check for combo-specific strategy first, fallback to global
const comboStrategies = chatSettings.comboStrategies || {};
const comboSpecificStrategy = comboStrategies[modelStr]?.fallbackStrategy;
const comboStrategy =
comboSpecificStrategy || chatSettings.comboStrategy || "fallback";
if (comboStrategy === "fusion") {
log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models (strategy: fusion)`);
return handleFusionChat({
body,
models: comboModels,
handleSingleModel: (b, m, isPanel) => {
let cleanRawReq = clientRawRequest;
if (isPanel && clientRawRequest) {
const { tools, tool_choice, ...cleanBody } = clientRawRequest.body || {};
cleanRawReq = { ...clientRawRequest, body: cleanBody };
}
return handleSingleModelChat(b, m, cleanRawReq, request, apiKey);
},
log,
comboName: modelStr,
judgeModel: comboStrategies[modelStr]?.judgeModel,
tuning: comboStrategies[modelStr]?.fusionTuning,
});
}
if (comboStrategy === "fusion") {
log.info(
"CHAT",
`Combo "${modelStr}" with ${comboModels.length} models (strategy: fusion)`,
);
return handleFusionChat({
body,
models: comboModels,
handleSingleModel: (b, m, isPanel) => {
let cleanRawReq = clientRawRequest;
if (isPanel && clientRawRequest) {
const { tools, tool_choice, ...cleanBody } =
clientRawRequest.body || {};
cleanRawReq = { ...clientRawRequest, body: cleanBody };
}
return handleSingleModelChat(b, m, cleanRawReq, request, apiKey);
},
log,
comboName: modelStr,
judgeModel: comboStrategies[modelStr]?.judgeModel,
tuning: comboStrategies[modelStr]?.fusionTuning,
});
}
const comboStickyLimit = chatSettings.comboStickyRoundRobinLimit;
log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models (strategy: ${comboStrategy}, sticky: ${comboStickyLimit})`);
return handleComboChat({
body,
models: comboModels,
handleSingleModel: (b, m) => handleSingleModelChat(b, m, clientRawRequest, request, apiKey),
log,
comboName: modelStr,
comboStrategy,
comboStickyLimit
});
}
log.warn("CHAT", "Invalid model format", { model: modelStr });
return errorResponse(HTTP_STATUS.BAD_REQUEST, "Invalid model format");
}
const comboStickyLimit = chatSettings.comboStickyRoundRobinLimit;
log.info(
"CHAT",
`Combo "${modelStr}" with ${comboModels.length} models (strategy: ${comboStrategy}, sticky: ${comboStickyLimit})`,
);
return handleComboChat({
body,
models: comboModels,
handleSingleModel: (b, m) =>
handleSingleModelChat(b, m, clientRawRequest, request, apiKey),
log,
comboName: modelStr,
comboStrategy,
comboStickyLimit,
});
}
log.warn("CHAT", "Invalid model format", { model: modelStr });
return errorResponse(HTTP_STATUS.BAD_REQUEST, "Invalid model format");
}
const { provider, model } = modelInfo;
const { provider, model } = modelInfo;
// Routing shown in the unified "▶" line (client model → provider/model)
// Routing shown in the unified "▶" line (client model → provider/model)
// Extract userAgent from request
const userAgent = request?.headers?.get("user-agent") || "";
// Optional pin to a specific connection (dashboard test / client override)
const preferredConnectionId = request?.headers?.get("x-connection-id") || null;
// Extract userAgent from request
const userAgent = request?.headers?.get("user-agent") || "";
// Optional pin to a specific connection (dashboard test / client override)
const preferredConnectionId =
request?.headers?.get("x-connection-id") || null;
// Try with available accounts (fallback on errors unless pinned)
const excludeConnectionIds = new Set();
let lastError = null;
let lastStatus = null;
// Try with available accounts (fallback on errors unless pinned)
const excludeConnectionIds = new Set();
let lastError = null;
let lastStatus = null;
while (true) {
const credentials = await getProviderCredentials(provider, excludeConnectionIds, model, { preferredConnectionId });
while (true) {
const credentials = await getProviderCredentials(
provider,
excludeConnectionIds,
model,
{ preferredConnectionId },
);
// All accounts unavailable
if (!credentials || credentials.allRateLimited) {
if (credentials?.allRateLimited) {
const errorMsg = lastError || credentials.lastError || "Unavailable";
const status = lastStatus || Number(credentials.lastErrorCode) || HTTP_STATUS.SERVICE_UNAVAILABLE;
log.warn("CHAT", `[${provider}/${model}] ${errorMsg} (${credentials.retryAfterHuman})`);
return unavailableResponse(status, `[${provider}/${model}] ${errorMsg}`, credentials.retryAfter, credentials.retryAfterHuman);
}
if (excludeConnectionIds.size === 0) {
log.warn("AUTH", `No active credentials for provider: ${provider}`);
return errorResponse(HTTP_STATUS.NOT_FOUND, `No active credentials for provider: ${provider}`);
}
log.warn("CHAT", "No more accounts available", { provider });
return errorResponse(lastStatus || HTTP_STATUS.SERVICE_UNAVAILABLE, lastError || "All accounts unavailable");
}
// All accounts unavailable
if (!credentials || credentials.allRateLimited) {
if (credentials?.allRateLimited) {
const errorMsg = lastError || credentials.lastError || "Unavailable";
const status =
lastStatus ||
Number(credentials.lastErrorCode) ||
HTTP_STATUS.SERVICE_UNAVAILABLE;
log.warn(
"CHAT",
`[${provider}/${model}] ${errorMsg} (${credentials.retryAfterHuman})`,
);
return unavailableResponse(
status,
`[${provider}/${model}] ${errorMsg}`,
credentials.retryAfter,
credentials.retryAfterHuman,
);
}
if (excludeConnectionIds.size === 0) {
log.warn("AUTH", `No active credentials for provider: ${provider}`);
return errorResponse(
HTTP_STATUS.NOT_FOUND,
`No active credentials for provider: ${provider}`,
);
}
log.warn("CHAT", "No more accounts available", { provider });
return errorResponse(
lastStatus || HTTP_STATUS.SERVICE_UNAVAILABLE,
lastError || "All accounts unavailable",
);
}
// Account selection shown in the unified "▶" line (acc:...)
const refreshedCredentials = await checkAndRefreshToken(provider, credentials);
// Account selection shown in the unified "▶" line (acc:...)
const refreshedCredentials = await checkAndRefreshToken(
provider,
credentials,
);
// Ensure real project ID is available for providers that need it (P0 fix: cold miss)
if ((provider === "antigravity" || provider === "gemini-cli") && !refreshedCredentials.projectId) {
const pid = await getProjectIdForConnection(credentials.connectionId, refreshedCredentials.accessToken, provider);
if (pid) {
refreshedCredentials.projectId = pid;
// Persist to DB in background so subsequent requests have it immediately
updateProviderCredentials(credentials.connectionId, { projectId: pid }).catch(() => { });
}
}
// Ensure real project ID is available for providers that need it (P0 fix: cold miss)
if (
(provider === "antigravity" || provider === "gemini-cli") &&
!refreshedCredentials.projectId
) {
const pid = await getProjectIdForConnection(
credentials.connectionId,
refreshedCredentials.accessToken,
provider,
);
if (pid) {
refreshedCredentials.projectId = pid;
// Persist to DB in background so subsequent requests have it immediately
updateProviderCredentials(credentials.connectionId, {
projectId: pid,
}).catch(() => {});
}
}
// Use shared chatCore
const chatSettings = await getSettings();
const providerThinking = (chatSettings.providerThinking || {})[provider] || null;
const result = await handleChatCore({
body: { ...body, model: `${provider}/${model}` },
modelInfo: { provider, model },
credentials: refreshedCredentials,
log,
clientRawRequest,
connectionId: credentials.connectionId,
userAgent,
apiKey,
ccFilterNaming: !!chatSettings.ccFilterNaming,
rtkEnabled: !!chatSettings.rtkEnabled,
headroomEnabled: !!chatSettings.headroomEnabled,
headroomUrl: chatSettings.headroomUrl || DEFAULT_HEADROOM_URL,
headroomCompressUserMessages: !!chatSettings.headroomCompressUserMessages,
cavemanEnabled: !!chatSettings.cavemanEnabled,
cavemanLevel: chatSettings.cavemanLevel || "full",
ponytailEnabled: !!chatSettings.ponytailEnabled,
ponytailLevel: chatSettings.ponytailLevel || "full",
pxpipeEnabled: !!chatSettings.pxpipeEnabled,
pxpipeMinChars: chatSettings.pxpipeMinChars,
pxpipeTimeoutMs: chatSettings.pxpipeTimeoutMs,
// Lazily warms the in-process module on first use; null when not installed (fail-open)
pxpipeTransform: chatSettings.pxpipeEnabled ? await getPxpipeTransform() : null,
onPxpipeEvent: appendPxpipeEvent,
providerThinking,
// Detect source format by endpoint + body
sourceFormatOverride: request?.url ? detectFormatByEndpoint(new URL(request.url).pathname, body) : null,
onCredentialsRefreshed: async (newCreds) => {
await updateProviderCredentials(credentials.connectionId, {
...newCreds,
existingProviderSpecificData: credentials.providerSpecificData,
testStatus: "active"
});
},
onRequestSuccess: async () => {
await clearAccountError(credentials.connectionId, credentials, model);
}
});
// Use shared chatCore
const chatSettings = await getSettings();
const providerThinking =
(chatSettings.providerThinking || {})[provider] || null;
const result = await handleChatCore({
body: { ...body, model: `${provider}/${model}` },
modelInfo: { provider, model },
credentials: refreshedCredentials,
log,
clientRawRequest,
connectionId: credentials.connectionId,
userAgent,
apiKey,
ccFilterNaming: !!chatSettings.ccFilterNaming,
rtkEnabled: !!chatSettings.rtkEnabled,
headroomEnabled: !!chatSettings.headroomEnabled,
headroomUrl: chatSettings.headroomUrl || DEFAULT_HEADROOM_URL,
headroomCompressUserMessages: !!chatSettings.headroomCompressUserMessages,
cavemanEnabled: !!chatSettings.cavemanEnabled,
cavemanLevel: chatSettings.cavemanLevel || "full",
ponytailEnabled: !!chatSettings.ponytailEnabled,
ponytailLevel: chatSettings.ponytailLevel || "full",
pxpipeEnabled: !!chatSettings.pxpipeEnabled,
pxpipeMinChars: chatSettings.pxpipeMinChars,
pxpipeTimeoutMs: chatSettings.pxpipeTimeoutMs,
// Lazily warms the in-process module on first use; null when not installed (fail-open)
pxpipeTransform: chatSettings.pxpipeEnabled
? await getPxpipeTransform()
: null,
onPxpipeEvent: appendPxpipeEvent,
providerThinking,
streamErrorPatterns: chatSettings.streamErrorPatterns || {},
// Detect source format by endpoint + body
sourceFormatOverride: request?.url
? detectFormatByEndpoint(new URL(request.url).pathname, body)
: null,
onCredentialsRefreshed: async (newCreds) => {
await updateProviderCredentials(credentials.connectionId, {
...newCreds,
existingProviderSpecificData: credentials.providerSpecificData,
testStatus: "active",
});
},
onRequestSuccess: async () => {
await clearAccountError(credentials.connectionId, credentials, model);
},
});
if (result.success) return result.response;
if (result.success) return result.response;
// Mark account unavailable (auto-calculates cooldown with exponential backoff, or precise resetsAtMs)
const { shouldFallback } = await markAccountUnavailable(credentials.connectionId, result.status, result.error, provider, model, result.resetsAtMs);
// Mark account unavailable (auto-calculates cooldown with exponential backoff, or precise resetsAtMs)
const { shouldFallback } = await markAccountUnavailable(
credentials.connectionId,
result.status,
result.error,
provider,
model,
result.resetsAtMs,
);
if (shouldFallback) {
// When a connection is explicitly pinned, never rotate to another account.
if (preferredConnectionId) {
log.warn("AUTH", `Pinned account ${credentials.connectionName} unavailable (${result.status}), no fallback`);
return result.response;
}
log.warn("FALLBACK", `⇄ ACC:${credentials.connectionName} UNAVAILABLE (${result.status}) → NEXT ACCOUNT`);
excludeConnectionIds.add(credentials.connectionId);
lastError = result.error;
lastStatus = result.status;
continue;
}
if (shouldFallback) {
// When a connection is explicitly pinned, never rotate to another account.
if (preferredConnectionId) {
log.warn(
"AUTH",
`Pinned account ${credentials.connectionName} unavailable (${result.status}), no fallback`,
);
return result.response;
}
log.warn(
"FALLBACK",
`⇄ ACC:${credentials.connectionName} UNAVAILABLE (${result.status}) → NEXT ACCOUNT`,
);
excludeConnectionIds.add(credentials.connectionId);
lastError = result.error;
lastStatus = result.status;
continue;
}
return result.response;
}
return result.response;
}
}