feat: pre-request token validation, mid-stream error handling, and usage stats improvements

- Qoder: handle mid-stream errors by returning proper error Response instead of embedding in stream - Qoder: add refreshCredentials() to validate token via quota endpoint before requests - chatCore: validate and refresh provider tokens before sending chat requests - chatCore: bail early with 401 on unrecoverable token refresh errors - Usage stats: track apiKey, comboName, fallbackHistory in request details - Dashboard: improve Combos, Endpoint, Provider, Usage, and RequestDetails pages - API keys route: upsert logic with provider_type support - DB repos: usageRepo query improvements, requestDetailsRepo pagination, apiKeysRepo updates
2026-06-29 10:08:07 +07:00
parent 1fe8115dad
commit 2305e26e25
27 changed files with 1433 additions and 459 deletions
--- a/open-sse/executors/qoder.js
+++ b/open-sse/executors/qoder.js
@@ -368,16 +368,10 @@ async function wrapQoderSSE(response, model, midStreamError = {}) {
    const inner = typeof envelope.body === "string" ? envelope.body : "";
    if (statusVal !== 200) {
      const parsed = parseQoderStreamError(statusVal, inner);
-      // Store error in shared state so onStreamComplete can trigger cooldown
+      // Store error in shared state so the caller can return a proper error Response
      midStreamError.error = { status: parsed.statusCode, message: parsed.message, errorCode: parsed.errorCode };
-      const errChunk = JSON.stringify({
-        id: `qoder-error-${Date.now()}`,
-        object: "chat.completion.chunk",
-        created: Math.floor(Date.now() / 1000),
-        model,
-        choices: [{ index: 0, delta: { content: `\n\n${parsed.message}` }, finish_reason: "stop" }],
-      });
-      controller.enqueue(encoder.encode(`data: ${errChunk}\n\n`));
+      // End the stream with [DONE]; the caller (QoderExecutor.execute) will detect
+      // midStreamError and return a non-2xx Response so the API client gets a real error.
      controller.enqueue(encoder.encode(SSE_DONE));
      doneEmitted = true;
      return;
@@ -555,19 +549,82 @@ export class QoderExecutor extends BaseExecutor {
    }

    const midStreamError = {};
-    const wrapped = await wrapQoderSSE(response, `qoder/${qoderKey}`, midStreamError);
+    let wrapped = await wrapQoderSSE(response, `qoder/${qoderKey}`, midStreamError);
+
+    // If a mid-stream error was detected, return a proper error Response instead of
+    // a successful one with an error message inside.
+    if (midStreamError?.error) {
+      const err = midStreamError.error;
+      log?.error?.("QODER", `Upstream error ${err.status}: ${err.message}`);
+      wrapped = new Response(
+        JSON.stringify({
+          error: {
+            message: err.message,
+            type: "upstream_error",
+            code: String(err.status),
+          }
+        }),
+        {
+          status: err.status >= 400 && err.status < 600 ? err.status : 502,
+          headers: { "Content-Type": "application/json" },
+        }
+      );
+    }
+
    return { response: wrapped, url, headers, transformedBody: payload, midStreamError };
  }

-  // Qoder device tokens don't refresh through OAuth — the upstream returns
-  // 403 for our flow. Surfacing failure via 401-on-chat is enough; the
-  // dashboard tells users to re-login when their token expires (~30 days).
-  async refreshCredentials() {
-    return null;
+  // Validate Qoder token by calling the quota endpoint. If it returns 403,
+  // return a structured unrecoverable error so chatCore bails before sending
+  // the actual chat request (instead of forwarding a "quota exceeded" success).
+  async refreshCredentials(credentials, log) {
+    // Qoder's quota endpoint validates the access token. If it returns 403,
+    // the token is invalid and the user needs to reconnect.
+    const oauth = PROVIDERS.qoder?.oauth;
+    const url = oauth?.quotaUsageUrl || "https://openapi.qoder.sh/api/v2/quota/usage";
+    const authToken = credentials?.accessToken;
+    if (!authToken) return null;
+
+    try {
+      const res = await fetch(url, {
+        headers: {
+          Authorization: `Bearer ${authToken}`,
+          "Content-Type": "application/json",
+        },
+        signal: AbortSignal.timeout(10_000),
+      });
+
+      if (res.status === 403) {
+        const errText = await res.text().catch(() => "");
+        log?.error?.("TOKEN_REFRESH", `Qoder token invalid (403): ${errText}`);
+        return {
+          error: "unrecoverable_refresh_error",
+          code: "invalid_token",
+          message: "qoder token invalid; reconnect the account",
+        };
+      }
+
+      // Token is valid — stamp lastRefreshAt so needsRefresh stays quiet
+      const psd = { ...credentials.providerSpecificData, lastRefreshAt: new Date().toISOString() };
+      return {
+        accessToken: authToken,
+        expiresIn: 86400,
+        providerSpecificData: psd,
+        lastRefreshAt: new Date().toISOString(),
+      };
+    } catch (err) {
+      log?.warn?.("TOKEN_REFRESH", `Qoder refresh check failed: ${err.message}`);
+      return null; // Network blip — let the request proceed, failover handles 403
+    }
  }

-  needsRefresh() {
-    return false;
+  // 24h cooldown before re-checking token validity
+  needsRefresh(credentials) {
+    if (!credentials?.accessToken) return false;
+    const psd = credentials?.providerSpecificData || {};
+    if (!psd.lastRefreshAt) return true;
+    const elapsed = Date.now() - new Date(psd.lastRefreshAt).getTime();
+    return elapsed > 24 * 60 * 60 * 1000;
  }
 }

--- a/open-sse/handlers/chatCore.js
+++ b/open-sse/handlers/chatCore.js
@@ -13,7 +13,7 @@ import { HTTP_STATUS } from "../config/runtimeConfig.js";
 import { handleBypassRequest } from "../utils/bypassHandler.js";
 import { trackPendingRequest, appendRequestLog, saveRequestDetail } from "@/lib/usageDb.js";
 import { getExecutor } from "../executors/index.js";
-import { buildRequestDetail, extractRequestConfig } from "./chatCore/requestDetail.js";
+import { buildRequestDetail, extractRequestConfig, saveUsageStats } from "./chatCore/requestDetail.js";
 import { handleForcedSSEToJson } from "./chatCore/sseToJsonHandler.js";
 import { handleNonStreamingResponse } from "./chatCore/nonStreamingHandler.js";
 import { handleStreamingResponse, buildOnStreamComplete } from "./chatCore/streamingHandler.js";
@@ -34,7 +34,7 @@ import { prefetchRemoteImages } from "../translator/concerns/prefetch.js";
 * @param {object} options.credentials - Provider credentials
 * @param {string} options.sourceFormatOverride - Override detected source format (e.g. "openai-responses")
 */
-export async function handleChatCore({ body, modelInfo, credentials, log, onCredentialsRefreshed, onRequestSuccess, onDisconnect, onMidStreamError, clientRawRequest, connectionId, userAgent, apiKey, ccFilterNaming, rtkEnabled, headroomEnabled, headroomUrl, headroomCompressUserMessages, cavemanEnabled, cavemanLevel, ponytailEnabled, ponytailLevel, sourceFormatOverride, providerThinking }) {
+export async function handleChatCore({ body, modelInfo, credentials, log, onCredentialsRefreshed, onRequestSuccess, onDisconnect, onMidStreamError, clientRawRequest, connectionId, userAgent, apiKey, comboName, fallbackHistory, ccFilterNaming, rtkEnabled, headroomEnabled, headroomUrl, headroomCompressUserMessages, cavemanEnabled, cavemanLevel, ponytailEnabled, ponytailLevel, sourceFormatOverride, providerThinking }) {
  const { provider, model } = modelInfo;
  const requestStartTime = Date.now();

@@ -185,6 +185,36 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
  const msgCount = translatedBody.messages?.length || translatedBody.input?.length || translatedBody.contents?.length || translatedBody.request?.contents?.length || 0;
  log?.debug?.("REQUEST", `${provider.toUpperCase()} | ${model} | ${msgCount} msgs`);

+  // Validate credentials before sending request - try refresh if needed
+  if (!executor.noAuth && credentials?.accessToken) {
+    const needsRefresh = executor.needsRefresh?.(credentials) ?? shouldRefreshCredentials(provider, credentials);
+    if (needsRefresh) {
+      try {
+        const newCredentials = await refreshWithRetry(() => executor.refreshCredentials(credentials, log), 3, log);
+        if (newCredentials?.accessToken || newCredentials?.copilotToken) {
+          log?.info?.("TOKEN", `${provider.toUpperCase()} | refreshed before request`);
+          Object.assign(credentials, newCredentials);
+          if (onCredentialsRefreshed) {
+            try { await onCredentialsRefreshed(newCredentials); } catch (e) { log?.warn?.("TOKEN", `onCredentialsRefreshed failed: ${e.message}`); }
+          }
+        } else if (newCredentials?.error === "unrecoverable_refresh_error") {
+          const msg = newCredentials.message || `${provider} token invalid; reconnect the account`;
+          log?.warn?.("TOKEN", `${provider.toUpperCase()} | unrecoverable refresh error: ${msg}`);
+          trackPendingRequest(model, provider, connectionId, false, true);
+          return createErrorResult(HTTP_STATUS.UNAUTHORIZED, msg);
+        } else {
+          log?.warn?.("TOKEN", `${provider.toUpperCase()} | refresh failed before request`);
+          trackPendingRequest(model, provider, connectionId, false, true);
+          return createErrorResult(HTTP_STATUS.UNAUTHORIZED, `${provider} token refresh failed. Please reconnect the account.`);
+        }
+      } catch (e) {
+        log?.warn?.("TOKEN", `${provider.toUpperCase()} | refresh threw before request: ${e.message}`);
+        trackPendingRequest(model, provider, connectionId, false, true);
+        return createErrorResult(HTTP_STATUS.UNAUTHORIZED, `${provider} token refresh failed: ${e.message}`);
+      }
+    }
+  }
+
  const streamController = createStreamController({
    onDisconnect: (reason) => {
      trackPendingRequest(model, provider, connectionId, false);
@@ -241,7 +271,7 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
    trackPendingRequest(model, provider, connectionId, false, true);
    appendRequestLog({ model, provider, connectionId, status: `FAILED ${error.name === "AbortError" ? 499 : HTTP_STATUS.BAD_GATEWAY}` }).catch(() => { });
    saveRequestDetail(buildRequestDetail({
-      provider, model, connectionId,
+      provider, model, connectionId, apiKey, comboName, fallbackHistory,
      latency: { ttft: 0, total: Date.now() - requestStartTime },
      tokens: { prompt_tokens: 0, completion_tokens: 0 },
      request: extractRequestConfig(body, stream),
@@ -249,6 +279,7 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
      response: { error: error.message || String(error), status: error.name === "AbortError" ? 499 : 502, thinking: null },
      status: "error"
    })).catch(() => { });
+    saveUsageStats({ provider, model, tokens: { prompt_tokens: 0, completion_tokens: 0 }, connectionId, apiKey, endpoint: clientRawRequest?.endpoint, comboName, fallbackHistory, status: "error", label: "ERROR" });

    if (error.name === "AbortError") {
      streamController.handleError(error);
@@ -287,7 +318,7 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
    const { statusCode, message, resetsAtMs } = await parseUpstreamError(providerResponse, executor);
    appendRequestLog({ model, provider, connectionId, status: `FAILED ${statusCode}` }).catch(() => { });
    saveRequestDetail(buildRequestDetail({
-      provider, model, connectionId,
+      provider, model, connectionId, apiKey, comboName, fallbackHistory,
      latency: { ttft: 0, total: Date.now() - requestStartTime },
      tokens: { prompt_tokens: 0, completion_tokens: 0 },
      request: extractRequestConfig(body, stream),
@@ -295,6 +326,7 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
      response: { error: message, status: statusCode, thinking: null },
      status: "error"
    })).catch(() => { });
+    saveUsageStats({ provider, model, tokens: { prompt_tokens: 0, completion_tokens: 0 }, connectionId, apiKey, endpoint: clientRawRequest?.endpoint, comboName, fallbackHistory, status: "error", label: "ERROR" });

    const errMsg = formatProviderError(new Error(message), provider, model, statusCode);
    console.log(`${COLORS.red}[ERROR] ${errMsg}${COLORS.reset}`);
@@ -302,7 +334,7 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
    return createErrorResult(statusCode, errMsg, resetsAtMs);
  }

-  const sharedCtx = { provider, model, body, stream, translatedBody, finalBody, requestStartTime, connectionId, apiKey, clientRawRequest, onRequestSuccess, midStreamError, onMidStreamError };
+  const sharedCtx = { provider, model, body, stream, translatedBody, finalBody, requestStartTime, connectionId, apiKey, comboName, fallbackHistory, clientRawRequest, onRequestSuccess, midStreamError, onMidStreamError };
  const appendLog = (extra) => appendRequestLog({ model, provider, connectionId, ...extra }).catch(() => { });
  const trackDone = () => trackPendingRequest(model, provider, connectionId, false);

--- a/open-sse/handlers/chatCore/nonStreamingHandler.js
+++ b/open-sse/handlers/chatCore/nonStreamingHandler.js
@@ -143,7 +143,7 @@ export function translateNonStreamingResponse(responseBody, targetFormat, source
 /**
 * Handle non-streaming response from provider.
 */
-export async function handleNonStreamingResponse({ providerResponse, provider, model, sourceFormat, targetFormat, body, stream, translatedBody, finalBody, requestStartTime, connectionId, apiKey, clientRawRequest, onRequestSuccess, reqLogger, toolNameMap, trackDone, appendLog, midStreamError, onMidStreamError }) {
+export async function handleNonStreamingResponse({ providerResponse, provider, model, sourceFormat, targetFormat, body, stream, translatedBody, finalBody, requestStartTime, connectionId, apiKey, comboName, fallbackHistory, clientRawRequest, onRequestSuccess, reqLogger, toolNameMap, trackDone, appendLog, midStreamError, onMidStreamError }) {
  trackDone();
  const contentType = providerResponse.headers.get("content-type") || "";
  let responseBody;
@@ -189,7 +189,7 @@ export async function handleNonStreamingResponse({ providerResponse, provider, m

  const usage = extractUsageFromResponse(responseBody);
  appendLog({ tokens: usage, status: "200 OK" });
-  saveUsageStats({ provider, model, tokens: usage, connectionId, apiKey, endpoint: clientRawRequest?.endpoint });
+  saveUsageStats({ provider, model, tokens: usage, connectionId, apiKey, endpoint: clientRawRequest?.endpoint, comboName });

  const translatedResponse = needsTranslation(targetFormat, sourceFormat)
    ? translateNonStreamingResponse(responseBody, targetFormat, sourceFormat)
@@ -234,7 +234,7 @@ export async function handleNonStreamingResponse({ providerResponse, provider, m

  const totalLatency = Date.now() - requestStartTime;
  saveRequestDetail(buildRequestDetail({
-    provider, model, connectionId,
+    provider, model, connectionId, apiKey, comboName, fallbackHistory,
    latency: { ttft: totalLatency, total: totalLatency },
    tokens: usage || { prompt_tokens: 0, completion_tokens: 0 },
    request: extractRequestConfig(body, stream),
--- a/open-sse/handlers/chatCore/requestDetail.js
+++ b/open-sse/handlers/chatCore/requestDetail.js
@@ -60,6 +60,9 @@ export function buildRequestDetail(base, overrides = {}) {
    provider: base.provider || "unknown",
    model: base.model || "unknown",
    connectionId: base.connectionId || undefined,
+    apiKey: base.apiKey || undefined,
+    comboName: base.comboName || null,
+    fallbackHistory: base.fallbackHistory || null,
    timestamp: new Date().toISOString(),
    latency: base.latency || { ttft: 0, total: 0 },
    tokens: base.tokens || { prompt_tokens: 0, completion_tokens: 0 },
@@ -72,13 +75,13 @@ export function buildRequestDetail(base, overrides = {}) {
  };
 }

-export function saveUsageStats({ provider, model, tokens, connectionId, apiKey, endpoint, label = "USAGE" }) {
+export function saveUsageStats({ provider, model, tokens, connectionId, apiKey, endpoint, comboName, fallbackHistory, status = "ok", label = "USAGE" }) {
  if (!tokens || typeof tokens !== "object") return;

  const inTokens = tokens.input_tokens ?? tokens.prompt_tokens ?? 0;
  const outTokens = tokens.output_tokens ?? tokens.completion_tokens ?? 0;

-  if (inTokens === 0 && outTokens === 0) return;
+  if (inTokens === 0 && outTokens === 0 && status !== "error") return;

  // Extract cache/reasoning tokens (unified from different formats)
  const cacheRead = tokens.cache_read_input_tokens || tokens.cached_tokens || tokens.prompt_tokens_details?.cached_tokens || 0;
@@ -112,6 +115,9 @@ export function saveUsageStats({ provider, model, tokens, connectionId, apiKey,
    timestamp: new Date().toISOString(),
    connectionId: connectionId || undefined,
    apiKey: apiKey || undefined,
-    endpoint: endpoint || null
+    endpoint: endpoint || null,
+    comboName: comboName || undefined,
+    fallbackHistory: fallbackHistory || undefined,
+    status: status || "ok",
  }).catch(() => {});
 }
--- a/open-sse/handlers/chatCore/sseToJsonHandler.js
+++ b/open-sse/handlers/chatCore/sseToJsonHandler.js
@@ -102,7 +102,7 @@ export function parseSSEToOpenAIResponse(rawSSE, fallbackModel) {
 * Handle case: provider forced streaming but client wants JSON.
 * Supports both Codex/Responses API SSE and standard Chat Completions SSE.
 */
-export async function handleForcedSSEToJson({ providerResponse, sourceFormat, provider, model, body, stream, translatedBody, finalBody, requestStartTime, connectionId, apiKey, clientRawRequest, onRequestSuccess, trackDone, appendLog }) {
+export async function handleForcedSSEToJson({ providerResponse, sourceFormat, provider, model, body, stream, translatedBody, finalBody, requestStartTime, connectionId, apiKey, comboName, fallbackHistory, clientRawRequest, onRequestSuccess, trackDone, appendLog }) {
  const contentType = providerResponse.headers.get("content-type") || "";
  const isSSE = contentType.includes("text/event-stream") || (contentType === "" && isResponsesProvider(provider));
  if (!isSSE) return null; // not handled here
@@ -110,7 +110,7 @@ export async function handleForcedSSEToJson({ providerResponse, sourceFormat, pr
  trackDone();

  const ctx = {
-    provider, model, connectionId,
+    provider, model, connectionId, apiKey, comboName, fallbackHistory,
    request: extractRequestConfig(body, stream),
    providerRequest: finalBody || translatedBody || null
  };
@@ -124,7 +124,7 @@ export async function handleForcedSSEToJson({ providerResponse, sourceFormat, pr

      const usage = jsonResponse.usage || {};
      appendLog({ tokens: usage, status: "200 OK" });
-      saveUsageStats({ provider, model, tokens: usage, connectionId, apiKey, endpoint: clientRawRequest?.endpoint });
+      saveUsageStats({ provider, model, tokens: usage, connectionId, apiKey, endpoint: clientRawRequest?.endpoint, comboName });

      const { msgItem, textContent } = pickAssistantMessageForChatCompletion(jsonResponse.output);
      const totalLatency = Date.now() - requestStartTime;
@@ -200,7 +200,7 @@ export async function handleForcedSSEToJson({ providerResponse, sourceFormat, pr

    const usage = parsed.usage || {};
    appendLog({ tokens: usage, status: "200 OK" });
-    saveUsageStats({ provider, model, tokens: usage, connectionId, apiKey, endpoint: clientRawRequest?.endpoint });
+    saveUsageStats({ provider, model, tokens: usage, connectionId, apiKey, endpoint: clientRawRequest?.endpoint, comboName });

    const totalLatency = Date.now() - requestStartTime;
    saveRequestDetail(buildRequestDetail({
--- a/open-sse/handlers/chatCore/streamingHandler.js
+++ b/open-sse/handlers/chatCore/streamingHandler.js
@@ -22,7 +22,7 @@ const CODEX_SOURCE_TO_TARGET = {
 /**
 * Determine which SSE transform stream to use based on provider/format.
 */
-function buildTransformStream({ provider, sourceFormat, targetFormat, userAgent, reqLogger, toolNameMap, model, connectionId, body, onStreamComplete, apiKey }) {
+function buildTransformStream({ provider, sourceFormat, targetFormat, userAgent, reqLogger, toolNameMap, model, connectionId, body, onStreamComplete, apiKey, endpoint, comboName }) {
  const isDroidCLI = userAgent?.toLowerCase().includes("droid") || userAgent?.toLowerCase().includes("codex-cli");
  // Responses-API providers (e.g. codex) emit Responses SSE → translate into client format
  const isResponsesProvider = PROVIDERS[provider]?.format === FORMATS.OPENAI_RESPONSES;
@@ -30,23 +30,23 @@ function buildTransformStream({ provider, sourceFormat, targetFormat, userAgent,

  if (needsCodexTranslation) {
    const codexTarget = CODEX_SOURCE_TO_TARGET[sourceFormat] || FORMATS.OPENAI;
-    return createSSETransformStreamWithLogger(FORMATS.OPENAI_RESPONSES, codexTarget, provider, reqLogger, toolNameMap, model, connectionId, body, onStreamComplete, apiKey);
+    return createSSETransformStreamWithLogger(FORMATS.OPENAI_RESPONSES, codexTarget, provider, reqLogger, toolNameMap, model, connectionId, body, onStreamComplete, apiKey, endpoint, comboName);
  }

  if (needsTranslation(targetFormat, sourceFormat)) {
-    return createSSETransformStreamWithLogger(targetFormat, sourceFormat, provider, reqLogger, toolNameMap, model, connectionId, body, onStreamComplete, apiKey);
+    return createSSETransformStreamWithLogger(targetFormat, sourceFormat, provider, reqLogger, toolNameMap, model, connectionId, body, onStreamComplete, apiKey, endpoint, comboName);
  }

-  return createPassthroughStreamWithLogger(provider, reqLogger, model, connectionId, body, onStreamComplete, apiKey);
+  return createPassthroughStreamWithLogger(provider, reqLogger, model, connectionId, body, onStreamComplete, apiKey, endpoint, comboName);
 }

 /**
 * Handle streaming response — pipe provider SSE through transform stream to client.
 */
-export function handleStreamingResponse({ providerResponse, provider, model, sourceFormat, targetFormat, userAgent, body, stream, translatedBody, finalBody, requestStartTime, connectionId, apiKey, clientRawRequest, onRequestSuccess, reqLogger, toolNameMap, streamController, onStreamComplete }) {
+export function handleStreamingResponse({ providerResponse, provider, model, sourceFormat, targetFormat, userAgent, body, stream, translatedBody, finalBody, requestStartTime, connectionId, apiKey, comboName, fallbackHistory, clientRawRequest, onRequestSuccess, reqLogger, toolNameMap, streamController, onStreamComplete }) {
  if (onRequestSuccess) onRequestSuccess();

-  const transformStream = buildTransformStream({ provider, sourceFormat, targetFormat, userAgent, reqLogger, toolNameMap, model, connectionId, body, onStreamComplete, apiKey });
+  const transformStream = buildTransformStream({ provider, sourceFormat, targetFormat, userAgent, reqLogger, toolNameMap, model, connectionId, body, onStreamComplete, apiKey, endpoint: clientRawRequest?.endpoint, comboName });

  // Responses passthrough: synthesize response.failed + [DONE] if the stream aborts/stalls before a terminal event
  const isResponsesPassthrough = sourceFormat === FORMATS.OPENAI_RESPONSES && targetFormat === FORMATS.OPENAI_RESPONSES;
@@ -56,7 +56,7 @@ export function handleStreamingResponse({ providerResponse, provider, model, sou

  const streamDetailId = `${Date.now()}-${Math.random().toString(36).slice(2, 11)}`;
  saveRequestDetail(buildRequestDetail({
-    provider, model, connectionId,
+    provider, model, connectionId, apiKey, comboName, fallbackHistory,
    latency: { ttft: 0, total: Date.now() - requestStartTime },
    tokens: { prompt_tokens: 0, completion_tokens: 0 },
    request: extractRequestConfig(body, stream),
@@ -79,7 +79,7 @@ export function handleStreamingResponse({ providerResponse, provider, model, sou
 * @param {object} options.midStreamError - Shared state object from executor (filled during streaming)
 * @param {function} options.onMidStreamError - Callback to invoke when mid-stream error is detected
 */
-export function buildOnStreamComplete({ provider, model, connectionId, apiKey, requestStartTime, body, stream, finalBody, translatedBody, clientRawRequest, midStreamError, onMidStreamError }) {
+export function buildOnStreamComplete({ provider, model, connectionId, apiKey, comboName, fallbackHistory, requestStartTime, body, stream, finalBody, translatedBody, clientRawRequest, midStreamError, onMidStreamError }) {
  const streamDetailId = `${Date.now()}-${Math.random().toString(36).slice(2, 11)}`;

  const onStreamComplete = (contentObj, usage, ttftAt) => {
@@ -100,7 +100,7 @@ export function buildOnStreamComplete({ provider, model, connectionId, apiKey, r
    }

    saveRequestDetail(buildRequestDetail({
-      provider, model, connectionId,
+      provider, model, connectionId, apiKey, comboName, fallbackHistory,
      latency,
      tokens: usage || { prompt_tokens: 0, completion_tokens: 0 },
      request: extractRequestConfig(body, stream),
--- a/open-sse/services/combo.js
+++ b/open-sse/services/combo.js
@@ -226,7 +226,7 @@ export function getComboModelsFromData(modelStr, combosData) {
 * @param {number|string} [options.comboStickyLimit=1] - Requests per combo model before switching
 * @returns {Promise<Response>}
 */
-export async function handleComboChat({ body, models, handleSingleModel, log, comboName, comboStrategy, comboStickyLimit = 1, autoSwitch = true }) {
+export async function handleComboChat({ body, models, handleSingleModel, log, comboName, comboStrategy, comboStickyLimit = 1, autoSwitch = true, retryCount = 1, maxFallbackDepth = null }) {
  // Apply rotation strategy if enabled
  let rotatedModels = getRotatedModels(models, comboName, comboStrategy, comboStickyLimit);

@@ -241,71 +241,102 @@ export async function handleComboChat({ body, models, handleSingleModel, log, co
      rotatedModels = reordered;
    }
  }
-  
+
+  const effectiveRetryCount = (Number.isInteger(retryCount) && retryCount >= 1) ? retryCount : 1;
+  const effectiveDepth = (Number.isInteger(maxFallbackDepth) && maxFallbackDepth >= 1) ? maxFallbackDepth : null;
+  const effectiveModels = effectiveDepth != null ? rotatedModels.slice(0, effectiveDepth) : rotatedModels;
+
  let lastError = null;
  let earliestRetryAfter = null;
  let lastStatus = null;
+  const comboFallbackHistory = [];

-  for (let i = 0; i < rotatedModels.length; i++) {
-    const modelStr = rotatedModels[i];
-    log.info("COMBO", `Trying model ${i + 1}/${rotatedModels.length}: ${modelStr}`);
+  for (let i = 0; i < effectiveModels.length; i++) {
+    const modelStr = effectiveModels[i];
+    log.info("COMBO", `Trying model ${i + 1}/${effectiveModels.length}: ${modelStr}`);

-    try {
-      const result = await handleSingleModel(body, modelStr);
-      
-      // Success (2xx) - return response
-      if (result.ok) {
-        log.info("COMBO", `Model ${modelStr} succeeded`);
-        return result;
+    for (let attempt = 0; attempt < effectiveRetryCount; attempt++) {
+      if (attempt > 0) {
+        log.info("COMBO", `Retrying model ${modelStr} (attempt ${attempt + 1}/${effectiveRetryCount})`);
      }

-      // Extract error info from response
-      let errorText = result.statusText || "";
-      let retryAfter = null;
      try {
-        const errorBody = await result.clone().json();
-        errorText = errorBody?.error?.message || errorBody?.error || errorBody?.message || errorText;
-        retryAfter = errorBody?.retryAfter || null;
-      } catch {
-        // Ignore JSON parse errors
+        const result = await handleSingleModel(body, modelStr, comboFallbackHistory.length ? [...comboFallbackHistory] : null);
+
+        // Success (2xx) - return response
+        if (result.ok) {
+          log.info("COMBO", `Model ${modelStr} succeeded${attempt > 0 ? ` on retry ${attempt + 1}` : ""}`);
+          return result;
+        }
+
+        // Extract error info from response
+        let errorText = result.statusText || "";
+        let retryAfter = null;
+        try {
+          const errorBody = await result.clone().json();
+          errorText = errorBody?.error?.message || errorBody?.error || errorBody?.message || errorText;
+          retryAfter = errorBody?.retryAfter || null;
+        } catch {
+          // Ignore JSON parse errors
+        }
+
+        // Track earliest retryAfter across all combo models
+        if (retryAfter && (!earliestRetryAfter || new Date(retryAfter) < new Date(earliestRetryAfter))) {
+          earliestRetryAfter = retryAfter;
+        }
+
+        // Normalize error text to string (Worker-safe)
+        if (typeof errorText !== "string") {
+          try { errorText = JSON.stringify(errorText); } catch { errorText = String(errorText); }
+        }
+
+        // Check if should fallback to next model
+        const { shouldFallback, cooldownMs } = checkFallbackError(result.status, errorText);
+
+        if (!shouldFallback) {
+          // Hard failure — no retry, no fallback
+          log.warn("COMBO", `Model ${modelStr} failed (no fallback)`, { status: result.status });
+          return result;
+        }
+
+        // For transient errors (503/502/504), wait for cooldown before retrying/falling through
+        if (cooldownMs && cooldownMs > 0 && cooldownMs <= 5000 &&
+            (result.status === 503 || result.status === 502 || result.status === 504)) {
+          log.info("COMBO", `Model ${modelStr} transient ${result.status}, waiting ${cooldownMs}ms`);
+          await new Promise(r => setTimeout(r, cooldownMs));
+        }
+
+        lastError = errorText || String(result.status);
+        if (!lastStatus) lastStatus = result.status;
+
+        if (attempt < effectiveRetryCount - 1) continue;
+
+        // Exhausted retries for this provider — push history and move to next
+        comboFallbackHistory.push({
+          model: modelStr,
+          status: result.status,
+          error: lastError,
+          timestamp: new Date().toISOString(),
+        });
+        log.warn("COMBO", `Model ${modelStr} exhausted ${effectiveRetryCount} attempt(s), trying next`, { status: result.status });
+      } catch (error) {
+        // Catch unexpected exceptions to ensure fallback continues
+        lastError = error.message || String(error);
+        if (!lastStatus) lastStatus = 500;
+
+        if (attempt < effectiveRetryCount - 1) {
+          log.warn("COMBO", `Model ${modelStr} threw (attempt ${attempt + 1}/${effectiveRetryCount}), retrying`, { error: lastError });
+          continue;
+        }
+
+        comboFallbackHistory.push({
+          model: modelStr,
+          status: 500,
+          error: lastError,
+          timestamp: new Date().toISOString(),
+        });
+        log.warn("COMBO", `Model ${modelStr} threw error, trying next`, { error: lastError });
      }
-
-      // Track earliest retryAfter across all combo models
-      if (retryAfter && (!earliestRetryAfter || new Date(retryAfter) < new Date(earliestRetryAfter))) {
-        earliestRetryAfter = retryAfter;
-      }
-
-      // Normalize error text to string (Worker-safe)
-      if (typeof errorText !== "string") {
-        try { errorText = JSON.stringify(errorText); } catch { errorText = String(errorText); }
-      }
-
-      // Check if should fallback to next model
-      const { shouldFallback, cooldownMs } = checkFallbackError(result.status, errorText);
-
-      if (!shouldFallback) {
-        log.warn("COMBO", `Model ${modelStr} failed (no fallback)`, { status: result.status });
-        return result;
-      }
-
-      // For transient errors (503/502/504), wait for cooldown before falling through
-      // so a briefly-overloaded provider gets a chance to recover rather than being
-      // skipped immediately (fixes: combo falls through on transient 503)
-      if (cooldownMs && cooldownMs > 0 && cooldownMs <= 5000 &&
-          (result.status === 503 || result.status === 502 || result.status === 504)) {
-        log.info("COMBO", `Model ${modelStr} transient ${result.status}, waiting ${cooldownMs}ms before next`);
-        await new Promise(r => setTimeout(r, cooldownMs));
-      }
-
-      // Fallback to next model
-      lastError = errorText || String(result.status);
-      if (!lastStatus) lastStatus = result.status;
-      log.warn("COMBO", `Model ${modelStr} failed, trying next`, { status: result.status });
-    } catch (error) {
-      // Catch unexpected exceptions to ensure fallback continues
-      lastError = error.message || String(error);
-      if (!lastStatus) lastStatus = 500;
-      log.warn("COMBO", `Model ${modelStr} threw error, trying next`, { error: lastError });
    }
  }

--- a/open-sse/utils/stream.js
+++ b/open-sse/utils/stream.js
@@ -49,7 +49,9 @@ export function createSSEStream(options = {}) {
    connectionId = null,
    body = null,
    onStreamComplete = null,
-    apiKey = null
+    apiKey = null,
+    endpoint = null,
+    comboName = null
  } = options;

  let buffer = "";
@@ -335,11 +337,11 @@ export function createSSEStream(options = {}) {
          }

          if (hasValidUsage(usage)) {
-            saveUsageStats({ provider, model, tokens: usage, connectionId, apiKey });
+            saveUsageStats({ provider, model, tokens: usage, connectionId, apiKey, endpoint, comboName });
          } else {
            appendRequestLog({ model, provider, connectionId, tokens: null, status: "200 OK" }).catch(() => { });
          }
-          
+
          // IMPORTANT: In passthrough mode we still must terminate the SSE stream.
          // Some clients (e.g. OpenClaw) expect the OpenAI-style sentinel:
          //   data: [DONE]\n\n
@@ -418,7 +420,7 @@ export function createSSEStream(options = {}) {
        }

        if (hasValidUsage(state?.usage)) {
-          saveUsageStats({ provider: state.provider || targetFormat, model, tokens: state.usage, connectionId, apiKey });
+          saveUsageStats({ provider: state.provider || targetFormat, model, tokens: state.usage, connectionId, apiKey, endpoint, comboName });
        } else {
          appendRequestLog({ model, provider, connectionId, tokens: null, status: "200 OK" }).catch(() => { });
        }
@@ -436,7 +438,7 @@ export function createSSEStream(options = {}) {
  });
 }

-export function createSSETransformStreamWithLogger(targetFormat, sourceFormat, provider = null, reqLogger = null, toolNameMap = null, model = null, connectionId = null, body = null, onStreamComplete = null, apiKey = null) {
+export function createSSETransformStreamWithLogger(targetFormat, sourceFormat, provider = null, reqLogger = null, toolNameMap = null, model = null, connectionId = null, body = null, onStreamComplete = null, apiKey = null, endpoint = null, comboName = null) {
  return createSSEStream({
    mode: STREAM_MODE.TRANSLATE,
    targetFormat,
@@ -448,11 +450,13 @@ export function createSSETransformStreamWithLogger(targetFormat, sourceFormat, p
    connectionId,
    body,
    onStreamComplete,
-    apiKey
+    apiKey,
+    endpoint,
+    comboName
  });
 }

-export function createPassthroughStreamWithLogger(provider = null, reqLogger = null, model = null, connectionId = null, body = null, onStreamComplete = null, apiKey = null) {
+export function createPassthroughStreamWithLogger(provider = null, reqLogger = null, model = null, connectionId = null, body = null, onStreamComplete = null, apiKey = null, endpoint = null, comboName = null) {
  return createSSEStream({
    mode: STREAM_MODE.PASSTHROUGH,
    provider,
@@ -461,6 +465,8 @@ export function createPassthroughStreamWithLogger(provider = null, reqLogger = n
    connectionId,
    body,
    onStreamComplete,
-    apiKey
+    apiKey,
+    endpoint,
+    comboName
  });
 }