luulam/9router
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: pre-request token validation, mid-stream error handling, and usage stats improvements

Browse Source 
- Qoder: handle mid-stream errors by returning proper error Response instead of embedding in stream
- Qoder: add refreshCredentials() to validate token via quota endpoint before requests
- chatCore: validate and refresh provider tokens before sending chat requests
- chatCore: bail early with 401 on unrecoverable token refresh errors
- Usage stats: track apiKey, comboName, fallbackHistory in request details
- Dashboard: improve Combos, Endpoint, Provider, Usage, and RequestDetails pages
- API keys route: upsert logic with provider_type support
- DB repos: usageRepo query improvements, requestDetailsRepo pagination, apiKeysRepo updates
This commit is contained in:
luulam
2026-06-29 10:08:07 +07:00
parent 1fe8115dad
commit 2305e26e25
27 changed files with 1433 additions and 459 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
open-sse/services/combo.js
View File

@@ -226,7 +226,7 @@ export function getComboModelsFromData(modelStr, combosData) {
* @param {number|string} [options.comboStickyLimit=1] - Requests per combo model before switching
* @returns {Promise<Response>}
*/
export async function handleComboChat({ body, models, handleSingleModel, log, comboName, comboStrategy, comboStickyLimit = 1, autoSwitch = true }) {
export async function handleComboChat({ body, models, handleSingleModel, log, comboName, comboStrategy, comboStickyLimit = 1, autoSwitch = true, retryCount = 1, maxFallbackDepth = null }) {
// Apply rotation strategy if enabled
let rotatedModels = getRotatedModels(models, comboName, comboStrategy, comboStickyLimit);
@@ -241,71 +241,102 @@ export async function handleComboChat({ body, models, handleSingleModel, log, co
rotatedModels = reordered;
}
}
const effectiveRetryCount = (Number.isInteger(retryCount) && retryCount >= 1) ? retryCount : 1;
const effectiveDepth = (Number.isInteger(maxFallbackDepth) && maxFallbackDepth >= 1) ? maxFallbackDepth : null;
const effectiveModels = effectiveDepth != null ? rotatedModels.slice(0, effectiveDepth) : rotatedModels;
let lastError = null;
let earliestRetryAfter = null;
let lastStatus = null;
const comboFallbackHistory = [];
for (let i = 0; i < rotatedModels.length; i++) {
const modelStr = rotatedModels[i];
log.info("COMBO", `Trying model ${i + 1}/${rotatedModels.length}: ${modelStr}`);
for (let i = 0; i < effectiveModels.length; i++) {
const modelStr = effectiveModels[i];
log.info("COMBO", `Trying model ${i + 1}/${effectiveModels.length}: ${modelStr}`);
try {
const result = await handleSingleModel(body, modelStr);
// Success (2xx) - return response
if (result.ok) {
log.info("COMBO", `Model ${modelStr} succeeded`);
return result;
for (let attempt = 0; attempt < effectiveRetryCount; attempt++) {
if (attempt > 0) {
log.info("COMBO", `Retrying model ${modelStr} (attempt ${attempt + 1}/${effectiveRetryCount})`);
}
// Extract error info from response
let errorText = result.statusText || "";
let retryAfter = null;
try {
const errorBody = await result.clone().json();
errorText = errorBody?.error?.message || errorBody?.error || errorBody?.message || errorText;
retryAfter = errorBody?.retryAfter || null;
} catch {
// Ignore JSON parse errors
const result = await handleSingleModel(body, modelStr, comboFallbackHistory.length ? [...comboFallbackHistory] : null);
// Success (2xx) - return response
if (result.ok) {
log.info("COMBO", `Model ${modelStr} succeeded${attempt > 0 ? ` on retry ${attempt + 1}` : ""}`);
return result;
}
// Extract error info from response
let errorText = result.statusText || "";
let retryAfter = null;
try {
const errorBody = await result.clone().json();
errorText = errorBody?.error?.message || errorBody?.error || errorBody?.message || errorText;
retryAfter = errorBody?.retryAfter || null;
} catch {
// Ignore JSON parse errors
}
// Track earliest retryAfter across all combo models
if (retryAfter && (!earliestRetryAfter || new Date(retryAfter) < new Date(earliestRetryAfter))) {
earliestRetryAfter = retryAfter;
}
// Normalize error text to string (Worker-safe)
if (typeof errorText !== "string") {
try { errorText = JSON.stringify(errorText); } catch { errorText = String(errorText); }
}
// Check if should fallback to next model
const { shouldFallback, cooldownMs } = checkFallbackError(result.status, errorText);
if (!shouldFallback) {
// Hard failure — no retry, no fallback
log.warn("COMBO", `Model ${modelStr} failed (no fallback)`, { status: result.status });
return result;
}
// For transient errors (503/502/504), wait for cooldown before retrying/falling through
if (cooldownMs && cooldownMs > 0 && cooldownMs <= 5000 &&
(result.status === 503 || result.status === 502 || result.status === 504)) {
log.info("COMBO", `Model ${modelStr} transient ${result.status}, waiting ${cooldownMs}ms`);
await new Promise(r => setTimeout(r, cooldownMs));
}
lastError = errorText || String(result.status);
if (!lastStatus) lastStatus = result.status;
if (attempt < effectiveRetryCount - 1) continue;
// Exhausted retries for this provider — push history and move to next
comboFallbackHistory.push({
model: modelStr,
status: result.status,
error: lastError,
timestamp: new Date().toISOString(),
});
log.warn("COMBO", `Model ${modelStr} exhausted ${effectiveRetryCount} attempt(s), trying next`, { status: result.status });
} catch (error) {
// Catch unexpected exceptions to ensure fallback continues
lastError = error.message || String(error);
if (!lastStatus) lastStatus = 500;
if (attempt < effectiveRetryCount - 1) {
log.warn("COMBO", `Model ${modelStr} threw (attempt ${attempt + 1}/${effectiveRetryCount}), retrying`, { error: lastError });
continue;
}
comboFallbackHistory.push({
model: modelStr,
status: 500,
error: lastError,
timestamp: new Date().toISOString(),
});
log.warn("COMBO", `Model ${modelStr} threw error, trying next`, { error: lastError });
}
// Track earliest retryAfter across all combo models
if (retryAfter && (!earliestRetryAfter || new Date(retryAfter) < new Date(earliestRetryAfter))) {
earliestRetryAfter = retryAfter;
}
// Normalize error text to string (Worker-safe)
if (typeof errorText !== "string") {
try { errorText = JSON.stringify(errorText); } catch { errorText = String(errorText); }
}
// Check if should fallback to next model
const { shouldFallback, cooldownMs } = checkFallbackError(result.status, errorText);
if (!shouldFallback) {
log.warn("COMBO", `Model ${modelStr} failed (no fallback)`, { status: result.status });
return result;
}
// For transient errors (503/502/504), wait for cooldown before falling through
// so a briefly-overloaded provider gets a chance to recover rather than being
// skipped immediately (fixes: combo falls through on transient 503)
if (cooldownMs && cooldownMs > 0 && cooldownMs <= 5000 &&
(result.status === 503 || result.status === 502 || result.status === 504)) {
log.info("COMBO", `Model ${modelStr} transient ${result.status}, waiting ${cooldownMs}ms before next`);
await new Promise(r => setTimeout(r, cooldownMs));
}
// Fallback to next model
lastError = errorText || String(result.status);
if (!lastStatus) lastStatus = result.status;
log.warn("COMBO", `Model ${modelStr} failed, trying next`, { status: result.status });
} catch (error) {
// Catch unexpected exceptions to ensure fallback continues
lastError = error.message || String(error);
if (!lastStatus) lastStatus = 500;
log.warn("COMBO", `Model ${modelStr} threw error, trying next`, { error: lastError });
}
}