luulam/9router
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feat : Auto restart after crash

Browse Source
This commit is contained in:
decolua
2026-03-14 09:37:29 +07:00
parent 549223c8cf
commit adae2605bf
31 changed files with 340 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/sse/handlers/chat.js
View File

@@ -111,7 +111,7 @@ async function handleSingleModelChat(body, modelStr, clientRawRequest = null, re
return handleComboChat({
body,
models: comboModels,
handleSingleModel: (b, m) => handleSingleModelChat(b, m, clientRawRequest, request, apiKey, forceSourceFormat),
handleSingleModel: (b, m) => handleSingleModelChat(b, m, clientRawRequest, request, apiKey),
log
});
}
@@ -132,12 +132,12 @@ async function handleSingleModelChat(body, modelStr, clientRawRequest = null, re
const userAgent = request?.headers?.get("user-agent") || "";
// Try with available accounts (fallback on errors)
let excludeConnectionId = null;
const excludeConnectionIds = new Set();
let lastError = null;
let lastStatus = null;
while (true) {
const credentials = await getProviderCredentials(provider, excludeConnectionId, model);
const credentials = await getProviderCredentials(provider, excludeConnectionIds, model);
// All accounts unavailable
if (!credentials || credentials.allRateLimited) {
@@ -147,7 +147,7 @@ async function handleSingleModelChat(body, modelStr, clientRawRequest = null, re
log.warn("CHAT", `[${provider}/${model}] ${errorMsg} (${credentials.retryAfterHuman})`);
return unavailableResponse(status, `[${provider}/${model}] ${errorMsg}`, credentials.retryAfter, credentials.retryAfterHuman);
}
if (!excludeConnectionId) {
if (excludeConnectionIds.size === 0) {
log.error("AUTH", `No credentials for provider: ${provider}`);
return errorResponse(HTTP_STATUS.BAD_REQUEST, `No credentials for provider: ${provider}`);
}
@@ -204,7 +204,7 @@ async function handleSingleModelChat(body, modelStr, clientRawRequest = null, re
if (shouldFallback) {
log.warn("AUTH", `Account ${credentials.connectionName} unavailable (${result.status}), trying fallback`);
excludeConnectionId = credentials.connectionId;
excludeConnectionIds.add(credentials.connectionId);
lastError = result.error;
lastStatus = result.status;
continue;

8
src/sse/handlers/embeddings.js
View File

@@ -80,12 +80,12 @@ export async function handleEmbeddings(request) {
}
// Credential + fallback loop (mirrors handleChat)
let excludeConnectionId = null;
const excludeConnectionIds = new Set();
let lastError = null;
let lastStatus = null;
while (true) {
const credentials = await getProviderCredentials(provider, excludeConnectionId, model);
const credentials = await getProviderCredentials(provider, excludeConnectionIds, model);
// All accounts unavailable
if (!credentials || credentials.allRateLimited) {
@@ -95,7 +95,7 @@ export async function handleEmbeddings(request) {
log.warn("EMBEDDINGS", `[${provider}/${model}] ${errorMsg} (${credentials.retryAfterHuman})`);
return unavailableResponse(status, `[${provider}/${model}] ${errorMsg}`, credentials.retryAfter, credentials.retryAfterHuman);
}
if (!excludeConnectionId) {
if (excludeConnectionIds.size === 0) {
log.error("AUTH", `No credentials for provider: ${provider}`);
return errorResponse(HTTP_STATUS.BAD_REQUEST, `No credentials for provider: ${provider}`);
}
@@ -131,7 +131,7 @@ export async function handleEmbeddings(request) {
if (shouldFallback) {
log.warn("AUTH", `Account ${credentials.connectionName} unavailable (${result.status}), trying fallback`);
excludeConnectionId = credentials.connectionId;
excludeConnectionIds.add(credentials.connectionId);
lastError = result.error;
lastStatus = result.status;
continue;

14
src/sse/services/auth.js
View File

@@ -11,10 +11,14 @@ let selectionMutex = Promise.resolve();
* Get provider credentials from localDb
* Filters out unavailable accounts and returns the selected account based on strategy
* @param {string} provider - Provider name
* @param {string|null} excludeConnectionId - Connection ID to exclude (for retry with next account)
* @param {Set<string>|string|null} excludeConnectionIds - Connection ID(s) to exclude (for retry with next account)
* @param {string|null} model - Model name for per-model rate limit filtering
*/
export async function getProviderCredentials(provider, excludeConnectionId = null, model = null) {
export async function getProviderCredentials(provider, excludeConnectionIds = null, model = null) {
// Normalize to Set for consistent handling
const excludeSet = excludeConnectionIds instanceof Set
? excludeConnectionIds
: (excludeConnectionIds ? new Set([excludeConnectionIds]) : new Set());
// Acquire mutex to prevent race conditions
const currentMutex = selectionMutex;
let resolveMutex;
@@ -27,7 +31,7 @@ export async function getProviderCredentials(provider, excludeConnectionId = nul
const providerId = resolveProviderId(provider);
const connections = await getProviderConnections({ provider: providerId, isActive: true });
log.debug("AUTH", `${provider} | total connections: ${connections.length}, excludeId: ${excludeConnectionId || "none"}, model: ${model || "any"}`);
log.debug("AUTH", `${provider} | total connections: ${connections.length}, excludeIds: ${excludeSet.size > 0 ? [...excludeSet].join(",") : "none"}, model: ${model || "any"}`);
if (connections.length === 0) {
log.warn("AUTH", `No credentials for ${provider}`);
@@ -36,14 +40,14 @@ export async function getProviderCredentials(provider, excludeConnectionId = nul
// Filter out model-locked and excluded connections
const availableConnections = connections.filter(c => {
if (excludeConnectionId && c.id === excludeConnectionId) return false;
if (excludeSet.has(c.id)) return false;
if (isModelLockActive(c, model)) return false;
return true;
});
log.debug("AUTH", `${provider} | available: ${availableConnections.length}/${connections.length}`);
connections.forEach(c => {
const excluded = excludeConnectionId && c.id === excludeConnectionId;
const excluded = excludeSet.has(c.id);
const locked = isModelLockActive(c, model);
if (excluded || locked) {
const lockUntil = getEarliestModelLockUntil(c);