From 96f5e5c92aded1b77945925daaeea7c65094d6fb Mon Sep 17 00:00:00 2001 From: bitgineer <133060368+bitgineer@users.noreply.github.com> Date: Sun, 22 Mar 2026 22:52:31 -0400 Subject: [PATCH] Add combo round-robin strategy to distribute load across providers (#390) - Add comboRotationState Map to track rotation per combo - Add getRotatedModels() to rotate model order based on strategy - Pass comboName and comboStrategy to handleComboChat() - Add comboStrategy setting (default: fallback) - Add UI toggle for Combo Round Robin in profile settings When enabled, each request to a combo starts with a different provider instead of always starting with the first one, distributing load evenly. Co-authored-by: Antigravity Agent --- open-sse/services/combo.js | 47 +++++++++++++++++-- src/app/(dashboard)/dashboard/profile/page.js | 30 ++++++++++++ src/lib/localDb.js | 1 + src/sse/handlers/chat.js | 15 ++++-- 4 files changed, 85 insertions(+), 8 deletions(-) diff --git a/open-sse/services/combo.js b/open-sse/services/combo.js index d61c813c..cefbac38 100644 --- a/open-sse/services/combo.js +++ b/open-sse/services/combo.js @@ -5,6 +5,40 @@ import { checkFallbackError, formatRetryAfter } from "./accountFallback.js"; import { unavailableResponse } from "../utils/error.js"; +/** + * Track rotation state per combo (for round-robin strategy) + * @type {Map} + */ +const comboRotationState = new Map(); + +/** + * Get rotated model list based on strategy + * @param {string[]} models - Array of model strings + * @param {string} comboName - Name of the combo + * @param {string} strategy - "fallback" or "round-robin" + * @returns {string[]} Rotated models array + */ +export function getRotatedModels(models, comboName, strategy) { + if (!models || models.length <= 1 || strategy !== "round-robin") { + return models; + } + + const currentIndex = comboRotationState.get(comboName) || 0; + const rotatedModels = [...models]; + + // Rotate: move models from currentIndex to front, preserving order after + for (let i = 0; i < currentIndex; i++) { + const moved = rotatedModels.shift(); + rotatedModels.push(moved); + } + + // Update state for next request (cycle through all models) + const nextIndex = (currentIndex + 1) % models.length; + comboRotationState.set(comboName, nextIndex); + + return rotatedModels; +} + /** * Get combo models from combos data * @param {string} modelStr - Model string to check @@ -32,16 +66,21 @@ export function getComboModelsFromData(modelStr, combosData) { * @param {string[]} options.models - Array of model strings to try * @param {Function} options.handleSingleModel - Function to handle single model: (body, modelStr) => Promise * @param {Object} options.log - Logger object + * @param {string} [options.comboName] - Name of the combo (for round-robin tracking) + * @param {string} [options.comboStrategy] - Strategy: "fallback" or "round-robin" * @returns {Promise} */ -export async function handleComboChat({ body, models, handleSingleModel, log }) { +export async function handleComboChat({ body, models, handleSingleModel, log, comboName, comboStrategy }) { + // Apply rotation strategy if enabled + const rotatedModels = getRotatedModels(models, comboName, comboStrategy); + let lastError = null; let earliestRetryAfter = null; let lastStatus = null; - for (let i = 0; i < models.length; i++) { - const modelStr = models[i]; - log.info("COMBO", `Trying model ${i + 1}/${models.length}: ${modelStr}`); + for (let i = 0; i < rotatedModels.length; i++) { + const modelStr = rotatedModels[i]; + log.info("COMBO", `Trying model ${i + 1}/${rotatedModels.length}: ${modelStr}`); try { const result = await handleSingleModel(body, modelStr); diff --git a/src/app/(dashboard)/dashboard/profile/page.js b/src/app/(dashboard)/dashboard/profile/page.js index 63d985b2..9d08b800 100644 --- a/src/app/(dashboard)/dashboard/profile/page.js +++ b/src/app/(dashboard)/dashboard/profile/page.js @@ -190,6 +190,21 @@ export default function ProfilePage() { } }; + const updateComboStrategy = async (strategy) => { + try { + const res = await fetch("/api/settings", { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ comboStrategy: strategy }), + }); + if (res.ok) { + setSettings(prev => ({ ...prev, comboStrategy: strategy })); + } + } catch (err) { + console.error("Failed to update combo strategy:", err); + } + }; + const updateStickyLimit = async (limit) => { const numLimit = parseInt(limit); if (isNaN(numLimit) || numLimit < 1) return; @@ -518,6 +533,21 @@ export default function ProfilePage() { )} + {/* Combo Round Robin */} +
+
+

Combo Round Robin

+

+ Cycle through providers in combos instead of always starting with first +

+
+ updateComboStrategy(settings.comboStrategy === "round-robin" ? "fallback" : "round-robin")} + disabled={loading} + /> +
+

{settings.fallbackStrategy === "round-robin" ? `Currently distributing requests across all available accounts with ${settings.stickyRoundRobinLimit || 3} calls per account.` diff --git a/src/lib/localDb.js b/src/lib/localDb.js index 9e7c7e79..053b2911 100644 --- a/src/lib/localDb.js +++ b/src/lib/localDb.js @@ -54,6 +54,7 @@ const defaultData = { tunnelUrl: "", stickyRoundRobinLimit: 3, providerStrategies: {}, + comboStrategy: "fallback", requireLogin: true, observabilityEnabled: true, observabilityMaxRecords: 1000, diff --git a/src/sse/handlers/chat.js b/src/sse/handlers/chat.js index b8b3eb9c..245365b1 100644 --- a/src/sse/handlers/chat.js +++ b/src/sse/handlers/chat.js @@ -84,12 +84,15 @@ export async function handleChat(request, clientRawRequest = null) { // Check if model is a combo (has multiple models with fallback) const comboModels = await getComboModels(modelStr); if (comboModels) { - log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models`); + const comboStrategy = settings.comboStrategy || "fallback"; + log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models (strategy: ${comboStrategy})`); return handleComboChat({ body, models: comboModels, handleSingleModel: (b, m) => handleSingleModelChat(b, m, clientRawRequest, request, apiKey), - log + log, + comboName: modelStr, + comboStrategy }); } @@ -107,12 +110,16 @@ async function handleSingleModelChat(body, modelStr, clientRawRequest = null, re if (!modelInfo.provider) { const comboModels = await getComboModels(modelStr); if (comboModels) { - log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models`); + const chatSettings = await getSettings(); + const comboStrategy = chatSettings.comboStrategy || "fallback"; + log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models (strategy: ${comboStrategy})`); return handleComboChat({ body, models: comboModels, handleSingleModel: (b, m) => handleSingleModelChat(b, m, clientRawRequest, request, apiKey), - log + log, + comboName: modelStr, + comboStrategy }); } log.warn("CHAT", "Invalid model format", { model: modelStr });