feat(combo): add Fusion strategy — parallel panel + judge synthesis
Adds Fusion as a third combo strategy alongside fallback/round-robin. A fusion combo fans the prompt out to all member models in parallel, then a configurable judge model synthesizes one final answer from the panel. - handleFusionChat in open-sse/services/combo.js: quorum-grace collection caps the straggler penalty, anonymized sources prevent judge brand-bias, degrades to a direct answer on single survivor and 503 on total failure. - chat.js dispatches strategy==="fusion" at both combo entry points. - Combos dashboard: per-combo strategy Select replaces the round-robin toggle, fusion reveals a judge picker, plus a strategy/capacity explainer. - tests/unit/combo-fusion.test.js covers fan-out, judge routing/default, quorum-grace straggler drop, single-survivor and total-failure degradation. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
committed by
decolua
parent
37bfcc3719
commit
87e5c1c6dd
BIN
images/fusion-combo-ui.png
Normal file
BIN
images/fusion-combo-ui.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 38 KiB |
@@ -5,6 +5,7 @@
|
|||||||
import { checkFallbackError, formatRetryAfter } from "./accountFallback.js";
|
import { checkFallbackError, formatRetryAfter } from "./accountFallback.js";
|
||||||
import { unavailableResponse } from "../utils/error.js";
|
import { unavailableResponse } from "../utils/error.js";
|
||||||
import { getCapabilitiesForModel } from "../providers/capabilities.js";
|
import { getCapabilitiesForModel } from "../providers/capabilities.js";
|
||||||
|
import { extractTextContent } from "../translator/formats/gemini.js";
|
||||||
|
|
||||||
// Hard capabilities = input modalities; missing one drops request data (e.g. image
|
// Hard capabilities = input modalities; missing one drops request data (e.g. image
|
||||||
// stripped). Must be prioritized. Soft (e.g. search) only degrades a feature.
|
// stripped). Must be prioritized. Soft (e.g. search) only degrades a feature.
|
||||||
@@ -281,3 +282,235 @@ export async function handleComboChat({ body, models, handleSingleModel, log, co
|
|||||||
{ status, headers: { "Content-Type": "application/json" } }
|
{ status, headers: { "Content-Type": "application/json" } }
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract assistant text from a non-stream completion across formats
|
||||||
|
* (OpenAI chat, Claude messages, Gemini, OpenAI Responses). Returns "" if none.
|
||||||
|
* Panel responses are already translated to the client format by chatCore, so the
|
||||||
|
* leaf content→string step reuses the translator's own extractTextContent.
|
||||||
|
*/
|
||||||
|
function extractPanelText(json) {
|
||||||
|
if (!json || typeof json !== "object") return "";
|
||||||
|
|
||||||
|
// OpenAI chat completion
|
||||||
|
const choice = json.choices?.[0];
|
||||||
|
if (choice) {
|
||||||
|
const msg = choice.message ?? choice.delta ?? {};
|
||||||
|
const t = extractTextContent(msg.content);
|
||||||
|
if (t.trim()) return t;
|
||||||
|
if (typeof choice.text === "string" && choice.text.trim()) return choice.text;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Claude messages (text blocks share OpenAI's {type:"text"} shape)
|
||||||
|
const claudeText = extractTextContent(json.content);
|
||||||
|
if (claudeText.trim()) return claudeText;
|
||||||
|
|
||||||
|
// Gemini (parts carry .text without a type discriminator)
|
||||||
|
const parts = json.candidates?.[0]?.content?.parts;
|
||||||
|
if (Array.isArray(parts)) {
|
||||||
|
const t = parts.map((p) => p?.text || "").join("");
|
||||||
|
if (t.trim()) return t;
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenAI Responses API
|
||||||
|
if (Array.isArray(json.output)) {
|
||||||
|
const t = json.output
|
||||||
|
.flatMap((o) => (Array.isArray(o.content) ? o.content.map((c) => c?.text || "") : []))
|
||||||
|
.join("");
|
||||||
|
if (t.trim()) return t;
|
||||||
|
}
|
||||||
|
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Append a synthesized user turn to whichever message array the request format uses.
|
||||||
|
* Preserves the original conversation + system prompt so the judge has full context.
|
||||||
|
*/
|
||||||
|
function appendUserTurn(body, text) {
|
||||||
|
const next = { ...body };
|
||||||
|
if (Array.isArray(body.messages)) {
|
||||||
|
next.messages = [...body.messages, { role: "user", content: text }];
|
||||||
|
} else if (Array.isArray(body.input)) {
|
||||||
|
next.input = [...body.input, { role: "user", content: text }];
|
||||||
|
} else if (Array.isArray(body.contents)) {
|
||||||
|
next.contents = [...body.contents, { role: "user", parts: [{ text }] }];
|
||||||
|
} else {
|
||||||
|
next.messages = [{ role: "user", content: text }];
|
||||||
|
}
|
||||||
|
return next;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build the judge directive. Per OpenRouter's Fusion design, the judge does NOT
|
||||||
|
* merge — it analyzes (consensus / contradictions / partial coverage / unique
|
||||||
|
* insights / blind spots) then writes one answer grounded in that analysis.
|
||||||
|
* ~3/4 of fusion's quality lift comes from this synthesis step.
|
||||||
|
*
|
||||||
|
* Sources are anonymized ("Source N") so the judge weighs substance, not the
|
||||||
|
* reputation of a model brand.
|
||||||
|
*/
|
||||||
|
function buildJudgePrompt(answers) {
|
||||||
|
const panel = answers
|
||||||
|
.map((a, i) => `[Source ${i + 1}]\n${a.text}`)
|
||||||
|
.join("\n\n");
|
||||||
|
|
||||||
|
return [
|
||||||
|
`You are the JUDGE in a model-fusion panel. ${answers.length} expert models independently answered the user's most recent request. Their responses are below, anonymized by source.`,
|
||||||
|
"",
|
||||||
|
"Do NOT mention that multiple models were used, and do NOT refer to the sources. Produce ONE authoritative final answer addressed directly to the user.",
|
||||||
|
"",
|
||||||
|
"First, internally analyze the panel along these dimensions: consensus (points most sources agree on — treat as higher-confidence), contradictions (where they disagree — resolve with your own judgment), partial coverage, unique insights only one source surfaced, and blind spots every source missed. Then write the best possible final answer grounded in that analysis — more complete and correct than any single response, with no filler.",
|
||||||
|
"",
|
||||||
|
"=== PANEL RESPONSES ===",
|
||||||
|
panel,
|
||||||
|
"=== END PANEL RESPONSES ===",
|
||||||
|
"",
|
||||||
|
"Now write the final answer to the user's original request.",
|
||||||
|
].join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fusion tuning. Overridable per-combo via settings.comboStrategies[name].
|
||||||
|
const FUSION_DEFAULTS = {
|
||||||
|
minPanel: 2, // answers needed before stragglers get a grace window
|
||||||
|
stragglerGraceMs: 8000, // wait this long for laggards once quorum is reached
|
||||||
|
panelHardTimeoutMs: 90000, // absolute cap so one hung model can't stall forever
|
||||||
|
};
|
||||||
|
|
||||||
|
// Resolve a Response (or {__error}) within ms; the loser keeps running but is ignored.
|
||||||
|
function withTimeout(promise, ms) {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
const t = setTimeout(() => resolve({ __timeout: true }), ms);
|
||||||
|
Promise.resolve(promise)
|
||||||
|
.then((v) => { clearTimeout(t); resolve(v); })
|
||||||
|
.catch((e) => { clearTimeout(t); resolve({ __error: e }); });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Collect panel responses with quorum-grace: as soon as `minPanel` calls succeed,
|
||||||
|
* start a short grace timer for the rest, then proceed with whatever arrived. This
|
||||||
|
* caps the straggler penalty (the slowest model otherwise dominates wall time) while
|
||||||
|
* still preferring a full panel when everyone is fast. Bounded by a hard timeout.
|
||||||
|
* Returns a sparse array aligned to `calls` (undefined = not yet / dropped).
|
||||||
|
*/
|
||||||
|
function collectPanel(calls, { minPanel, stragglerGraceMs, panelHardTimeoutMs }) {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
const out = new Array(calls.length);
|
||||||
|
let settled = 0;
|
||||||
|
let ok = 0;
|
||||||
|
let finished = false;
|
||||||
|
let graceTimer = null;
|
||||||
|
const finish = () => {
|
||||||
|
if (finished) return;
|
||||||
|
finished = true;
|
||||||
|
clearTimeout(hardTimer);
|
||||||
|
if (graceTimer) clearTimeout(graceTimer);
|
||||||
|
resolve(out);
|
||||||
|
};
|
||||||
|
const hardTimer = setTimeout(finish, panelHardTimeoutMs);
|
||||||
|
calls.forEach((p, i) => {
|
||||||
|
Promise.resolve(p)
|
||||||
|
.then((v) => { out[i] = v; })
|
||||||
|
.catch((e) => { out[i] = { __error: e }; })
|
||||||
|
.finally(() => {
|
||||||
|
settled++;
|
||||||
|
if (out[i] && out[i].ok) ok++;
|
||||||
|
if (settled === calls.length) return finish();
|
||||||
|
if (ok >= minPanel && !graceTimer) graceTimer = setTimeout(finish, stragglerGraceMs);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle a fusion combo: fan the prompt out to every panel model in parallel,
|
||||||
|
* then a judge model synthesizes one final answer from all panel responses.
|
||||||
|
*
|
||||||
|
* Panel calls are forced non-streaming with tools stripped (the judge needs
|
||||||
|
* complete prose to synthesize). The judge call keeps the client's original
|
||||||
|
* stream flag + tools, so streaming and downstream tool use still work.
|
||||||
|
*
|
||||||
|
* Speed: quorum-grace collection caps the straggler penalty. Quality: the judge
|
||||||
|
* runs the consensus/contradiction/blind-spot analysis before writing.
|
||||||
|
*
|
||||||
|
* Degrades gracefully: 0 panel answers -> 503, exactly 1 -> return it directly.
|
||||||
|
*
|
||||||
|
* @param {Object} options
|
||||||
|
* @param {Object} options.body - Request body (client format)
|
||||||
|
* @param {string[]} options.models - Panel model strings
|
||||||
|
* @param {Function} options.handleSingleModel - (body, modelStr) => Promise<Response>
|
||||||
|
* @param {Object} options.log - Logger
|
||||||
|
* @param {string} [options.comboName] - Combo name (logging)
|
||||||
|
* @param {string} [options.judgeModel] - Judge model; falls back to panel[0]
|
||||||
|
* @param {Object} [options.tuning] - Override FUSION_DEFAULTS (minPanel, grace, timeout)
|
||||||
|
* @returns {Promise<Response>}
|
||||||
|
*/
|
||||||
|
export async function handleFusionChat({ body, models, handleSingleModel, log, comboName, judgeModel, tuning }) {
|
||||||
|
const panel = Array.isArray(models) ? models.filter(Boolean) : [];
|
||||||
|
if (panel.length === 0) {
|
||||||
|
return new Response(
|
||||||
|
JSON.stringify({ error: { message: "Fusion combo has no models" } }),
|
||||||
|
{ status: 400, headers: { "Content-Type": "application/json" } }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// A single-model fusion has nothing to fuse — just answer directly.
|
||||||
|
if (panel.length === 1) {
|
||||||
|
return handleSingleModel(body, panel[0]);
|
||||||
|
}
|
||||||
|
|
||||||
|
const cfg = { ...FUSION_DEFAULTS, ...(tuning || {}) };
|
||||||
|
const minPanel = Math.min(Math.max(2, cfg.minPanel), panel.length);
|
||||||
|
const judge = judgeModel && judgeModel.trim() ? judgeModel.trim() : panel[0];
|
||||||
|
log.info("FUSION", `Combo "${comboName}" | panel=${panel.length} [${panel.join(", ")}] | judge=${judge} | quorum=${minPanel}`);
|
||||||
|
|
||||||
|
// 1. Fan out to the panel in parallel: non-streaming, tools stripped (we want prose).
|
||||||
|
const { tools, tool_choice, ...rest } = body;
|
||||||
|
const panelBody = { ...rest, stream: false };
|
||||||
|
const t0 = Date.now();
|
||||||
|
const calls = panel.map((m) => withTimeout(handleSingleModel(panelBody, m), cfg.panelHardTimeoutMs));
|
||||||
|
const settled = await collectPanel(calls, { ...cfg, minPanel });
|
||||||
|
log.info("FUSION", `fan-out collected in ${Date.now() - t0}ms`);
|
||||||
|
|
||||||
|
// 2. Collect successful answers.
|
||||||
|
const answers = [];
|
||||||
|
for (let i = 0; i < settled.length; i++) {
|
||||||
|
const res = settled[i];
|
||||||
|
const model = panel[i];
|
||||||
|
if (!res) { log.warn("FUSION", `Panel ${model} dropped (straggler/timeout)`); continue; }
|
||||||
|
if (res.__timeout) { log.warn("FUSION", `Panel ${model} timed out`); continue; }
|
||||||
|
if (res.__error) { log.warn("FUSION", `Panel ${model} threw`, { error: res.__error?.message || String(res.__error) }); continue; }
|
||||||
|
if (!res.ok) { log.warn("FUSION", `Panel ${model} failed`, { status: res.status }); continue; }
|
||||||
|
try {
|
||||||
|
const json = await res.clone().json();
|
||||||
|
const text = extractPanelText(json);
|
||||||
|
if (text) {
|
||||||
|
answers.push({ model, text });
|
||||||
|
log.info("FUSION", `Panel ${model} ok (${text.length} chars)`);
|
||||||
|
} else {
|
||||||
|
log.warn("FUSION", `Panel ${model} returned empty content`);
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
log.warn("FUSION", `Panel ${model} unparseable`, { error: e.message || String(e) });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Degrade gracefully when the panel is too thin to fuse.
|
||||||
|
if (answers.length === 0) {
|
||||||
|
log.warn("FUSION", "All panel models failed");
|
||||||
|
return new Response(
|
||||||
|
JSON.stringify({ error: { message: "All fusion panel models failed" } }),
|
||||||
|
{ status: 503, headers: { "Content-Type": "application/json" } }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (answers.length === 1) {
|
||||||
|
log.info("FUSION", `Only ${answers[0].model} succeeded — answering directly (no fusion)`);
|
||||||
|
return handleSingleModel(body, answers[0].model);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Judge analyzes + writes one final answer (streams to client if requested).
|
||||||
|
const judgeBody = appendUserTurn(body, buildJudgePrompt(answers));
|
||||||
|
log.info("FUSION", `Judging ${answers.length} answers with ${judge}`);
|
||||||
|
return handleSingleModel(judgeBody, judge);
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import { DndContext, closestCenter, KeyboardSensor, PointerSensor, useSensor, us
|
|||||||
import { arrayMove, SortableContext, sortableKeyboardCoordinates, useSortable, verticalListSortingStrategy } from "@dnd-kit/sortable";
|
import { arrayMove, SortableContext, sortableKeyboardCoordinates, useSortable, verticalListSortingStrategy } from "@dnd-kit/sortable";
|
||||||
import { CSS } from "@dnd-kit/utilities";
|
import { CSS } from "@dnd-kit/utilities";
|
||||||
import { restrictToVerticalAxis, restrictToParentElement } from "@dnd-kit/modifiers";
|
import { restrictToVerticalAxis, restrictToParentElement } from "@dnd-kit/modifiers";
|
||||||
import { Card, Button, Modal, Input, CardSkeleton, ModelSelectModal, Toggle, ConfirmModal, CapacityBadges } from "@/shared/components";
|
import { Card, Button, Modal, Input, CardSkeleton, ModelSelectModal, ConfirmModal, CapacityBadges, Select } from "@/shared/components";
|
||||||
import { useCopyToClipboard } from "@/shared/hooks/useCopyToClipboard";
|
import { useCopyToClipboard } from "@/shared/hooks/useCopyToClipboard";
|
||||||
import { isOpenAICompatibleProvider, isAnthropicCompatibleProvider } from "@/shared/constants/providers";
|
import { isOpenAICompatibleProvider, isAnthropicCompatibleProvider } from "@/shared/constants/providers";
|
||||||
|
|
||||||
@@ -115,21 +115,25 @@ export default function CombosPage() {
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleToggleRoundRobin = async (comboName, enabled) => {
|
// Merge a per-combo strategy patch into settings.comboStrategies. Passing an empty
|
||||||
|
// patch (strategy back to default "fallback") drops the entry entirely.
|
||||||
|
const handleSetComboStrategy = async (comboName, patch) => {
|
||||||
try {
|
try {
|
||||||
const updated = { ...comboStrategies };
|
const updated = { ...comboStrategies };
|
||||||
if (enabled) {
|
const next = { ...(updated[comboName] || {}), ...patch };
|
||||||
updated[comboName] = { fallbackStrategy: "round-robin" };
|
// Prune to keep settings clean: default fallback with no extras = no entry.
|
||||||
} else {
|
if (!next.fallbackStrategy || next.fallbackStrategy === "fallback") {
|
||||||
delete updated[comboName];
|
delete updated[comboName];
|
||||||
|
} else {
|
||||||
|
updated[comboName] = next;
|
||||||
}
|
}
|
||||||
|
|
||||||
await fetch("/api/settings", {
|
await fetch("/api/settings", {
|
||||||
method: "PATCH",
|
method: "PATCH",
|
||||||
headers: { "Content-Type": "application/json" },
|
headers: { "Content-Type": "application/json" },
|
||||||
body: JSON.stringify({ comboStrategies: updated }),
|
body: JSON.stringify({ comboStrategies: updated }),
|
||||||
});
|
});
|
||||||
|
|
||||||
setComboStrategies(updated);
|
setComboStrategies(updated);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.log("Error updating combo strategy:", error);
|
console.log("Error updating combo strategy:", error);
|
||||||
@@ -150,10 +154,15 @@ export default function CombosPage() {
|
|||||||
{/* Header */}
|
{/* Header */}
|
||||||
<div className="flex flex-col gap-3 sm:flex-row sm:items-center sm:justify-between">
|
<div className="flex flex-col gap-3 sm:flex-row sm:items-center sm:justify-between">
|
||||||
<div className="min-w-0">
|
<div className="min-w-0">
|
||||||
<h1 className="text-2xl font-semibold">Combos</h1>
|
|
||||||
<p className="text-sm text-text-muted mt-1">
|
<p className="text-sm text-text-muted mt-1">
|
||||||
Create model combos with fallback support — auto-adapts per request: routes images to vision models and web search to search-capable models.
|
Group models under one name, then pick a strategy per combo:
|
||||||
</p>
|
</p>
|
||||||
|
<ul className="text-sm text-text-muted mt-2 flex flex-col gap-1">
|
||||||
|
<li><span className="font-medium text-text-main">Fallback</span> — tries models in order (next on failure)</li>
|
||||||
|
<li><span className="font-medium text-text-main">Round Robin</span> — rotates models across requests to spread load</li>
|
||||||
|
<li><span className="font-medium text-text-main">Fusion</span> — queries all models in parallel, then a judge synthesizes one answer</li>
|
||||||
|
<li><span className="font-medium text-text-main">Capacity auto-switch</span> — reorders models per request so images/PDFs route to capable models first</li>
|
||||||
|
</ul>
|
||||||
</div>
|
</div>
|
||||||
<Button icon="add" onClick={() => setShowCreateModal(true)} className="w-full sm:w-auto">
|
<Button icon="add" onClick={() => setShowCreateModal(true)} className="w-full sm:w-auto">
|
||||||
Create Combo
|
Create Combo
|
||||||
@@ -181,12 +190,13 @@ export default function CombosPage() {
|
|||||||
key={combo.id}
|
key={combo.id}
|
||||||
combo={combo}
|
combo={combo}
|
||||||
modelCaps={modelCaps}
|
modelCaps={modelCaps}
|
||||||
|
activeProviders={activeProviders}
|
||||||
copied={copied}
|
copied={copied}
|
||||||
onCopy={copy}
|
onCopy={copy}
|
||||||
onEdit={() => setEditingCombo(combo)}
|
onEdit={() => setEditingCombo(combo)}
|
||||||
onDelete={() => handleDelete(combo.id)}
|
onDelete={() => handleDelete(combo.id)}
|
||||||
roundRobinEnabled={comboStrategies[combo.name]?.fallbackStrategy === "round-robin"}
|
strategy={comboStrategies[combo.name] || {}}
|
||||||
onToggleRoundRobin={(enabled) => handleToggleRoundRobin(combo.name, enabled)}
|
onSetStrategy={(patch) => handleSetComboStrategy(combo.name, patch)}
|
||||||
/>
|
/>
|
||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
@@ -224,7 +234,18 @@ export default function CombosPage() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function ComboCard({ combo, modelCaps = {}, copied, onCopy, onEdit, onDelete, roundRobinEnabled, onToggleRoundRobin }) {
|
const STRATEGY_OPTIONS = [
|
||||||
|
{ value: "fallback", label: "Fallback — try in order" },
|
||||||
|
{ value: "round-robin", label: "Round Robin — rotate" },
|
||||||
|
{ value: "fusion", label: "Fusion — panel + judge" },
|
||||||
|
];
|
||||||
|
|
||||||
|
function ComboCard({ combo, modelCaps = {}, activeProviders = [], copied, onCopy, onEdit, onDelete, strategy = {}, onSetStrategy }) {
|
||||||
|
const [showJudgeSelect, setShowJudgeSelect] = useState(false);
|
||||||
|
const current = strategy.fallbackStrategy || "fallback";
|
||||||
|
const judge = strategy.judgeModel || "";
|
||||||
|
const isFusion = current === "fusion";
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<Card padding="sm" className="group">
|
<Card padding="sm" className="group">
|
||||||
<div className="flex min-w-0 flex-col gap-3 sm:flex-row sm:items-center sm:justify-between">
|
<div className="flex min-w-0 flex-col gap-3 sm:flex-row sm:items-center sm:justify-between">
|
||||||
@@ -249,18 +270,41 @@ function ComboCard({ combo, modelCaps = {}, copied, onCopy, onEdit, onDelete, ro
|
|||||||
<span className="text-[10px] text-text-muted">+{combo.models.length - 3} more</span>
|
<span className="text-[10px] text-text-muted">+{combo.models.length - 3} more</span>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
{/* Fusion: judge picker (Auto = first model) */}
|
||||||
|
{isFusion && (
|
||||||
|
<div className="mt-2 flex min-w-0 flex-wrap items-center gap-1.5">
|
||||||
|
<span className="text-[11px] font-medium text-text-muted">Judge</span>
|
||||||
|
<button
|
||||||
|
onClick={() => setShowJudgeSelect(true)}
|
||||||
|
className="inline-flex max-w-full items-center gap-1 rounded border border-dashed border-primary/40 px-1.5 py-0.5 font-mono text-[11px] text-primary hover:border-primary hover:bg-primary/5 transition-colors"
|
||||||
|
title="Pick the model that fuses panel answers"
|
||||||
|
>
|
||||||
|
<span className="material-symbols-outlined text-[13px]">gavel</span>
|
||||||
|
<span className="truncate">{judge || `Auto — ${combo.models[0] || "first model"}`}</span>
|
||||||
|
</button>
|
||||||
|
{judge && (
|
||||||
|
<button
|
||||||
|
onClick={() => onSetStrategy({ judgeModel: "" })}
|
||||||
|
className="p-0.5 rounded text-text-muted hover:text-red-500 hover:bg-red-500/10 transition-colors"
|
||||||
|
title="Reset judge to Auto"
|
||||||
|
>
|
||||||
|
<span className="material-symbols-outlined text-[13px]">close</span>
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Actions */}
|
{/* Actions */}
|
||||||
<div className="flex w-full flex-col gap-2 sm:w-auto sm:flex-row sm:items-center sm:gap-3 sm:shrink-0">
|
<div className="flex w-full flex-col gap-2 sm:w-auto sm:flex-row sm:items-center sm:gap-3 sm:shrink-0">
|
||||||
{/* Round Robin Toggle — always visible */}
|
{/* Strategy selector — always visible */}
|
||||||
<div className="flex items-center justify-between gap-1.5 rounded-lg bg-black/[0.02] px-2 py-1.5 dark:bg-white/[0.02] sm:justify-start sm:bg-transparent sm:px-0 sm:py-0 sm:dark:bg-transparent">
|
<div className="w-full sm:w-[200px]">
|
||||||
<span className="text-xs text-text-muted font-medium">Round Robin</span>
|
<Select
|
||||||
<Toggle
|
options={STRATEGY_OPTIONS}
|
||||||
size="sm"
|
value={current}
|
||||||
checked={roundRobinEnabled}
|
onChange={(e) => onSetStrategy({ fallbackStrategy: e.target.value })}
|
||||||
onChange={onToggleRoundRobin}
|
selectClassName="py-1.5 text-xs"
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -294,6 +338,17 @@ function ComboCard({ combo, modelCaps = {}, copied, onCopy, onEdit, onDelete, ro
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* Judge model picker (single-select; combo members make natural judges too) */}
|
||||||
|
<ModelSelectModal
|
||||||
|
isOpen={showJudgeSelect}
|
||||||
|
onClose={() => setShowJudgeSelect(false)}
|
||||||
|
onSelect={(m) => { onSetStrategy({ judgeModel: m?.value || "" }); setShowJudgeSelect(false); }}
|
||||||
|
activeProviders={activeProviders}
|
||||||
|
title="Select Judge Model"
|
||||||
|
addedModelValues={judge ? [judge] : []}
|
||||||
|
closeOnSelect={true}
|
||||||
|
/>
|
||||||
</Card>
|
</Card>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import { getSettings } from "@/lib/localDb";
|
|||||||
import { getModelInfo, getComboModels } from "../services/model.js";
|
import { getModelInfo, getComboModels } from "../services/model.js";
|
||||||
import { handleChatCore } from "open-sse/handlers/chatCore.js";
|
import { handleChatCore } from "open-sse/handlers/chatCore.js";
|
||||||
import { errorResponse, unavailableResponse } from "open-sse/utils/error.js";
|
import { errorResponse, unavailableResponse } from "open-sse/utils/error.js";
|
||||||
import { handleComboChat } from "open-sse/services/combo.js";
|
import { handleComboChat, handleFusionChat } from "open-sse/services/combo.js";
|
||||||
import { handleBypassRequest } from "open-sse/utils/bypassHandler.js";
|
import { handleBypassRequest } from "open-sse/utils/bypassHandler.js";
|
||||||
import { HTTP_STATUS } from "open-sse/config/runtimeConfig.js";
|
import { HTTP_STATUS } from "open-sse/config/runtimeConfig.js";
|
||||||
import { detectFormatByEndpoint } from "open-sse/translator/formats.js";
|
import { detectFormatByEndpoint } from "open-sse/translator/formats.js";
|
||||||
@@ -96,7 +96,20 @@ export async function handleChat(request, clientRawRequest = null) {
|
|||||||
const comboStrategies = settings.comboStrategies || {};
|
const comboStrategies = settings.comboStrategies || {};
|
||||||
const comboSpecificStrategy = comboStrategies[modelStr]?.fallbackStrategy;
|
const comboSpecificStrategy = comboStrategies[modelStr]?.fallbackStrategy;
|
||||||
const comboStrategy = comboSpecificStrategy || settings.comboStrategy || "fallback";
|
const comboStrategy = comboSpecificStrategy || settings.comboStrategy || "fallback";
|
||||||
|
|
||||||
|
if (comboStrategy === "fusion") {
|
||||||
|
log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models (strategy: fusion)`);
|
||||||
|
return handleFusionChat({
|
||||||
|
body,
|
||||||
|
models: comboModels,
|
||||||
|
handleSingleModel: (b, m) => handleSingleModelChat(b, m, clientRawRequest, request, apiKey),
|
||||||
|
log,
|
||||||
|
comboName: modelStr,
|
||||||
|
judgeModel: comboStrategies[modelStr]?.judgeModel,
|
||||||
|
tuning: comboStrategies[modelStr]?.fusionTuning,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
const comboStickyLimit = settings.comboStickyRoundRobinLimit;
|
const comboStickyLimit = settings.comboStickyRoundRobinLimit;
|
||||||
log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models (strategy: ${comboStrategy}, sticky: ${comboStickyLimit})`);
|
log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models (strategy: ${comboStrategy}, sticky: ${comboStickyLimit})`);
|
||||||
return handleComboChat({
|
return handleComboChat({
|
||||||
@@ -129,7 +142,20 @@ async function handleSingleModelChat(body, modelStr, clientRawRequest = null, re
|
|||||||
const comboStrategies = chatSettings.comboStrategies || {};
|
const comboStrategies = chatSettings.comboStrategies || {};
|
||||||
const comboSpecificStrategy = comboStrategies[modelStr]?.fallbackStrategy;
|
const comboSpecificStrategy = comboStrategies[modelStr]?.fallbackStrategy;
|
||||||
const comboStrategy = comboSpecificStrategy || chatSettings.comboStrategy || "fallback";
|
const comboStrategy = comboSpecificStrategy || chatSettings.comboStrategy || "fallback";
|
||||||
|
|
||||||
|
if (comboStrategy === "fusion") {
|
||||||
|
log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models (strategy: fusion)`);
|
||||||
|
return handleFusionChat({
|
||||||
|
body,
|
||||||
|
models: comboModels,
|
||||||
|
handleSingleModel: (b, m) => handleSingleModelChat(b, m, clientRawRequest, request, apiKey),
|
||||||
|
log,
|
||||||
|
comboName: modelStr,
|
||||||
|
judgeModel: comboStrategies[modelStr]?.judgeModel,
|
||||||
|
tuning: comboStrategies[modelStr]?.fusionTuning,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
const comboStickyLimit = chatSettings.comboStickyRoundRobinLimit;
|
const comboStickyLimit = chatSettings.comboStickyRoundRobinLimit;
|
||||||
log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models (strategy: ${comboStrategy}, sticky: ${comboStickyLimit})`);
|
log.info("CHAT", `Combo "${modelStr}" with ${comboModels.length} models (strategy: ${comboStrategy}, sticky: ${comboStickyLimit})`);
|
||||||
return handleComboChat({
|
return handleComboChat({
|
||||||
|
|||||||
142
tests/unit/combo-fusion.test.js
Normal file
142
tests/unit/combo-fusion.test.js
Normal file
@@ -0,0 +1,142 @@
|
|||||||
|
import { describe, it, expect, vi } from "vitest";
|
||||||
|
|
||||||
|
import { handleFusionChat } from "../../open-sse/services/combo.js";
|
||||||
|
|
||||||
|
const log = { info: () => {}, warn: () => {}, debug: () => {} };
|
||||||
|
|
||||||
|
// Minimal OpenAI-chat Response stub with the .ok + .clone().json() surface the engine uses.
|
||||||
|
function okResponse(content, { delayMs = 0 } = {}) {
|
||||||
|
const json = { choices: [{ message: { role: "assistant", content } }] };
|
||||||
|
const make = () => ({ ok: true, status: 200, clone: make, json: async () => json });
|
||||||
|
const res = make();
|
||||||
|
return delayMs > 0 ? new Promise((r) => setTimeout(() => r(res), delayMs)) : res;
|
||||||
|
}
|
||||||
|
|
||||||
|
function errResponse(status = 500) {
|
||||||
|
const make = () => ({ ok: false, status, clone: make, json: async () => ({ error: { message: "boom" } }) });
|
||||||
|
return make();
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("fusion combo", () => {
|
||||||
|
it("answers directly with a single-model panel (nothing to fuse)", async () => {
|
||||||
|
const handleSingleModel = vi.fn(async () => okResponse("solo"));
|
||||||
|
await handleFusionChat({
|
||||||
|
body: { messages: [{ role: "user", content: "hi" }] },
|
||||||
|
models: ["p/only"],
|
||||||
|
handleSingleModel,
|
||||||
|
log,
|
||||||
|
});
|
||||||
|
expect(handleSingleModel).toHaveBeenCalledTimes(1);
|
||||||
|
expect(handleSingleModel.mock.calls[0][1]).toBe("p/only");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fans out to the panel then routes a synthesis turn to the judge", async () => {
|
||||||
|
const seen = [];
|
||||||
|
const handleSingleModel = vi.fn(async (body, model) => {
|
||||||
|
seen.push(model);
|
||||||
|
if (model === "p/judge") return okResponse("FINAL");
|
||||||
|
return okResponse(`ans-${model}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
const res = await handleFusionChat({
|
||||||
|
body: { messages: [{ role: "user", content: "Q" }], stream: true, tools: [{ name: "x" }] },
|
||||||
|
models: ["p/a", "p/b", "p/c"],
|
||||||
|
handleSingleModel,
|
||||||
|
log,
|
||||||
|
judgeModel: "p/judge",
|
||||||
|
});
|
||||||
|
|
||||||
|
// 3 panel calls + 1 judge call.
|
||||||
|
expect(handleSingleModel).toHaveBeenCalledTimes(4);
|
||||||
|
expect(seen.slice(0, 3).sort()).toEqual(["p/a", "p/b", "p/c"]);
|
||||||
|
expect(seen[3]).toBe("p/judge");
|
||||||
|
|
||||||
|
// Panel calls are non-streaming with tools stripped.
|
||||||
|
for (const [body, model] of handleSingleModel.mock.calls.filter(([, m]) => m !== "p/judge")) {
|
||||||
|
expect(body.stream).toBe(false);
|
||||||
|
expect(body.tools).toBeUndefined();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Judge call carries every panel answer + keeps the client's stream flag.
|
||||||
|
const [judgeBody] = handleSingleModel.mock.calls.find(([, m]) => m === "p/judge");
|
||||||
|
const judgeText = judgeBody.messages.at(-1).content;
|
||||||
|
expect(judgeText).toContain("ans-p/a");
|
||||||
|
expect(judgeText).toContain("ans-p/b");
|
||||||
|
expect(judgeText).toContain("ans-p/c");
|
||||||
|
expect(judgeText).toContain("Source 1");
|
||||||
|
expect(judgeBody.stream).toBe(true);
|
||||||
|
|
||||||
|
expect(res.ok).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("defaults the judge to the first panel model when none is set", async () => {
|
||||||
|
const seen = [];
|
||||||
|
const handleSingleModel = vi.fn(async (_body, model) => { seen.push(model); return okResponse(`ans-${model}`); });
|
||||||
|
await handleFusionChat({
|
||||||
|
body: { messages: [{ role: "user", content: "Q" }] },
|
||||||
|
models: ["p/first", "p/second"],
|
||||||
|
handleSingleModel,
|
||||||
|
log,
|
||||||
|
});
|
||||||
|
// Last call is the judge; defaults to panel[0].
|
||||||
|
expect(seen.at(-1)).toBe("p/first");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("proceeds on quorum without waiting for a straggler (grace window)", async () => {
|
||||||
|
const handleSingleModel = vi.fn(async (_body, model) => {
|
||||||
|
if (model === "p/slow") return okResponse("slow", { delayMs: 5000 });
|
||||||
|
if (model === "p/judge") return okResponse("FINAL");
|
||||||
|
return okResponse(`fast-${model}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
const t0 = Date.now();
|
||||||
|
await handleFusionChat({
|
||||||
|
body: { messages: [{ role: "user", content: "Q" }] },
|
||||||
|
models: ["p/x", "p/y", "p/slow"],
|
||||||
|
handleSingleModel,
|
||||||
|
log,
|
||||||
|
judgeModel: "p/judge",
|
||||||
|
tuning: { minPanel: 2, stragglerGraceMs: 50, panelHardTimeoutMs: 10000 },
|
||||||
|
});
|
||||||
|
const elapsed = Date.now() - t0;
|
||||||
|
|
||||||
|
// Two fast answers reach quorum; grace is 50ms, so we never wait ~5s for p/slow.
|
||||||
|
expect(elapsed).toBeLessThan(2000);
|
||||||
|
|
||||||
|
const judgeCall = handleSingleModel.mock.calls.find(([, m]) => m === "p/judge");
|
||||||
|
const judgeText = judgeCall[0].messages.at(-1).content;
|
||||||
|
expect(judgeText).toContain("fast-p/x");
|
||||||
|
expect(judgeText).toContain("fast-p/y");
|
||||||
|
expect(judgeText).not.toContain("slow");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns the lone survivor directly when only one panel model succeeds", async () => {
|
||||||
|
const handleSingleModel = vi.fn(async (_body, model) => {
|
||||||
|
if (model === "p/ok") return okResponse("lone");
|
||||||
|
return errResponse(500);
|
||||||
|
});
|
||||||
|
await handleFusionChat({
|
||||||
|
body: { messages: [{ role: "user", content: "Q" }] },
|
||||||
|
models: ["p/ok", "p/bad"],
|
||||||
|
handleSingleModel,
|
||||||
|
log,
|
||||||
|
judgeModel: "p/judge",
|
||||||
|
tuning: { minPanel: 2, stragglerGraceMs: 50, panelHardTimeoutMs: 5000 },
|
||||||
|
});
|
||||||
|
// No judge call — single answer means there is nothing to fuse.
|
||||||
|
const judged = handleSingleModel.mock.calls.some(([, m]) => m === "p/judge");
|
||||||
|
expect(judged).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns 503 when the whole panel fails", async () => {
|
||||||
|
const handleSingleModel = vi.fn(async () => errResponse(500));
|
||||||
|
const res = await handleFusionChat({
|
||||||
|
body: { messages: [{ role: "user", content: "Q" }] },
|
||||||
|
models: ["p/a", "p/b"],
|
||||||
|
handleSingleModel,
|
||||||
|
log,
|
||||||
|
tuning: { minPanel: 2, stragglerGraceMs: 50, panelHardTimeoutMs: 5000 },
|
||||||
|
});
|
||||||
|
expect(res.status).toBe(503);
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user