feat: implement model lock functionality for connection management
This commit is contained in:
@@ -138,6 +138,64 @@ export function formatRetryAfter(rateLimitedUntil) {
|
|||||||
return `reset after ${parts.join(" ")}`;
|
return `reset after ${parts.join(" ")}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Prefix for model lock flat fields on connection record */
|
||||||
|
export const MODEL_LOCK_PREFIX = "modelLock_";
|
||||||
|
|
||||||
|
/** Special key used when no model is known (account-level lock) */
|
||||||
|
export const MODEL_LOCK_ALL = `${MODEL_LOCK_PREFIX}__all`;
|
||||||
|
|
||||||
|
/** Build the flat field key for a model lock */
|
||||||
|
export function getModelLockKey(model) {
|
||||||
|
return model ? `${MODEL_LOCK_PREFIX}${model}` : MODEL_LOCK_ALL;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if a model lock on a connection is still active.
|
||||||
|
* Reads flat field `modelLock_${model}` (or `modelLock___all` when model=null).
|
||||||
|
*/
|
||||||
|
export function isModelLockActive(connection, model) {
|
||||||
|
const key = getModelLockKey(model);
|
||||||
|
const expiry = connection[key] || connection[MODEL_LOCK_ALL];
|
||||||
|
if (!expiry) return false;
|
||||||
|
return new Date(expiry).getTime() > Date.now();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get earliest active model lock expiry across all modelLock_* fields.
|
||||||
|
* Used for UI cooldown display.
|
||||||
|
*/
|
||||||
|
export function getEarliestModelLockUntil(connection) {
|
||||||
|
if (!connection) return null;
|
||||||
|
let earliest = null;
|
||||||
|
const now = Date.now();
|
||||||
|
for (const [key, val] of Object.entries(connection)) {
|
||||||
|
if (!key.startsWith(MODEL_LOCK_PREFIX) || !val) continue;
|
||||||
|
const t = new Date(val).getTime();
|
||||||
|
if (t <= now) continue;
|
||||||
|
if (!earliest || t < earliest) earliest = t;
|
||||||
|
}
|
||||||
|
return earliest ? new Date(earliest).toISOString() : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build update object to set a model lock on a connection.
|
||||||
|
*/
|
||||||
|
export function buildModelLockUpdate(model, cooldownMs) {
|
||||||
|
const key = getModelLockKey(model);
|
||||||
|
return { [key]: new Date(Date.now() + cooldownMs).toISOString() };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build update object to clear all model locks on a connection.
|
||||||
|
*/
|
||||||
|
export function buildClearModelLocksUpdate(connection) {
|
||||||
|
const cleared = {};
|
||||||
|
for (const key of Object.keys(connection)) {
|
||||||
|
if (key.startsWith(MODEL_LOCK_PREFIX)) cleared[key] = null;
|
||||||
|
}
|
||||||
|
return cleared;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Filter available accounts (not in cooldown)
|
* Filter available accounts (not in cooldown)
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -31,10 +31,7 @@ export default function CombosPage() {
|
|||||||
|
|
||||||
if (combosRes.ok) setCombos(combosData.combos || []);
|
if (combosRes.ok) setCombos(combosData.combos || []);
|
||||||
if (providersRes.ok) {
|
if (providersRes.ok) {
|
||||||
const active = (providersData.connections || []).filter(
|
setActiveProviders(providersData.connections || []);
|
||||||
c => c.testStatus === "active" || c.testStatus === "success"
|
|
||||||
);
|
|
||||||
setActiveProviders(active);
|
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.log("Error fetching data:", error);
|
console.log("Error fetching data:", error);
|
||||||
@@ -228,6 +225,80 @@ function ComboCard({ combo, copied, onCopy, onEdit, onDelete }) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Inline editable model item
|
||||||
|
function ModelItem({ index, model, isFirst, isLast, onEdit, onMoveUp, onMoveDown, onRemove }) {
|
||||||
|
const [editing, setEditing] = useState(false);
|
||||||
|
const [draft, setDraft] = useState(model);
|
||||||
|
|
||||||
|
const commit = () => {
|
||||||
|
const trimmed = draft.trim();
|
||||||
|
if (trimmed && trimmed !== model) onEdit(trimmed);
|
||||||
|
else setDraft(model); // revert if empty or unchanged
|
||||||
|
setEditing(false);
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleKeyDown = (e) => {
|
||||||
|
if (e.key === "Enter") commit();
|
||||||
|
if (e.key === "Escape") { setDraft(model); setEditing(false); }
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="group flex items-center gap-1.5 px-2 py-1 rounded-md bg-black/[0.02] dark:bg-white/[0.02] hover:bg-black/[0.04] dark:hover:bg-white/[0.04] transition-colors">
|
||||||
|
{/* Index badge */}
|
||||||
|
<span className="text-[10px] font-medium text-text-muted w-3 text-center shrink-0">{index + 1}</span>
|
||||||
|
|
||||||
|
{/* Inline editable model value */}
|
||||||
|
{editing ? (
|
||||||
|
<input
|
||||||
|
autoFocus
|
||||||
|
value={draft}
|
||||||
|
onChange={(e) => setDraft(e.target.value)}
|
||||||
|
onBlur={commit}
|
||||||
|
onKeyDown={handleKeyDown}
|
||||||
|
className="flex-1 min-w-0 px-1.5 py-0.5 text-xs font-mono bg-white dark:bg-black/20 border border-primary/40 rounded outline-none text-text-main"
|
||||||
|
/>
|
||||||
|
) : (
|
||||||
|
<div
|
||||||
|
className="flex-1 min-w-0 px-1.5 py-0.5 text-xs font-mono text-text-main truncate cursor-text hover:bg-black/5 dark:hover:bg-white/5 rounded"
|
||||||
|
onClick={() => setEditing(true)}
|
||||||
|
title="Click to edit"
|
||||||
|
>
|
||||||
|
{model}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Priority arrows */}
|
||||||
|
<div className="flex items-center gap-0.5">
|
||||||
|
<button
|
||||||
|
onClick={onMoveUp}
|
||||||
|
disabled={isFirst}
|
||||||
|
className={`p-0.5 rounded ${isFirst ? "text-text-muted/20 cursor-not-allowed" : "text-text-muted hover:text-primary hover:bg-black/5 dark:hover:bg-white/5"}`}
|
||||||
|
title="Move up"
|
||||||
|
>
|
||||||
|
<span className="material-symbols-outlined text-[12px]">arrow_upward</span>
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
onClick={onMoveDown}
|
||||||
|
disabled={isLast}
|
||||||
|
className={`p-0.5 rounded ${isLast ? "text-text-muted/20 cursor-not-allowed" : "text-text-muted hover:text-primary hover:bg-black/5 dark:hover:bg-white/5"}`}
|
||||||
|
title="Move down"
|
||||||
|
>
|
||||||
|
<span className="material-symbols-outlined text-[12px]">arrow_downward</span>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Remove */}
|
||||||
|
<button
|
||||||
|
onClick={onRemove}
|
||||||
|
className="p-0.5 hover:bg-red-500/10 rounded text-text-muted hover:text-red-500 transition-all"
|
||||||
|
title="Remove"
|
||||||
|
>
|
||||||
|
<span className="material-symbols-outlined text-[12px]">close</span>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) {
|
function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) {
|
||||||
// Initialize state with combo values - key prop on parent handles reset on remount
|
// Initialize state with combo values - key prop on parent handles reset on remount
|
||||||
const [name, setName] = useState(combo?.name || "");
|
const [name, setName] = useState(combo?.name || "");
|
||||||
@@ -236,25 +307,13 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) {
|
|||||||
const [saving, setSaving] = useState(false);
|
const [saving, setSaving] = useState(false);
|
||||||
const [nameError, setNameError] = useState("");
|
const [nameError, setNameError] = useState("");
|
||||||
const [modelAliases, setModelAliases] = useState({});
|
const [modelAliases, setModelAliases] = useState({});
|
||||||
const [providerNodes, setProviderNodes] = useState([]);
|
|
||||||
|
|
||||||
const fetchModalData = async () => {
|
const fetchModalData = async () => {
|
||||||
try {
|
try {
|
||||||
const [aliasesRes, nodesRes] = await Promise.all([
|
const aliasesRes = await fetch("/api/models/alias");
|
||||||
fetch("/api/models/alias"),
|
if (!aliasesRes.ok) return;
|
||||||
fetch("/api/provider-nodes"),
|
const aliasesData = await aliasesRes.json();
|
||||||
]);
|
|
||||||
|
|
||||||
if (!aliasesRes.ok || !nodesRes.ok) {
|
|
||||||
throw new Error(`Failed to fetch data: aliases=${aliasesRes.status}, nodes=${nodesRes.status}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const [aliasesData, nodesData] = await Promise.all([
|
|
||||||
aliasesRes.json(),
|
|
||||||
nodesRes.json(),
|
|
||||||
]);
|
|
||||||
setModelAliases(aliasesData.aliases || {});
|
setModelAliases(aliasesData.aliases || {});
|
||||||
setProviderNodes(nodesData.nodes || []);
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Error fetching modal data:", error);
|
console.error("Error fetching modal data:", error);
|
||||||
}
|
}
|
||||||
@@ -294,21 +353,6 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) {
|
|||||||
setModels(models.filter((_, i) => i !== index));
|
setModels(models.filter((_, i) => i !== index));
|
||||||
};
|
};
|
||||||
|
|
||||||
// Format model display name with readable provider name
|
|
||||||
const formatModelDisplay = useCallback((modelValue) => {
|
|
||||||
const parts = modelValue.split('/');
|
|
||||||
if (parts.length !== 2) return modelValue;
|
|
||||||
|
|
||||||
const [providerId, modelId] = parts;
|
|
||||||
const matchedNode = providerNodes.find(node => node.id === providerId);
|
|
||||||
|
|
||||||
if (matchedNode) {
|
|
||||||
return `${matchedNode.name}/${modelId}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
return modelValue;
|
|
||||||
}, [providerNodes]);
|
|
||||||
|
|
||||||
const handleMoveUp = (index) => {
|
const handleMoveUp = (index) => {
|
||||||
if (index === 0) return;
|
if (index === 0) return;
|
||||||
const newModels = [...models];
|
const newModels = [...models];
|
||||||
@@ -366,52 +410,26 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) {
|
|||||||
) : (
|
) : (
|
||||||
<div className="flex flex-col gap-1 max-h-[200px] overflow-y-auto">
|
<div className="flex flex-col gap-1 max-h-[200px] overflow-y-auto">
|
||||||
{models.map((model, index) => (
|
{models.map((model, index) => (
|
||||||
<div
|
<ModelItem
|
||||||
key={index}
|
key={index}
|
||||||
className="group flex items-center gap-1.5 px-2 py-1 rounded-md bg-black/[0.02] dark:bg-white/[0.02] hover:bg-black/[0.04] dark:hover:bg-white/[0.04] transition-colors"
|
index={index}
|
||||||
>
|
model={model}
|
||||||
{/* Index badge */}
|
isFirst={index === 0}
|
||||||
<span className="text-[10px] font-medium text-text-muted w-3 text-center shrink-0">{index + 1}</span>
|
isLast={index === models.length - 1}
|
||||||
|
onEdit={(newVal) => {
|
||||||
{/* Model display - show readable name only */}
|
const updated = [...models];
|
||||||
<div className="flex-1 min-w-0 px-1.5 py-0.5 text-xs text-text-main truncate">
|
updated[index] = newVal;
|
||||||
{formatModelDisplay(model)}
|
setModels(updated);
|
||||||
</div>
|
}}
|
||||||
|
onMoveUp={() => handleMoveUp(index)}
|
||||||
{/* Priority arrows - horizontal, always visible */}
|
onMoveDown={() => handleMoveDown(index)}
|
||||||
<div className="flex items-center gap-0.5">
|
onRemove={() => handleRemoveModel(index)}
|
||||||
<button
|
/>
|
||||||
onClick={() => handleMoveUp(index)}
|
|
||||||
disabled={index === 0}
|
|
||||||
className={`p-0.5 rounded ${index === 0 ? "text-text-muted/20 cursor-not-allowed" : "text-text-muted hover:text-primary hover:bg-black/5 dark:hover:bg-white/5"}`}
|
|
||||||
title="Move up"
|
|
||||||
>
|
|
||||||
<span className="material-symbols-outlined text-[12px]">arrow_upward</span>
|
|
||||||
</button>
|
|
||||||
<button
|
|
||||||
onClick={() => handleMoveDown(index)}
|
|
||||||
disabled={index === models.length - 1}
|
|
||||||
className={`p-0.5 rounded ${index === models.length - 1 ? "text-text-muted/20 cursor-not-allowed" : "text-text-muted hover:text-primary hover:bg-black/5 dark:hover:bg-white/5"}`}
|
|
||||||
title="Move down"
|
|
||||||
>
|
|
||||||
<span className="material-symbols-outlined text-[12px]">arrow_downward</span>
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{/* Remove - always visible */}
|
|
||||||
<button
|
|
||||||
onClick={() => handleRemoveModel(index)}
|
|
||||||
className="p-0.5 hover:bg-red-500/10 rounded text-text-muted hover:text-red-500 transition-all"
|
|
||||||
title="Remove"
|
|
||||||
>
|
|
||||||
<span className="material-symbols-outlined text-[12px]">close</span>
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{/* Add Model button - moved to bottom */}
|
{/* Add Model button */}
|
||||||
<button
|
<button
|
||||||
onClick={() => setShowModelSelect(true)}
|
onClick={() => setShowModelSelect(true)}
|
||||||
className="w-full mt-2 py-2 border border-dashed border-black/10 dark:border-white/10 rounded-lg text-xs text-text-muted hover:text-primary hover:border-primary/30 transition-colors flex items-center justify-center gap-1"
|
className="w-full mt-2 py-2 border border-dashed border-black/10 dark:border-white/10 rounded-lg text-xs text-text-muted hover:text-primary hover:border-primary/30 transition-colors flex items-center justify-center gap-1"
|
||||||
@@ -450,4 +468,3 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) {
|
|||||||
</>
|
</>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -919,20 +919,29 @@ function ConnectionRow({ connection, isOAuth, isFirst, isLast, onMoveUp, onMoveD
|
|||||||
// Use useState + useEffect for impure Date.now() to avoid calling during render
|
// Use useState + useEffect for impure Date.now() to avoid calling during render
|
||||||
const [isCooldown, setIsCooldown] = useState(false);
|
const [isCooldown, setIsCooldown] = useState(false);
|
||||||
|
|
||||||
|
// Get earliest model lock timestamp (useEffect handles the Date.now() comparison)
|
||||||
|
const modelLockUntil = Object.entries(connection)
|
||||||
|
.filter(([k]) => k.startsWith("modelLock_"))
|
||||||
|
.map(([, v]) => v)
|
||||||
|
.filter(v => !!v)
|
||||||
|
.sort()[0] || null;
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const checkCooldown = () => {
|
const checkCooldown = () => {
|
||||||
const cooldown = connection.rateLimitedUntil &&
|
const until = Object.entries(connection)
|
||||||
new Date(connection.rateLimitedUntil).getTime() > Date.now();
|
.filter(([k]) => k.startsWith("modelLock_"))
|
||||||
setIsCooldown(cooldown);
|
.map(([, v]) => v)
|
||||||
|
.filter(v => v && new Date(v).getTime() > Date.now())
|
||||||
|
.sort()[0] || null;
|
||||||
|
setIsCooldown(!!until);
|
||||||
};
|
};
|
||||||
|
|
||||||
checkCooldown();
|
checkCooldown();
|
||||||
// Update every second while in cooldown
|
const interval = modelLockUntil ? setInterval(checkCooldown, 1000) : null;
|
||||||
const interval = connection.rateLimitedUntil ? setInterval(checkCooldown, 1000) : null;
|
|
||||||
return () => {
|
return () => {
|
||||||
if (interval) clearInterval(interval);
|
if (interval) clearInterval(interval);
|
||||||
};
|
};
|
||||||
}, [connection.rateLimitedUntil]);
|
}, [modelLockUntil]);
|
||||||
|
|
||||||
// Determine effective status (override unavailable if cooldown expired)
|
// Determine effective status (override unavailable if cooldown expired)
|
||||||
const effectiveStatus = (connection.testStatus === "unavailable" && !isCooldown)
|
const effectiveStatus = (connection.testStatus === "unavailable" && !isCooldown)
|
||||||
@@ -975,7 +984,7 @@ function ConnectionRow({ connection, isOAuth, isFirst, isLast, onMoveUp, onMoveD
|
|||||||
<Badge variant={getStatusVariant()} size="sm" dot>
|
<Badge variant={getStatusVariant()} size="sm" dot>
|
||||||
{connection.isActive === false ? "disabled" : (effectiveStatus || "Unknown")}
|
{connection.isActive === false ? "disabled" : (effectiveStatus || "Unknown")}
|
||||||
</Badge>
|
</Badge>
|
||||||
{isCooldown && connection.isActive !== false && <CooldownTimer until={connection.rateLimitedUntil} />}
|
{isCooldown && connection.isActive !== false && <CooldownTimer until={modelLockUntil} />}
|
||||||
{connection.lastError && connection.isActive !== false && (
|
{connection.lastError && connection.isActive !== false && (
|
||||||
<span className="text-xs text-red-500 truncate max-w-[300px]" title={connection.lastError}>
|
<span className="text-xs text-red-500 truncate max-w-[300px]" title={connection.lastError}>
|
||||||
{connection.lastError}
|
{connection.lastError}
|
||||||
@@ -1014,7 +1023,7 @@ ConnectionRow.propTypes = {
|
|||||||
name: PropTypes.string,
|
name: PropTypes.string,
|
||||||
email: PropTypes.string,
|
email: PropTypes.string,
|
||||||
displayName: PropTypes.string,
|
displayName: PropTypes.string,
|
||||||
rateLimitedUntil: PropTypes.string,
|
modelLockUntil: PropTypes.string,
|
||||||
testStatus: PropTypes.string,
|
testStatus: PropTypes.string,
|
||||||
isActive: PropTypes.bool,
|
isActive: PropTypes.bool,
|
||||||
lastError: PropTypes.string,
|
lastError: PropTypes.string,
|
||||||
|
|||||||
@@ -1144,20 +1144,28 @@ function ConnectionRow({ connection, isOAuth, isFirst, isLast, onMoveUp, onMoveD
|
|||||||
// Use useState + useEffect for impure Date.now() to avoid calling during render
|
// Use useState + useEffect for impure Date.now() to avoid calling during render
|
||||||
const [isCooldown, setIsCooldown] = useState(false);
|
const [isCooldown, setIsCooldown] = useState(false);
|
||||||
|
|
||||||
|
const modelLockUntil = Object.entries(connection)
|
||||||
|
.filter(([k]) => k.startsWith("modelLock_"))
|
||||||
|
.map(([, v]) => v)
|
||||||
|
.filter(v => v && new Date(v).getTime() > Date.now())
|
||||||
|
.sort()[0] || null;
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const checkCooldown = () => {
|
const checkCooldown = () => {
|
||||||
const cooldown = connection.rateLimitedUntil &&
|
const until = Object.entries(connection)
|
||||||
new Date(connection.rateLimitedUntil).getTime() > Date.now();
|
.filter(([k]) => k.startsWith("modelLock_"))
|
||||||
setIsCooldown(cooldown);
|
.map(([, v]) => v)
|
||||||
|
.filter(v => v && new Date(v).getTime() > Date.now())
|
||||||
|
.sort()[0] || null;
|
||||||
|
setIsCooldown(!!until);
|
||||||
};
|
};
|
||||||
|
|
||||||
checkCooldown();
|
checkCooldown();
|
||||||
// Update every second while in cooldown
|
const interval = modelLockUntil ? setInterval(checkCooldown, 1000) : null;
|
||||||
const interval = connection.rateLimitedUntil ? setInterval(checkCooldown, 1000) : null;
|
|
||||||
return () => {
|
return () => {
|
||||||
if (interval) clearInterval(interval);
|
if (interval) clearInterval(interval);
|
||||||
};
|
};
|
||||||
}, [connection.rateLimitedUntil]);
|
}, [modelLockUntil]);
|
||||||
|
|
||||||
// Determine effective status (override unavailable if cooldown expired)
|
// Determine effective status (override unavailable if cooldown expired)
|
||||||
const effectiveStatus = (connection.testStatus === "unavailable" && !isCooldown)
|
const effectiveStatus = (connection.testStatus === "unavailable" && !isCooldown)
|
||||||
@@ -1200,7 +1208,7 @@ function ConnectionRow({ connection, isOAuth, isFirst, isLast, onMoveUp, onMoveD
|
|||||||
<Badge variant={getStatusVariant()} size="sm" dot>
|
<Badge variant={getStatusVariant()} size="sm" dot>
|
||||||
{connection.isActive === false ? "disabled" : (effectiveStatus || "Unknown")}
|
{connection.isActive === false ? "disabled" : (effectiveStatus || "Unknown")}
|
||||||
</Badge>
|
</Badge>
|
||||||
{isCooldown && connection.isActive !== false && <CooldownTimer until={connection.rateLimitedUntil} />}
|
{isCooldown && connection.isActive !== false && <CooldownTimer until={modelLockUntil} />}
|
||||||
{connection.lastError && connection.isActive !== false && (
|
{connection.lastError && connection.isActive !== false && (
|
||||||
<span className="text-xs text-red-500 truncate max-w-[300px]" title={connection.lastError}>
|
<span className="text-xs text-red-500 truncate max-w-[300px]" title={connection.lastError}>
|
||||||
{connection.lastError}
|
{connection.lastError}
|
||||||
@@ -1239,7 +1247,7 @@ ConnectionRow.propTypes = {
|
|||||||
name: PropTypes.string,
|
name: PropTypes.string,
|
||||||
email: PropTypes.string,
|
email: PropTypes.string,
|
||||||
displayName: PropTypes.string,
|
displayName: PropTypes.string,
|
||||||
rateLimitedUntil: PropTypes.string,
|
modelLockUntil: PropTypes.string,
|
||||||
testStatus: PropTypes.string,
|
testStatus: PropTypes.string,
|
||||||
isActive: PropTypes.bool,
|
isActive: PropTypes.bool,
|
||||||
lastError: PropTypes.string,
|
lastError: PropTypes.string,
|
||||||
|
|||||||
@@ -99,7 +99,8 @@ export default function ProvidersPage() {
|
|||||||
);
|
);
|
||||||
|
|
||||||
const getEffectiveStatus = (conn) => {
|
const getEffectiveStatus = (conn) => {
|
||||||
const isCooldown = conn.rateLimitedUntil && new Date(conn.rateLimitedUntil).getTime() > Date.now();
|
const isCooldown = Object.entries(conn)
|
||||||
|
.some(([k, v]) => k.startsWith("modelLock_") && v && new Date(v).getTime() > Date.now());
|
||||||
return conn.testStatus === "unavailable" && !isCooldown ? "active" : conn.testStatus;
|
return conn.testStatus === "unavailable" && !isCooldown ? "active" : conn.testStatus;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -1,67 +1,11 @@
|
|||||||
import { getProviderConnections, validateApiKey, updateProviderConnection, getSettings } from "@/lib/localDb";
|
import { getProviderConnections, validateApiKey, updateProviderConnection, getSettings } from "@/lib/localDb";
|
||||||
import { isAccountUnavailable, getUnavailableUntil, getEarliestRateLimitedUntil, formatRetryAfter, checkFallbackError } from "open-sse/services/accountFallback.js";
|
import { formatRetryAfter, checkFallbackError, isModelLockActive, buildModelLockUpdate, buildClearModelLocksUpdate, getEarliestModelLockUntil } from "open-sse/services/accountFallback.js";
|
||||||
import { resolveProviderId } from "@/shared/constants/providers.js";
|
import { resolveProviderId } from "@/shared/constants/providers.js";
|
||||||
import * as log from "../utils/logger.js";
|
import * as log from "../utils/logger.js";
|
||||||
|
|
||||||
// Mutex to prevent race conditions during account selection
|
// Mutex to prevent race conditions during account selection
|
||||||
let selectionMutex = Promise.resolve();
|
let selectionMutex = Promise.resolve();
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Model-level rate-limit locking (in-memory)
|
|
||||||
//
|
|
||||||
// Providers like Antigravity maintain *separate* quota buckets per model
|
|
||||||
// family (e.g. Claude vs Gemini). When a 429 arrives for one model, the
|
|
||||||
// standard account-level DB lock would block ALL models on that account,
|
|
||||||
// wasting quota that is still available for other model families.
|
|
||||||
//
|
|
||||||
// This module tracks model-specific locks in memory so that only the
|
|
||||||
// affected model is skipped during account selection while the rest of the
|
|
||||||
// account's quota remains accessible.
|
|
||||||
//
|
|
||||||
// Locks are intentionally in-memory: they clear on restart and require no
|
|
||||||
// database schema migration.
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
/** Providers known to have independent per-model quota buckets */
|
|
||||||
const MULTI_BUCKET_PROVIDERS = new Set(["antigravity"]);
|
|
||||||
|
|
||||||
/** Map<"connectionId:model", expiryTimestamp> */
|
|
||||||
const modelLocks = new Map();
|
|
||||||
|
|
||||||
/** Default lock duration for model-level rate limits (5 minutes) */
|
|
||||||
const DEFAULT_MODEL_LOCK_MS = 5 * 60 * 1000;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check whether a specific model is temporarily locked on a connection.
|
|
||||||
* Expired locks are cleaned up lazily.
|
|
||||||
*/
|
|
||||||
function isModelLocked(connectionId, model) {
|
|
||||||
if (!connectionId || !model) return false;
|
|
||||||
const key = `${connectionId}:${model}`;
|
|
||||||
const expiry = modelLocks.get(key);
|
|
||||||
if (!expiry) return false;
|
|
||||||
if (expiry > Date.now()) return true;
|
|
||||||
modelLocks.delete(key); // clean up expired
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Lock a model on a specific connection for `durationMs` milliseconds.
|
|
||||||
*/
|
|
||||||
function lockModel(connectionId, model, durationMs) {
|
|
||||||
if (!connectionId || !model) return;
|
|
||||||
const key = `${connectionId}:${model}`;
|
|
||||||
modelLocks.set(key, Date.now() + durationMs);
|
|
||||||
log.warn("AUTH", `Model lock: ${model} on ${connectionId.slice(0, 8)} for ${Math.round(durationMs / 1000)}s`);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check whether a provider uses per-model quota buckets.
|
|
||||||
*/
|
|
||||||
function isMultiBucketProvider(provider) {
|
|
||||||
return MULTI_BUCKET_PROVIDERS.has(provider);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get provider credentials from localDb
|
* Get provider credentials from localDb
|
||||||
* Filters out unavailable accounts and returns the selected account based on strategy
|
* Filters out unavailable accounts and returns the selected account based on strategy
|
||||||
@@ -85,50 +29,35 @@ export async function getProviderCredentials(provider, excludeConnectionId = nul
|
|||||||
log.debug("AUTH", `${provider} | total connections: ${connections.length}, excludeId: ${excludeConnectionId || "none"}, model: ${model || "any"}`);
|
log.debug("AUTH", `${provider} | total connections: ${connections.length}, excludeId: ${excludeConnectionId || "none"}, model: ${model || "any"}`);
|
||||||
|
|
||||||
if (connections.length === 0) {
|
if (connections.length === 0) {
|
||||||
// Check all connections (including inactive) to see if rate limited
|
|
||||||
const allConnections = await getProviderConnections({ provider: providerId });
|
|
||||||
log.debug("AUTH", `${provider} | all connections (incl inactive): ${allConnections.length}`);
|
|
||||||
if (allConnections.length > 0) {
|
|
||||||
const earliest = getEarliestRateLimitedUntil(allConnections);
|
|
||||||
if (earliest) {
|
|
||||||
log.warn("AUTH", `${provider} | all ${allConnections.length} accounts rate limited (${formatRetryAfter(earliest)})`);
|
|
||||||
return { allRateLimited: true, retryAfter: earliest, retryAfterHuman: formatRetryAfter(earliest) };
|
|
||||||
}
|
|
||||||
log.warn("AUTH", `${provider} | ${allConnections.length} accounts found but none active`);
|
|
||||||
allConnections.forEach(c => {
|
|
||||||
log.debug("AUTH", ` → ${c.id?.slice(0, 8)} | isActive=${c.isActive} | rateLimitedUntil=${c.rateLimitedUntil || "none"} | testStatus=${c.testStatus}`);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
log.warn("AUTH", `No credentials for ${provider}`);
|
log.warn("AUTH", `No credentials for ${provider}`);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Filter out unavailable accounts and excluded connection
|
// Filter out model-locked and excluded connections
|
||||||
const multiBucket = isMultiBucketProvider(provider);
|
|
||||||
const availableConnections = connections.filter(c => {
|
const availableConnections = connections.filter(c => {
|
||||||
if (excludeConnectionId && c.id === excludeConnectionId) return false;
|
if (excludeConnectionId && c.id === excludeConnectionId) return false;
|
||||||
if (isAccountUnavailable(c.rateLimitedUntil)) return false;
|
if (isModelLockActive(c, model)) return false;
|
||||||
if (multiBucket && model && isModelLocked(c.id, model)) return false;
|
|
||||||
return true;
|
return true;
|
||||||
});
|
});
|
||||||
|
|
||||||
log.debug("AUTH", `${provider} | available: ${availableConnections.length}/${connections.length}`);
|
log.debug("AUTH", `${provider} | available: ${availableConnections.length}/${connections.length}`);
|
||||||
connections.forEach(c => {
|
connections.forEach(c => {
|
||||||
const excluded = excludeConnectionId && c.id === excludeConnectionId;
|
const excluded = excludeConnectionId && c.id === excludeConnectionId;
|
||||||
const rateLimited = isAccountUnavailable(c.rateLimitedUntil);
|
const locked = isModelLockActive(c, model);
|
||||||
const modelLocked = multiBucket && model && isModelLocked(c.id, model);
|
if (excluded || locked) {
|
||||||
if (excluded || rateLimited || modelLocked) {
|
const lockUntil = getEarliestModelLockUntil(c);
|
||||||
log.debug("AUTH", ` → ${c.id?.slice(0, 8)} | ${excluded ? "excluded" : ""} ${rateLimited ? `rateLimited until ${c.rateLimitedUntil}` : ""} ${modelLocked ? `modelLocked(${model})` : ""}`);
|
log.debug("AUTH", ` → ${c.id?.slice(0, 8)} | ${excluded ? "excluded" : ""} ${locked ? `modelLocked(${model}) until ${lockUntil}` : ""}`);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
if (availableConnections.length === 0) {
|
if (availableConnections.length === 0) {
|
||||||
const earliest = getEarliestRateLimitedUntil(connections);
|
// Find earliest lock expiry across all connections for retry timing
|
||||||
|
const lockedConns = connections.filter(c => isModelLockActive(c, model));
|
||||||
|
const expiries = lockedConns.map(c => getEarliestModelLockUntil(c)).filter(Boolean);
|
||||||
|
const earliest = expiries.sort()[0] || null;
|
||||||
if (earliest) {
|
if (earliest) {
|
||||||
// Find the connection with the earliest rateLimitedUntil to get its error info
|
const earliestConn = lockedConns[0];
|
||||||
const rateLimitedConns = connections.filter(c => c.rateLimitedUntil && new Date(c.rateLimitedUntil).getTime() > Date.now());
|
log.warn("AUTH", `${provider} | all ${connections.length} accounts locked for ${model || "all"} (${formatRetryAfter(earliest)}) | lastError=${earliestConn?.lastError?.slice(0, 50)}`);
|
||||||
const earliestConn = rateLimitedConns.sort((a, b) => new Date(a.rateLimitedUntil) - new Date(b.rateLimitedUntil))[0];
|
|
||||||
log.warn("AUTH", `${provider} | all ${connections.length} active accounts rate limited (${formatRetryAfter(earliest)}) | lastErrorCode=${earliestConn?.errorCode}, lastError=${earliestConn?.lastError?.slice(0, 50)}`);
|
|
||||||
return {
|
return {
|
||||||
allRateLimited: true,
|
allRateLimited: true,
|
||||||
retryAfter: earliest,
|
retryAfter: earliest,
|
||||||
@@ -137,15 +66,6 @@ export async function getProviderCredentials(provider, excludeConnectionId = nul
|
|||||||
lastErrorCode: earliestConn?.errorCode || null
|
lastErrorCode: earliestConn?.errorCode || null
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
if (multiBucket && model) {
|
|
||||||
log.warn("AUTH", `${provider} | all accounts model-locked for ${model}`);
|
|
||||||
return {
|
|
||||||
allRateLimited: true,
|
|
||||||
retryAfter: new Date(Date.now() + 60000).toISOString(),
|
|
||||||
retryAfterHuman: "reset after 1m",
|
|
||||||
lastError: `All accounts rate limited for model ${model}`
|
|
||||||
};
|
|
||||||
}
|
|
||||||
log.warn("AUTH", `${provider} | all ${connections.length} accounts unavailable`);
|
log.warn("AUTH", `${provider} | all ${connections.length} accounts unavailable`);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@@ -209,7 +129,8 @@ export async function getProviderCredentials(provider, excludeConnectionId = nul
|
|||||||
// Include current status for optimization check
|
// Include current status for optimization check
|
||||||
testStatus: connection.testStatus,
|
testStatus: connection.testStatus,
|
||||||
lastError: connection.lastError,
|
lastError: connection.lastError,
|
||||||
rateLimitedUntil: connection.rateLimitedUntil
|
// Pass full connection for clearAccountError to read modelLock_* keys
|
||||||
|
_connection: connection
|
||||||
};
|
};
|
||||||
} finally {
|
} finally {
|
||||||
if (resolveMutex) resolveMutex();
|
if (resolveMutex) resolveMutex();
|
||||||
@@ -217,9 +138,8 @@ export async function getProviderCredentials(provider, excludeConnectionId = nul
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Mark account as unavailable — reads backoffLevel from DB, calculates cooldown with exponential backoff, saves new level.
|
* Mark account+model as unavailable — locks modelLock_${model} in DB.
|
||||||
* For multi-bucket providers (e.g. Antigravity), 429 errors lock only the specific model in memory
|
* All errors (429, 401, 5xx, etc.) lock per model, not per account.
|
||||||
* rather than the entire account in the database.
|
|
||||||
* @param {string} connectionId
|
* @param {string} connectionId
|
||||||
* @param {number} status - HTTP status code from upstream
|
* @param {number} status - HTTP status code from upstream
|
||||||
* @param {string} errorText
|
* @param {string} errorText
|
||||||
@@ -228,7 +148,6 @@ export async function getProviderCredentials(provider, excludeConnectionId = nul
|
|||||||
* @returns {{ shouldFallback: boolean, cooldownMs: number }}
|
* @returns {{ shouldFallback: boolean, cooldownMs: number }}
|
||||||
*/
|
*/
|
||||||
export async function markAccountUnavailable(connectionId, status, errorText, provider = null, model = null) {
|
export async function markAccountUnavailable(connectionId, status, errorText, provider = null, model = null) {
|
||||||
// Read current connection to get backoffLevel
|
|
||||||
const connections = await getProviderConnections({ provider });
|
const connections = await getProviderConnections({ provider });
|
||||||
const conn = connections.find(c => c.id === connectionId);
|
const conn = connections.find(c => c.id === connectionId);
|
||||||
const backoffLevel = conn?.backoffLevel || 0;
|
const backoffLevel = conn?.backoffLevel || 0;
|
||||||
@@ -236,17 +155,11 @@ export async function markAccountUnavailable(connectionId, status, errorText, pr
|
|||||||
const { shouldFallback, cooldownMs, newBackoffLevel } = checkFallbackError(status, errorText, backoffLevel);
|
const { shouldFallback, cooldownMs, newBackoffLevel } = checkFallbackError(status, errorText, backoffLevel);
|
||||||
if (!shouldFallback) return { shouldFallback: false, cooldownMs: 0 };
|
if (!shouldFallback) return { shouldFallback: false, cooldownMs: 0 };
|
||||||
|
|
||||||
if (isMultiBucketProvider(provider) && status === 429 && model) {
|
|
||||||
const lockDuration = cooldownMs > 0 ? cooldownMs : DEFAULT_MODEL_LOCK_MS;
|
|
||||||
lockModel(connectionId, model, lockDuration);
|
|
||||||
return { shouldFallback: true, cooldownMs: 0 };
|
|
||||||
}
|
|
||||||
|
|
||||||
const rateLimitedUntil = getUnavailableUntil(cooldownMs);
|
|
||||||
const reason = typeof errorText === "string" ? errorText.slice(0, 100) : "Provider error";
|
const reason = typeof errorText === "string" ? errorText.slice(0, 100) : "Provider error";
|
||||||
|
const lockUpdate = buildModelLockUpdate(model, cooldownMs);
|
||||||
|
|
||||||
await updateProviderConnection(connectionId, {
|
await updateProviderConnection(connectionId, {
|
||||||
rateLimitedUntil,
|
...lockUpdate,
|
||||||
testStatus: "unavailable",
|
testStatus: "unavailable",
|
||||||
lastError: reason,
|
lastError: reason,
|
||||||
errorCode: status,
|
errorCode: status,
|
||||||
@@ -254,6 +167,9 @@ export async function markAccountUnavailable(connectionId, status, errorText, pr
|
|||||||
backoffLevel: newBackoffLevel ?? backoffLevel
|
backoffLevel: newBackoffLevel ?? backoffLevel
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const lockKey = Object.keys(lockUpdate)[0];
|
||||||
|
log.warn("AUTH", `${connectionId.slice(0, 8)} locked ${lockKey} for ${Math.round(cooldownMs / 1000)}s [${status}]`);
|
||||||
|
|
||||||
if (provider && status && reason) {
|
if (provider && status && reason) {
|
||||||
console.error(`❌ ${provider} [${status}]: ${reason}`);
|
console.error(`❌ ${provider} [${status}]: ${reason}`);
|
||||||
}
|
}
|
||||||
@@ -263,21 +179,26 @@ export async function markAccountUnavailable(connectionId, status, errorText, pr
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Clear account error status (only if currently has error)
|
* Clear account error status (only if currently has error)
|
||||||
* Optimized to avoid unnecessary DB updates
|
* Clears all modelLock_* fields and resets error state.
|
||||||
*/
|
*/
|
||||||
export async function clearAccountError(connectionId, currentConnection) {
|
export async function clearAccountError(connectionId, currentConnection) {
|
||||||
// Only update if currently has error status
|
// Support both direct connection object and credentials wrapper
|
||||||
const hasError = currentConnection.testStatus === "unavailable" ||
|
const conn = currentConnection._connection || currentConnection;
|
||||||
currentConnection.lastError ||
|
const now = Date.now();
|
||||||
currentConnection.rateLimitedUntil;
|
|
||||||
|
// Collect all modelLock_* keys (both active and expired)
|
||||||
|
const allLockKeys = Object.keys(conn).filter(k => k.startsWith("modelLock_"));
|
||||||
|
const hasError = conn.testStatus === "unavailable" || conn.lastError || allLockKeys.length > 0;
|
||||||
|
|
||||||
if (!hasError) return; // Skip if already clean
|
if (!hasError) return; // Skip if already clean
|
||||||
|
|
||||||
|
// Clear all modelLock_* keys (lazy cleanup of expired ones included)
|
||||||
|
const clearLocks = Object.fromEntries(allLockKeys.map(k => [k, null]));
|
||||||
await updateProviderConnection(connectionId, {
|
await updateProviderConnection(connectionId, {
|
||||||
|
...clearLocks,
|
||||||
testStatus: "active",
|
testStatus: "active",
|
||||||
lastError: null,
|
lastError: null,
|
||||||
lastErrorAt: null,
|
lastErrorAt: null,
|
||||||
rateLimitedUntil: null,
|
|
||||||
backoffLevel: 0
|
backoffLevel: 0
|
||||||
});
|
});
|
||||||
log.info("AUTH", `Account ${connectionId.slice(0, 8)} error cleared`);
|
log.info("AUTH", `Account ${connectionId.slice(0, 8)} error cleared`);
|
||||||
|
|||||||
Reference in New Issue
Block a user