fix: reset account backoff after provider success

This commit is contained in:
Zac Gaetano 2026-08-24 21:58:41 -04:00
parent 39ddda477e
commit c06905aa44
6 changed files with 35 additions and 43 deletions

View file

@ -185,9 +185,13 @@ export function resetAccountState(account) {
if (!account) return account;
return {
...account,
...buildClearModelLocksUpdate(account),
rateLimitedUntil: null,
backoffLevel: 0,
testStatus: "active",
lastError: null,
errorCode: null,
lastErrorAt: null,
status: "active"
};
}

View file

@ -189,13 +189,14 @@ export async function createProviderConnection(data) {
}
// Critical: OAuth refresh token race — atomic merge inside transaction
export async function updateProviderConnection(id, data) {
export async function updateProviderConnection(id, update) {
const db = await getAdapter();
let result;
db.transaction(() => {
const row = db.get(`SELECT * FROM providerConnections WHERE id = ?`, [id]);
if (!row) { result = null; return; }
const existing = rowToConn(row);
const data = typeof update === "function" ? update(existing) : update;
const merged = { ...existing, ...data, updatedAt: new Date().toISOString() };
upsert(db, merged);
if (data.priority !== undefined) reorderInTx(db, existing.provider);

View file

@ -1,6 +1,6 @@
import {
extractApiKey, isValidApiKey,
getProviderCredentials, markAccountUnavailable,
getProviderCredentials, markAccountUnavailable, clearAccountError,
} from "../services/auth.js";
import { getSettings } from "@/lib/localDb";
import { getModelInfo } from "../services/model.js";
@ -74,7 +74,10 @@ export async function handleStt(request) {
const result = await handleSttCore({ provider, model, formData, credentials, sttConfig: AI_PROVIDERS[provider]?.sttConfig });
if (result.success) return result.response;
if (result.success) {
await clearAccountError(credentials.connectionId, credentials, model);
return result.response;
}
const { shouldFallback } = await markAccountUnavailable(credentials.connectionId, result.status, result.error, provider, model);
if (shouldFallback) {

View file

@ -1,6 +1,6 @@
import {
extractApiKey, isValidApiKey,
getProviderCredentials, markAccountUnavailable,
getProviderCredentials, markAccountUnavailable, clearAccountError,
} from "../services/auth.js";
import { getSettings } from "@/lib/localDb";
import { getModelInfo, getComboModels } from "../services/model.js";
@ -101,7 +101,10 @@ async function handleSingleModelTts(body, modelStr, responseFormat, language, st
const result = await handleTtsCore({ provider, model, input: body.input, credentials, responseFormat, language, style });
if (result.success) return result.response;
if (result.success) {
await clearAccountError(credentials.connectionId, credentials, model);
return result.response;
}
const { shouldFallback } = await markAccountUnavailable(credentials.connectionId, result.status, result.error, provider, model);
if (shouldFallback) {

View file

@ -1,6 +1,6 @@
import { getProviderConnections, validateApiKey, updateProviderConnection, getSettings, getProxyPools } from "@/lib/localDb";
import { resolveConnectionProxyConfig, pickProxyPoolId } from "@/lib/network/connectionProxy";
import { formatRetryAfter, checkFallbackError, isModelLockActive, buildModelLockUpdate, getEarliestModelLockUntil } from "open-sse/services/accountFallback.js";
import { formatRetryAfter, checkFallbackError, isModelLockActive, buildModelLockUpdate, getEarliestModelLockUntil, resetAccountState } from "open-sse/services/accountFallback.js";
import { MAX_RATE_LIMIT_COOLDOWN_MS } from "open-sse/config/errorConfig.js";
import { resolveProviderId, FREE_PROVIDERS } from "@/shared/constants/providers.js";
import * as log from "../utils/logger.js";
@ -274,43 +274,8 @@ export async function markAccountUnavailable(connectionId, status, errorText, pr
*/
export async function clearAccountError(connectionId, currentConnection, model = null) {
if (!connectionId || connectionId === "noauth") return;
const conn = currentConnection._connection || currentConnection;
const now = Date.now();
const allLockKeys = Object.keys(conn).filter(k => k.startsWith("modelLock_"));
if (!conn.testStatus && !conn.lastError && allLockKeys.length === 0) return;
// Keys to clear: current model's lock + all expired locks
const keysToClear = allLockKeys.filter(k => {
if (model && k === `modelLock_${model}`) return true; // succeeded model
if (model && k === "modelLock___all") return true; // account-level lock
const expiry = conn[k];
return expiry && new Date(expiry).getTime() <= now; // expired
});
if (keysToClear.length === 0 && conn.testStatus !== "unavailable" && !conn.lastError) return;
// Check if any active locks remain after clearing
const remainingActiveLocks = allLockKeys.filter(k => {
if (keysToClear.includes(k)) return false;
const expiry = conn[k];
return expiry && new Date(expiry).getTime() > now;
});
const clearObj = Object.fromEntries(keysToClear.map(k => [k, null]));
// Only reset error state if no active locks remain
if (remainingActiveLocks.length === 0) {
Object.assign(clearObj, {
testStatus: "active",
lastError: null,
errorCode: null,
lastErrorAt: null,
backoffLevel: 0
});
}
await updateProviderConnection(connectionId, clearObj);
// Reset inside transaction so concurrent 429 writes cannot leave stale locks.
await updateProviderConnection(connectionId, resetAccountState);
}
/**

View file

@ -0,0 +1,16 @@
import { describe, expect, it } from "vitest";
import { applyErrorState, resetAccountState } from "../../open-sse/services/accountFallback.js";
describe("resetAccountState", () => {
it("clears every model lock and backoff after success", () => {
const reset = resetAccountState({ backoffLevel: 9, modelLock_alpha: "2099-01-01T00:00:00.000Z", modelLock___all: "2099-01-01T00:00:00.000Z", lastError: "429", errorCode: 429 });
expect(reset.backoffLevel).toBe(0);
expect(reset.modelLock_alpha).toBeNull();
expect(reset.modelLock___all).toBeNull();
expect(reset.lastError).toBeNull();
});
it("keeps 429 on error path ratcheting upward", () => {
expect(applyErrorState({ backoffLevel: 3 }, 429, "rate limited").backoffLevel).toBe(4);
});
});