From c06905aa44b2ae598f557d9148203c301e4b74a6 Mon Sep 17 00:00:00 2001 From: Zac Gaetano Date: Mon, 24 Aug 2026 21:58:41 -0400 Subject: [PATCH] fix: reset account backoff after provider success --- open-sse/services/accountFallback.js | 4 +++ src/lib/db/repos/connectionsRepo.js | 3 +- src/sse/handlers/stt.js | 7 ++-- src/sse/handlers/tts.js | 7 ++-- src/sse/services/auth.js | 41 ++--------------------- tests/unit/account-fallback-reset.test.js | 16 +++++++++ 6 files changed, 35 insertions(+), 43 deletions(-) create mode 100644 tests/unit/account-fallback-reset.test.js diff --git a/open-sse/services/accountFallback.js b/open-sse/services/accountFallback.js index 8d280da4..4f8fa692 100644 --- a/open-sse/services/accountFallback.js +++ b/open-sse/services/accountFallback.js @@ -185,9 +185,13 @@ export function resetAccountState(account) { if (!account) return account; return { ...account, + ...buildClearModelLocksUpdate(account), rateLimitedUntil: null, backoffLevel: 0, + testStatus: "active", lastError: null, + errorCode: null, + lastErrorAt: null, status: "active" }; } diff --git a/src/lib/db/repos/connectionsRepo.js b/src/lib/db/repos/connectionsRepo.js index 4181843f..56727003 100644 --- a/src/lib/db/repos/connectionsRepo.js +++ b/src/lib/db/repos/connectionsRepo.js @@ -189,13 +189,14 @@ export async function createProviderConnection(data) { } // Critical: OAuth refresh token race — atomic merge inside transaction -export async function updateProviderConnection(id, data) { +export async function updateProviderConnection(id, update) { const db = await getAdapter(); let result; db.transaction(() => { const row = db.get(`SELECT * FROM providerConnections WHERE id = ?`, [id]); if (!row) { result = null; return; } const existing = rowToConn(row); + const data = typeof update === "function" ? update(existing) : update; const merged = { ...existing, ...data, updatedAt: new Date().toISOString() }; upsert(db, merged); if (data.priority !== undefined) reorderInTx(db, existing.provider); diff --git a/src/sse/handlers/stt.js b/src/sse/handlers/stt.js index 1840965e..703595e4 100644 --- a/src/sse/handlers/stt.js +++ b/src/sse/handlers/stt.js @@ -1,6 +1,6 @@ import { extractApiKey, isValidApiKey, - getProviderCredentials, markAccountUnavailable, + getProviderCredentials, markAccountUnavailable, clearAccountError, } from "../services/auth.js"; import { getSettings } from "@/lib/localDb"; import { getModelInfo } from "../services/model.js"; @@ -74,7 +74,10 @@ export async function handleStt(request) { const result = await handleSttCore({ provider, model, formData, credentials, sttConfig: AI_PROVIDERS[provider]?.sttConfig }); - if (result.success) return result.response; + if (result.success) { + await clearAccountError(credentials.connectionId, credentials, model); + return result.response; + } const { shouldFallback } = await markAccountUnavailable(credentials.connectionId, result.status, result.error, provider, model); if (shouldFallback) { diff --git a/src/sse/handlers/tts.js b/src/sse/handlers/tts.js index 539c8c7c..48688d50 100644 --- a/src/sse/handlers/tts.js +++ b/src/sse/handlers/tts.js @@ -1,6 +1,6 @@ import { extractApiKey, isValidApiKey, - getProviderCredentials, markAccountUnavailable, + getProviderCredentials, markAccountUnavailable, clearAccountError, } from "../services/auth.js"; import { getSettings } from "@/lib/localDb"; import { getModelInfo, getComboModels } from "../services/model.js"; @@ -101,7 +101,10 @@ async function handleSingleModelTts(body, modelStr, responseFormat, language, st const result = await handleTtsCore({ provider, model, input: body.input, credentials, responseFormat, language, style }); - if (result.success) return result.response; + if (result.success) { + await clearAccountError(credentials.connectionId, credentials, model); + return result.response; + } const { shouldFallback } = await markAccountUnavailable(credentials.connectionId, result.status, result.error, provider, model); if (shouldFallback) { diff --git a/src/sse/services/auth.js b/src/sse/services/auth.js index feaaa2ab..b14de87b 100644 --- a/src/sse/services/auth.js +++ b/src/sse/services/auth.js @@ -1,6 +1,6 @@ import { getProviderConnections, validateApiKey, updateProviderConnection, getSettings, getProxyPools } from "@/lib/localDb"; import { resolveConnectionProxyConfig, pickProxyPoolId } from "@/lib/network/connectionProxy"; -import { formatRetryAfter, checkFallbackError, isModelLockActive, buildModelLockUpdate, getEarliestModelLockUntil } from "open-sse/services/accountFallback.js"; +import { formatRetryAfter, checkFallbackError, isModelLockActive, buildModelLockUpdate, getEarliestModelLockUntil, resetAccountState } from "open-sse/services/accountFallback.js"; import { MAX_RATE_LIMIT_COOLDOWN_MS } from "open-sse/config/errorConfig.js"; import { resolveProviderId, FREE_PROVIDERS } from "@/shared/constants/providers.js"; import * as log from "../utils/logger.js"; @@ -274,43 +274,8 @@ export async function markAccountUnavailable(connectionId, status, errorText, pr */ export async function clearAccountError(connectionId, currentConnection, model = null) { if (!connectionId || connectionId === "noauth") return; - const conn = currentConnection._connection || currentConnection; - const now = Date.now(); - const allLockKeys = Object.keys(conn).filter(k => k.startsWith("modelLock_")); - - if (!conn.testStatus && !conn.lastError && allLockKeys.length === 0) return; - - // Keys to clear: current model's lock + all expired locks - const keysToClear = allLockKeys.filter(k => { - if (model && k === `modelLock_${model}`) return true; // succeeded model - if (model && k === "modelLock___all") return true; // account-level lock - const expiry = conn[k]; - return expiry && new Date(expiry).getTime() <= now; // expired - }); - - if (keysToClear.length === 0 && conn.testStatus !== "unavailable" && !conn.lastError) return; - - // Check if any active locks remain after clearing - const remainingActiveLocks = allLockKeys.filter(k => { - if (keysToClear.includes(k)) return false; - const expiry = conn[k]; - return expiry && new Date(expiry).getTime() > now; - }); - - const clearObj = Object.fromEntries(keysToClear.map(k => [k, null])); - - // Only reset error state if no active locks remain - if (remainingActiveLocks.length === 0) { - Object.assign(clearObj, { - testStatus: "active", - lastError: null, - errorCode: null, - lastErrorAt: null, - backoffLevel: 0 - }); - } - - await updateProviderConnection(connectionId, clearObj); + // Reset inside transaction so concurrent 429 writes cannot leave stale locks. + await updateProviderConnection(connectionId, resetAccountState); } /** diff --git a/tests/unit/account-fallback-reset.test.js b/tests/unit/account-fallback-reset.test.js new file mode 100644 index 00000000..fee59b17 --- /dev/null +++ b/tests/unit/account-fallback-reset.test.js @@ -0,0 +1,16 @@ +import { describe, expect, it } from "vitest"; +import { applyErrorState, resetAccountState } from "../../open-sse/services/accountFallback.js"; + +describe("resetAccountState", () => { + it("clears every model lock and backoff after success", () => { + const reset = resetAccountState({ backoffLevel: 9, modelLock_alpha: "2099-01-01T00:00:00.000Z", modelLock___all: "2099-01-01T00:00:00.000Z", lastError: "429", errorCode: 429 }); + expect(reset.backoffLevel).toBe(0); + expect(reset.modelLock_alpha).toBeNull(); + expect(reset.modelLock___all).toBeNull(); + expect(reset.lastError).toBeNull(); + }); + + it("keeps 429 on error path ratcheting upward", () => { + expect(applyErrorState({ backoffLevel: 3 }, 429, "rate limited").backoffLevel).toBe(4); + }); +});