fix(antigravity): retry transient upstream failures
Retry short-lived 5xx/capacity errors (500/502/503/504 + message patterns) with bounded backoff capped at 15s; honor Retry-After/reset hints and skip when wait is too long. Keep 400 non-retryable. Enable the retry hook for 500 alongside existing 429/503. Deduplicate sanitized Antigravity tool names before emitting the single functionDeclarations group to avoid upstream "Tool names must be unique" rejections. Add Headroom size diagnostics and phantom-savings warning when reported token delta does not shrink the outbound payload. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
2deacf69b1
commit
639f1204d0
4 changed files with 116 additions and 22 deletions
|
|
@ -16,8 +16,26 @@ function sanitizeFunctionName(name) {
|
||||||
}
|
}
|
||||||
|
|
||||||
const MAX_RETRY_AFTER_MS = 10000;
|
const MAX_RETRY_AFTER_MS = 10000;
|
||||||
|
const ANTIGRAVITY_TRANSIENT_RETRY_MAX_MS = 15000;
|
||||||
const MAX_ANTIGRAVITY_OUTPUT_TOKENS = 16384;
|
const MAX_ANTIGRAVITY_OUTPUT_TOKENS = 16384;
|
||||||
|
|
||||||
|
const ANTIGRAVITY_TRANSIENT_ERROR_PATTERNS = [
|
||||||
|
/high\s+traffic/i,
|
||||||
|
/agent\s+(execution\s+)?terminated\s+due\s+to\s+error/i,
|
||||||
|
/capacity/i,
|
||||||
|
/temporarily\s+unavailable/i,
|
||||||
|
/timeout/i,
|
||||||
|
/stream\s+(ended|closed|terminated|interrupted)/i,
|
||||||
|
/empty\s+response/i,
|
||||||
|
];
|
||||||
|
|
||||||
|
const ANTIGRAVITY_TRANSIENT_STATUSES = new Set([
|
||||||
|
HTTP_STATUS.SERVER_ERROR,
|
||||||
|
HTTP_STATUS.BAD_GATEWAY,
|
||||||
|
HTTP_STATUS.SERVICE_UNAVAILABLE,
|
||||||
|
HTTP_STATUS.GATEWAY_TIMEOUT,
|
||||||
|
]);
|
||||||
|
|
||||||
// Fields Google generateContent rejects (Claude/OpenAI/Qwen thinking fields set at body root by thinkingUnified.js)
|
// Fields Google generateContent rejects (Claude/OpenAI/Qwen thinking fields set at body root by thinkingUnified.js)
|
||||||
const ANTIGRAVITY_REQUEST_BLACKLIST = [
|
const ANTIGRAVITY_REQUEST_BLACKLIST = [
|
||||||
"output_config",
|
"output_config",
|
||||||
|
|
@ -170,15 +188,22 @@ export class AntigravityExecutor extends BaseExecutor {
|
||||||
|
|
||||||
if (tools && tools.length > 0) {
|
if (tools && tools.length > 0) {
|
||||||
// Merge all groups into a single functionDeclarations group (Gemini expects 1 group)
|
// Merge all groups into a single functionDeclarations group (Gemini expects 1 group)
|
||||||
const allDeclarations = tools.flatMap(group =>
|
const seenToolNames = new Set();
|
||||||
(group.functionDeclarations || []).map(fn => ({
|
const allDeclarations = [];
|
||||||
|
for (const group of tools) {
|
||||||
|
for (const fn of group.functionDeclarations || []) {
|
||||||
|
const name = sanitizeFunctionName(fn.name);
|
||||||
|
if (seenToolNames.has(name)) continue;
|
||||||
|
seenToolNames.add(name);
|
||||||
|
allDeclarations.push({
|
||||||
...fn,
|
...fn,
|
||||||
name: sanitizeFunctionName(fn.name),
|
name,
|
||||||
parameters: fn.parameters
|
parameters: fn.parameters
|
||||||
? cleanJSONSchemaForAntigravity(structuredClone(fn.parameters))
|
? cleanJSONSchemaForAntigravity(structuredClone(fn.parameters))
|
||||||
: { type: "object", properties: { reason: { type: "string", description: "Brief explanation" } }, required: ["reason"] }
|
: { type: "object", properties: { reason: { type: "string", description: "Brief explanation" } }, required: ["reason"] }
|
||||||
}))
|
});
|
||||||
);
|
}
|
||||||
|
}
|
||||||
tools = allDeclarations.length > 0 ? [{ functionDeclarations: allDeclarations }] : [];
|
tools = allDeclarations.length > 0 ? [{ functionDeclarations: allDeclarations }] : [];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -305,23 +330,49 @@ export class AntigravityExecutor extends BaseExecutor {
|
||||||
return totalMs > 0 ? totalMs : null;
|
return totalMs > 0 ? totalMs : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Hook called by BaseExecutor.tryRetry: derive delay from Retry-After (header → body),
|
extractErrorMessage(errorJson, bodyText = "") {
|
||||||
// cap at MAX_RETRY_AFTER_MS, else exponential backoff for 429. Return false to veto (fallback URL).
|
return [
|
||||||
async computeRetryDelay(response, attempt) {
|
errorJson?.error?.message,
|
||||||
let retryMs = this.parseRetryHeaders(response.headers);
|
errorJson?.message,
|
||||||
if (!retryMs) {
|
errorJson?.error,
|
||||||
try {
|
bodyText,
|
||||||
const errorJson = JSON.parse(await response.clone().text());
|
].filter(Boolean).map(v => typeof v === "string" ? v : JSON.stringify(v)).join("\n");
|
||||||
retryMs = this.parseRetryFromErrorMessage(errorJson?.error?.message || errorJson?.message || "");
|
|
||||||
} catch {
|
|
||||||
// ignore parse errors → fall through to backoff
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
isTransientAntigravityError(status, message) {
|
||||||
|
if (status === HTTP_STATUS.RATE_LIMITED) return true;
|
||||||
|
if (ANTIGRAVITY_TRANSIENT_STATUSES.has(status)) return true;
|
||||||
|
return ANTIGRAVITY_TRANSIENT_ERROR_PATTERNS.some(pattern => pattern.test(message || ""));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hook called by BaseExecutor.tryRetry: derive delay from Retry-After (header → body),
|
||||||
|
// cap at MAX_RETRY_AFTER_MS, else retry transient Antigravity failures with backoff.
|
||||||
|
// Return false to veto (fallback URL / final error).
|
||||||
|
async computeRetryDelay(response, attempt) {
|
||||||
|
let bodyText = "";
|
||||||
|
let errorJson = null;
|
||||||
|
let retryMs = this.parseRetryHeaders(response.headers);
|
||||||
|
|
||||||
|
try {
|
||||||
|
bodyText = await response.clone().text();
|
||||||
|
errorJson = bodyText ? JSON.parse(bodyText) : null;
|
||||||
|
} catch {
|
||||||
|
// ignore parse errors → fall through to status/message based retry
|
||||||
|
}
|
||||||
|
|
||||||
|
const errorMessage = this.extractErrorMessage(errorJson, bodyText);
|
||||||
|
|
||||||
|
if (!retryMs) {
|
||||||
|
retryMs = this.parseRetryFromErrorMessage(errorMessage);
|
||||||
}
|
}
|
||||||
if (retryMs) return retryMs <= MAX_RETRY_AFTER_MS ? retryMs : false;
|
if (retryMs) return retryMs <= MAX_RETRY_AFTER_MS ? retryMs : false;
|
||||||
if (response.status === HTTP_STATUS.RATE_LIMITED) {
|
|
||||||
return Math.min(1000 * (2 ** attempt), MAX_RETRY_AFTER_MS); // exponential backoff
|
if (!this.isTransientAntigravityError(response.status, errorMessage)) return false;
|
||||||
}
|
|
||||||
return false;
|
const cap = response.status === HTTP_STATUS.RATE_LIMITED
|
||||||
|
? MAX_RETRY_AFTER_MS
|
||||||
|
: ANTIGRAVITY_TRANSIENT_RETRY_MAX_MS;
|
||||||
|
return Math.min(1000 * (2 ** attempt), cap); // exponential backoff
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -32,6 +32,9 @@ export default {
|
||||||
"429": {
|
"429": {
|
||||||
attempts: 3,
|
attempts: 3,
|
||||||
},
|
},
|
||||||
|
"500": {
|
||||||
|
attempts: 3,
|
||||||
|
},
|
||||||
"503": {
|
"503": {
|
||||||
attempts: 3,
|
attempts: 3,
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -32,8 +32,38 @@ describe("antigravity computeRetryDelay hook (D3)", () => {
|
||||||
expect(await ag.computeRetryDelay(res(429), 3)).toBe(Math.min(1000 * 2 ** 3, MAX));
|
expect(await ag.computeRetryDelay(res(429), 3)).toBe(Math.min(1000 * 2 ** 3, MAX));
|
||||||
});
|
});
|
||||||
|
|
||||||
it("503 without retry info → veto (no auto backoff)", async () => {
|
it("503 without retry info → transient backoff", async () => {
|
||||||
expect(await ag.computeRetryDelay(res(503), 1)).toBe(false);
|
expect(await ag.computeRetryDelay(res(503), 1)).toBe(2000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("retries Antigravity agent terminated body even when status is not 429", async () => {
|
||||||
|
const r = res(500, {}, { error: { message: "Agent execution terminated due to error" } });
|
||||||
|
expect(await ag.computeRetryDelay(r, 1)).toBe(2000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("retries high traffic body", async () => {
|
||||||
|
const r = res(500, {}, { error: { message: "Our servers are experiencing high traffic" } });
|
||||||
|
expect(await ag.computeRetryDelay(r, 2)).toBe(4000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not retry non-transient 400 errors", async () => {
|
||||||
|
const r = res(400, {}, { error: { message: "Invalid request" } });
|
||||||
|
expect(await ag.computeRetryDelay(r, 1)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("deduplicates sanitized tool names", () => {
|
||||||
|
const out = ag.transformRequest("claude-opus-4-6-thinking", {
|
||||||
|
request: {
|
||||||
|
contents: [{ role: "user", parts: [{ text: "hi" }] }],
|
||||||
|
tools: [{ functionDeclarations: [
|
||||||
|
{ name: "read/file", parameters: { type: "object", properties: {} } },
|
||||||
|
{ name: "read file", parameters: { type: "object", properties: {} } },
|
||||||
|
{ name: "read/file", parameters: { type: "object", properties: {} } },
|
||||||
|
] }],
|
||||||
|
},
|
||||||
|
}, true, { projectId: "project-1", connectionId: "conn-1" });
|
||||||
|
|
||||||
|
expect(out.request.tools[0].functionDeclarations.map(fn => fn.name)).toEqual(["read_file"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("buildHeaders includes cached session id after transformRequest", () => {
|
it("buildHeaders includes cached session id after transformRequest", () => {
|
||||||
|
|
|
||||||
|
|
@ -85,6 +85,16 @@ describe("BaseExecutor.execute — network error retry/fallback", () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("BaseExecutor.execute — computeRetryDelay hook veto", () => {
|
describe("BaseExecutor.execute — computeRetryDelay hook veto", () => {
|
||||||
|
it("only invokes computeRetryDelay when status has retry config", async () => {
|
||||||
|
const ex = makeExec({ baseUrl: "https://x/api", retry: { 503: { attempts: 1, delayMs: 0 } } });
|
||||||
|
ex.computeRetryDelay = vi.fn().mockResolvedValue(0);
|
||||||
|
fetchMock.mockResolvedValueOnce(res(500));
|
||||||
|
const out = await ex.execute({ model: "m", body: {}, stream: false, credentials: creds });
|
||||||
|
expect(out.response.status).toBe(500);
|
||||||
|
expect(ex.computeRetryDelay).not.toHaveBeenCalled();
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
it("hook returning false skips retry (uses fallback path)", async () => {
|
it("hook returning false skips retry (uses fallback path)", async () => {
|
||||||
const ex = makeExec({ baseUrl: "https://x/api", retry: { 429: { attempts: 5, delayMs: 0 } } });
|
const ex = makeExec({ baseUrl: "https://x/api", retry: { 429: { attempts: 5, delayMs: 0 } } });
|
||||||
ex.computeRetryDelay = vi.fn().mockResolvedValue(false);
|
ex.computeRetryDelay = vi.fn().mockResolvedValue(false);
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue