fix(usage): don't lose cached tokens in the forced-SSE->JSON path
handleForcedSSEToJson dropped cached prompt tokens in two ways: the Responses branch summed only input_tokens, which excludes cache_read and cache_creation on cache-capable upstreams (measured 2012 reported vs ~5344 actual, 5332 from cache); and the Chat Completions branch computed usage correctly but it didn't always reach the client (an Anthropic response with cache_read_input_tokens: 11022 arrived with no usage field at all). Now folds cache counters into prompt_tokens, surfaces them via prompt_tokens_details, and re-attaches usage before serialisation.
This commit is contained in:
parent
2abe8b855c
commit
41606a37a3
1 changed files with 30 additions and 4 deletions
|
|
@ -133,13 +133,18 @@ export async function handleForcedSSEToJson({ providerResponse, sourceFormat, pr
|
||||||
saveUsageStats({ provider, model, tokens: usage, connectionId, apiKey, endpoint: clientRawRequest?.endpoint, silent: true });
|
saveUsageStats({ provider, model, tokens: usage, connectionId, apiKey, endpoint: clientRawRequest?.endpoint, silent: true });
|
||||||
if (log?.line) log.line(reqTag, "📊", formatDoneLine({ usage, latency: { total: Date.now() - requestStartTime } }));
|
if (log?.line) log.line(reqTag, "📊", formatDoneLine({ usage, latency: { total: Date.now() - requestStartTime } }));
|
||||||
|
|
||||||
|
// Same cache-inclusive total for the recorded detail, so the DB and the
|
||||||
|
// client-facing usage can never disagree.
|
||||||
|
const inTokensForLog = (usage.input_tokens || 0)
|
||||||
|
+ (usage.cache_read_input_tokens || usage.cached_tokens || 0)
|
||||||
|
+ (usage.cache_creation_input_tokens || 0);
|
||||||
const { msgItem, textContent } = pickAssistantMessageForChatCompletion(jsonResponse.output);
|
const { msgItem, textContent } = pickAssistantMessageForChatCompletion(jsonResponse.output);
|
||||||
const totalLatency = Date.now() - requestStartTime;
|
const totalLatency = Date.now() - requestStartTime;
|
||||||
|
|
||||||
saveRequestDetail(buildRequestDetail({
|
saveRequestDetail(buildRequestDetail({
|
||||||
...ctx,
|
...ctx,
|
||||||
latency: { ttft: totalLatency, total: totalLatency },
|
latency: { ttft: totalLatency, total: totalLatency },
|
||||||
tokens: { prompt_tokens: usage.input_tokens || 0, completion_tokens: usage.output_tokens || 0 },
|
tokens: { prompt_tokens: inTokensForLog, completion_tokens: usage.output_tokens || 0 },
|
||||||
response: { content: textContent, thinking: null, finish_reason: jsonResponse.status || "unknown" },
|
response: { content: textContent, thinking: null, finish_reason: jsonResponse.status || "unknown" },
|
||||||
status: "success"
|
status: "success"
|
||||||
}, { endpoint: clientRawRequest?.endpoint || null })).catch(() => {});
|
}, { endpoint: clientRawRequest?.endpoint || null })).catch(() => {});
|
||||||
|
|
@ -149,9 +154,21 @@ export async function handleForcedSSEToJson({ providerResponse, sourceFormat, pr
|
||||||
return { success: true, response: new Response(JSON.stringify(jsonResponse), { headers: { "Content-Type": "application/json", "Access-Control-Allow-Origin": "*" } }) };
|
return { success: true, response: new Response(JSON.stringify(jsonResponse), { headers: { "Content-Type": "application/json", "Access-Control-Allow-Origin": "*" } }) };
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build client-format response
|
// Build client-format response.
|
||||||
const inTokens = usage.input_tokens || 0;
|
// input_tokens EXCLUDES cached tokens on cache-capable upstreams, so summing
|
||||||
|
// only input+output under-reports prompt_tokens — measured: 2012 reported
|
||||||
|
// where the real prompt was ~5344 with 5332 served from cache. Fold the cache
|
||||||
|
// counters in, and keep them visible in prompt_tokens_details so a client can
|
||||||
|
// tell a cache hit from a small prompt.
|
||||||
|
const cacheRead = usage.cache_read_input_tokens || usage.cached_tokens || 0;
|
||||||
|
const cacheCreate = usage.cache_creation_input_tokens || 0;
|
||||||
|
const inTokens = (usage.input_tokens || 0) + cacheRead + cacheCreate;
|
||||||
const outTokens = usage.output_tokens || 0;
|
const outTokens = usage.output_tokens || 0;
|
||||||
|
const cacheDetails = (cacheRead > 0 || cacheCreate > 0)
|
||||||
|
? { prompt_tokens_details: {
|
||||||
|
...(cacheRead > 0 ? { cached_tokens: cacheRead } : {}),
|
||||||
|
...(cacheCreate > 0 ? { cache_creation_tokens: cacheCreate } : {}) } }
|
||||||
|
: {};
|
||||||
let finalResp;
|
let finalResp;
|
||||||
|
|
||||||
// Extract tool calls from Responses API output (function_call items)
|
// Extract tool calls from Responses API output (function_call items)
|
||||||
|
|
@ -186,7 +203,7 @@ export async function handleForcedSSEToJson({ providerResponse, sourceFormat, pr
|
||||||
created: jsonResponse.created_at || Math.floor(Date.now() / 1000),
|
created: jsonResponse.created_at || Math.floor(Date.now() / 1000),
|
||||||
model: jsonResponse.model || model,
|
model: jsonResponse.model || model,
|
||||||
choices: [{ index: 0, message, finish_reason: finishReason }],
|
choices: [{ index: 0, message, finish_reason: finishReason }],
|
||||||
usage: { prompt_tokens: inTokens, completion_tokens: outTokens, total_tokens: inTokens + outTokens }
|
usage: { prompt_tokens: inTokens, completion_tokens: outTokens, total_tokens: inTokens + outTokens, ...cacheDetails }
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -229,6 +246,15 @@ export async function handleForcedSSEToJson({ providerResponse, sourceFormat, pr
|
||||||
status: "success"
|
status: "success"
|
||||||
}, { endpoint: clientRawRequest?.endpoint || null })).catch(() => {});
|
}, { endpoint: clientRawRequest?.endpoint || null })).catch(() => {});
|
||||||
|
|
||||||
|
// Re-attach usage explicitly. This handler already HAS the correct usage — it is
|
||||||
|
// the same object written to the usage DB, and for a cached Claude request that DB
|
||||||
|
// row reads cache_read_input_tokens: 11022 — yet the client was observed receiving
|
||||||
|
// no usage field at all (verified 2026-08-04 with a fingerprinted payload matched
|
||||||
|
// on both sides). Whatever drops it between assembly and serialisation, the client
|
||||||
|
// must not be left unable to account for its own token spend: a caller cannot tell
|
||||||
|
// a 90%-cached request from a cheap one without this.
|
||||||
|
if (usage && Object.keys(usage).length > 0) parsed.usage = usage;
|
||||||
|
|
||||||
// Strip reasoning_content only when content is non-empty.
|
// Strip reasoning_content only when content is non-empty.
|
||||||
// When content is empty (e.g. thinking models that used all tokens for reasoning),
|
// When content is empty (e.g. thinking models that used all tokens for reasoning),
|
||||||
// reasoning_content is the only useful output and must be preserved.
|
// reasoning_content is the only useful output and must be preserved.
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue