fix(jina-reader): recover after transient errors and use JSON POST API
Clear stale provider error code and account lock after a successful web fetch (the core fetch handler never consumed the onRequestSuccess callback), switch Jina Reader to its documented JSON POST request, and parse the Title: metadata line before falling back to a Markdown heading.
This commit is contained in:
parent
007d372724
commit
3c17d3406b
5 changed files with 176 additions and 9 deletions
|
|
@ -49,7 +49,10 @@ function truncate(text, max) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function parseJinaTitle(text) {
|
function parseJinaTitle(text) {
|
||||||
const m = String(text || "").match(/^\s*#\s+(.+)$/m);
|
const source = String(text || "");
|
||||||
|
const metadataTitle = source.match(/^\s*Title:\s*(.+)$/mi);
|
||||||
|
if (metadataTitle) return metadataTitle[1].trim();
|
||||||
|
const m = source.match(/^\s*#\s+(.+)$/m);
|
||||||
return m ? m[1].trim() : null;
|
return m ? m[1].trim() : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -151,11 +154,14 @@ async function runFirecrawl({ url, fmt, timeoutMs, apiKey, maxCharacters, costPe
|
||||||
}
|
}
|
||||||
|
|
||||||
async function runJina({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt }) {
|
async function runJina({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt }) {
|
||||||
const target = `https://r.jina.ai/${encodeURIComponent(url)}`;
|
|
||||||
const upstreamStart = Date.now();
|
const upstreamStart = Date.now();
|
||||||
const r = await tryFetch(target, {
|
const r = await tryFetch("https://r.jina.ai/", {
|
||||||
method: "GET",
|
method: "POST",
|
||||||
headers: apiKey ? { authorization: `Bearer ${apiKey}` } : {}
|
headers: {
|
||||||
|
"content-type": "application/json",
|
||||||
|
...(apiKey ? { authorization: `Bearer ${apiKey}` } : {})
|
||||||
|
},
|
||||||
|
body: JSON.stringify({ url })
|
||||||
}, timeoutMs);
|
}, timeoutMs);
|
||||||
|
|
||||||
if (!r.ok) {
|
if (!r.ok) {
|
||||||
|
|
|
||||||
|
|
@ -195,13 +195,11 @@ async function handleSingleProviderFetch(body, providerInput, request, apiKey, s
|
||||||
providerSpecificData: newCreds.providerSpecificData,
|
providerSpecificData: newCreds.providerSpecificData,
|
||||||
testStatus: "active"
|
testStatus: "active"
|
||||||
});
|
});
|
||||||
},
|
|
||||||
onRequestSuccess: async () => {
|
|
||||||
await clearAccountError(credentials.connectionId, credentials);
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
if (result.success) {
|
if (result.success) {
|
||||||
|
await clearAccountError(credentials.connectionId, credentials);
|
||||||
return new Response(JSON.stringify(result.data), {
|
return new Response(JSON.stringify(result.data), {
|
||||||
headers: { "Content-Type": "application/json", "Access-Control-Allow-Origin": "*" }
|
headers: { "Content-Type": "application/json", "Access-Control-Allow-Origin": "*" }
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -285,7 +285,13 @@ export async function clearAccountError(connectionId, currentConnection, model =
|
||||||
|
|
||||||
// Only reset error state if no active locks remain
|
// Only reset error state if no active locks remain
|
||||||
if (remainingActiveLocks.length === 0) {
|
if (remainingActiveLocks.length === 0) {
|
||||||
Object.assign(clearObj, { testStatus: "active", lastError: null, lastErrorAt: null, backoffLevel: 0 });
|
Object.assign(clearObj, {
|
||||||
|
testStatus: "active",
|
||||||
|
lastError: null,
|
||||||
|
errorCode: null,
|
||||||
|
lastErrorAt: null,
|
||||||
|
backoffLevel: 0
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
await updateProviderConnection(connectionId, clearObj);
|
await updateProviderConnection(connectionId, clearObj);
|
||||||
|
|
|
||||||
91
tests/unit/fetch-success-clears-account.test.js
Normal file
91
tests/unit/fetch-success-clears-account.test.js
Normal file
|
|
@ -0,0 +1,91 @@
|
||||||
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
|
||||||
|
const mocks = vi.hoisted(() => ({
|
||||||
|
getProviderCredentials: vi.fn(),
|
||||||
|
markAccountUnavailable: vi.fn(),
|
||||||
|
clearAccountError: vi.fn(),
|
||||||
|
extractApiKey: vi.fn(() => null),
|
||||||
|
isValidApiKey: vi.fn(),
|
||||||
|
getSettings: vi.fn(),
|
||||||
|
getCombos: vi.fn(),
|
||||||
|
handleFetchCore: vi.fn(),
|
||||||
|
checkAndRefreshToken: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("@/sse/services/auth.js", () => ({
|
||||||
|
getProviderCredentials: mocks.getProviderCredentials,
|
||||||
|
markAccountUnavailable: mocks.markAccountUnavailable,
|
||||||
|
clearAccountError: mocks.clearAccountError,
|
||||||
|
extractApiKey: mocks.extractApiKey,
|
||||||
|
isValidApiKey: mocks.isValidApiKey,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("@/lib/localDb", () => ({
|
||||||
|
getSettings: mocks.getSettings,
|
||||||
|
getCombos: mocks.getCombos,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("open-sse/handlers/fetch/index.js", () => ({
|
||||||
|
handleFetchCore: mocks.handleFetchCore,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("@/sse/services/tokenRefresh.js", () => ({
|
||||||
|
checkAndRefreshToken: mocks.checkAndRefreshToken,
|
||||||
|
updateProviderCredentials: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("@/sse/utils/logger.js", () => ({
|
||||||
|
request: vi.fn(),
|
||||||
|
info: vi.fn(),
|
||||||
|
debug: vi.fn(),
|
||||||
|
warn: vi.fn(),
|
||||||
|
error: vi.fn(),
|
||||||
|
maskKey: vi.fn(() => "masked"),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("@/shared/utils/ssrfGuard.js", () => ({
|
||||||
|
assertPublicUrl: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
import { handleFetch } from "@/sse/handlers/fetch.js";
|
||||||
|
|
||||||
|
describe("web fetch account state", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.clearAllMocks();
|
||||||
|
mocks.getSettings.mockResolvedValue({ requireApiKey: false });
|
||||||
|
mocks.getCombos.mockResolvedValue([]);
|
||||||
|
mocks.getProviderCredentials.mockResolvedValue({
|
||||||
|
apiKey: "jina-test-key",
|
||||||
|
connectionId: "jina-connection",
|
||||||
|
connectionName: "Jina Test",
|
||||||
|
_connection: {
|
||||||
|
testStatus: "unavailable",
|
||||||
|
lastError: "old error",
|
||||||
|
modelLock___all: "2026-01-01T00:00:00.000Z",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
mocks.checkAndRefreshToken.mockImplementation(async (_provider, credentials) => credentials);
|
||||||
|
mocks.handleFetchCore.mockResolvedValue({
|
||||||
|
success: true,
|
||||||
|
data: { provider: "jina-reader", content: { text: "ok" } },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("clears a stale provider lock after a successful fetch", async () => {
|
||||||
|
const response = await handleFetch(new Request("http://localhost/v1/web/fetch", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({
|
||||||
|
provider: "jina-reader",
|
||||||
|
url: "https://example.com/article",
|
||||||
|
}),
|
||||||
|
}));
|
||||||
|
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
expect(mocks.clearAccountError).toHaveBeenCalledWith(
|
||||||
|
"jina-connection",
|
||||||
|
expect.objectContaining({ connectionName: "Jina Test" }),
|
||||||
|
);
|
||||||
|
expect(mocks.markAccountUnavailable).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
66
tests/unit/jina-reader-fetch.test.js
Normal file
66
tests/unit/jina-reader-fetch.test.js
Normal file
|
|
@ -0,0 +1,66 @@
|
||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import { handleFetchCore } from "../../open-sse/handlers/fetch/index.js";
|
||||||
|
|
||||||
|
const originalFetch = global.fetch;
|
||||||
|
|
||||||
|
describe("Jina Reader fetch", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
global.fetch = vi.fn();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
global.fetch = originalFetch;
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses Jina's JSON POST API instead of embedding the URL in the path", async () => {
|
||||||
|
global.fetch.mockResolvedValueOnce(new Response([
|
||||||
|
"Title: Example page",
|
||||||
|
"",
|
||||||
|
"URL Source: https://example.com/article",
|
||||||
|
"",
|
||||||
|
"Markdown Content:",
|
||||||
|
"Hello",
|
||||||
|
].join("\n")));
|
||||||
|
|
||||||
|
const result = await handleFetchCore({
|
||||||
|
url: "https://example.com/article",
|
||||||
|
format: "markdown",
|
||||||
|
provider: "jina-reader",
|
||||||
|
providerConfig: { timeoutMs: 30000 },
|
||||||
|
credentials: { apiKey: "jina-test-key" },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
expect(result.data.title).toBe("Example page");
|
||||||
|
expect(global.fetch).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
const [requestUrl, init] = global.fetch.mock.calls[0];
|
||||||
|
expect(requestUrl).toBe("https://r.jina.ai/");
|
||||||
|
expect(init.method).toBe("POST");
|
||||||
|
expect(init.headers).toEqual({
|
||||||
|
"content-type": "application/json",
|
||||||
|
authorization: "Bearer jina-test-key",
|
||||||
|
});
|
||||||
|
expect(JSON.parse(init.body)).toEqual({ url: "https://example.com/article" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns the upstream status and error body", async () => {
|
||||||
|
global.fetch.mockResolvedValueOnce(new Response(
|
||||||
|
JSON.stringify({ detail: "Payment required" }),
|
||||||
|
{ status: 402, headers: { "Content-Type": "application/json" } },
|
||||||
|
));
|
||||||
|
|
||||||
|
const result = await handleFetchCore({
|
||||||
|
url: "https://example.com/article",
|
||||||
|
provider: "jina-reader",
|
||||||
|
providerConfig: { timeoutMs: 30000 },
|
||||||
|
credentials: { apiKey: "jina-test-key" },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
success: false,
|
||||||
|
status: 402,
|
||||||
|
});
|
||||||
|
expect(result.error).toContain("Payment required");
|
||||||
|
});
|
||||||
|
});
|
||||||
Loading…
Reference in a new issue