From c81743a07a203cb23c7ae7079ad130aa8b09cc36 Mon Sep 17 00:00:00 2001 From: gonzalo-zenrows Date: Tue, 3 Nov 2026 16:12:47 +0100 Subject: [PATCH] fix(fetch): wait past the API's 3-minute budget before giving up A Fetch request can take up to 3 minutes, as the API docs say, but the CLI stopped waiting at 120 s and its help and errors spoke of a 90 s budget. So a slow request was cut off before the API could answer. The default --timeout is now 200 s, above the API's 180 s budget, and the help, the timeout error and the trace-debug skill say 180 s. --- skills/trace-debug/SKILL.md | 2 +- src/cli/commands/fetch.ts | 2 +- src/core/http.ts | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/skills/trace-debug/SKILL.md b/skills/trace-debug/SKILL.md index 30f6db0..1f9603b 100644 --- a/skills/trace-debug/SKILL.md +++ b/skills/trace-debug/SKILL.md @@ -32,7 +32,7 @@ zenrows trace export # JSON for sharing it. A 429 ("Rate limited") keeps its advice with or without `--retries`: wait a few seconds and retry; if it persists, lower concurrency. - `REQUEST_TIMEOUT` → the CLI stopped waiting; the API was reached. Raise - `--timeout` (default 120000ms, above the API's own 90s budget), or drop + `--timeout` (default 200000ms, above the API's own 180s budget), or drop `--wait-for` so the request finishes inside that budget and the API returns its own error. Not a connectivity problem — do not chase the network. - `BACKEND_UNAVAILABLE` → a genuine transport failure (DNS/TCP/TLS). Check diff --git a/src/cli/commands/fetch.ts b/src/cli/commands/fetch.ts index 01c3b04..0325fa6 100644 --- a/src/cli/commands/fetch.ts +++ b/src/cli/commands/fetch.ts @@ -36,7 +36,7 @@ export const fetch_: Command = { " --output html (default) | markdown | text | pdf", " --screenshot capture an above-the-fold screenshot", " --out write the response body to a file", - " --timeout client-side timeout (default 120000; above the API's own 90s budget)", + " --timeout client-side timeout (default 200000; above the API's own 180s budget)", " --retries retry 429/502/503/504 and failed connects up to n times (default 0)", " --no-signup do not auto-create a Free plan account if no key exists", " --json print a structured result", diff --git a/src/core/http.ts b/src/core/http.ts index e55eb2a..a138e44 100644 --- a/src/core/http.ts +++ b/src/core/http.ts @@ -54,7 +54,7 @@ export interface ScraperParams { /** * Client-side timeout for a Fetch and Extract request, deliberately ABOVE the - * gateway's own 90s request budget. + * gateway's own request budget: a Fetch request can take up to 3 minutes. * * The two must not be equal. When they were both 90s, any request that used * the full server budget became a race between our own abort and the API's @@ -63,10 +63,10 @@ export interface ScraperParams { * client must outlive the server budget so the API always gets the chance to * answer for itself. */ -export const DEFAULT_TIMEOUT_MS = 120_000; +export const DEFAULT_TIMEOUT_MS = 200_000; /** The gateway's own request budget. Kept here only to justify the default above. */ -export const SERVER_BUDGET_MS = 90_000; +export const SERVER_BUDGET_MS = 180_000; /** No retry starts later than this after the first request. */ const RETRY_BUDGET_MS = 90_000; @@ -459,7 +459,7 @@ export function extractDomainNotPrepared(url: string, body: string): ToolkitErro * The CLI gave up waiting before the API answered. This is NOT unreachability: * the connection was established and the gateway was still working, so telling * the operator to check connectivity would send them at the wrong problem. The - * elapsed time is included so the 90s server budget is visible in the output. + * elapsed time is included so the server budget is visible in the output. */ export function requestTimeout(url: string, timeoutMs: number, elapsedMs: number): ToolkitError { const suggestedMs = Math.max(timeoutMs + 60_000, SERVER_BUDGET_MS + 60_000);