feat(fetch): add --retries to fetch and extract, following the shared retry policy, off by default

README.md

@@ -125,8 +125,22 @@ zenrows fetch <url> # Adaptive Stealth Mode (mode=auto)
 zenrows fetch <url> --output markdown
 zenrows fetch <url> --manual --js-render --premium-proxy
 zenrows fetch <url> --proxy-country us --wait-for ".price"
+zenrows fetch <url> --retries 2 # retry a busy answer up to 2 times; off by default
 ```
 
+`fetch` and `extract` send each request once. With `--retries <n>`, they retry
+up to `n` times on a 429, 502, 503 or 504 from the API, and on a network error
+before the request reached the API (refused connection, DNS failure, connect
+timeout, unreachable network). Each wait is the `Retry-After` header in whole
+seconds (0.25 s to 30 s) or a random delay that grows with each retry, and no
+retry starts more than 90 s after the first request. They never retry a client
+timeout (`--timeout`) or a connection that closed after the request was sent,
+because the API may still finish and charge it; a status the target site sent
+(`--original-status`); or any request of `zenrows extract prepare`. If a 502,
+503 or 504 is still the answer after one or more retries, the error says how
+many retries ran and to wait a minute and run the same command again, without
+suggesting costlier options. A 429 keeps the error it has without `--retries`.
+
 ## 8. Extract
 
 ```bash

skills/extract/SKILL.md

@@ -68,6 +68,7 @@ zenrows extract https://www.example.com/products/a  # 3. extract again
   `limit` / `remaining`.
 - `EXTRACT_PREPARE_OUT_OF_ALLOWANCE`: no slots left this month; use `--css`.
   `EXTRACT_PREPARE_UNAVAILABLE`: retry in a few minutes; do not loop.
+  `prepare` sends each request once: `--retries` does not apply to it.
   A `failed` status prints the next step for its failure code and says whether
   the attempt used an allowance slot.
 - A domain that is already prepared answers `ready` at once

skills/protected-fetch/SKILL.md

@@ -47,6 +47,11 @@ zenrows fetch <url> --manual --js-render --premium-proxy   # the most expensive
   needs `--premium-proxy` (else `PARAM_PROXY_COUNTRY_REQUIRES_PREMIUM`).
 - You are billed only for the configuration that succeeds.
 - On failure, read `zenrows trace explain <run-id>` before escalating.
+- The CLI sends each request once. `--retries <n>` retries a 429, 502, 503 or
+  504 and a network error before the request reached the API, up to `n` times,
+  within 90 s; prefer it to your own retry loop. It never retries a timeout, a
+  connection closed after the request was sent, or a target status under
+  `--original-status`.
 
 Every run is saved under `.zenrows/runs/<run-id>/` (no secrets). See [[extract]]
 for structured output and [[cost-control]] for the cost model.

skills/trace-debug/SKILL.md

@@ -26,6 +26,11 @@ zenrows trace export <run-id>     # JSON for sharing
   `--manual --js-render` run; auto mode may ignore it. Each step
   multiplies the cost of the request, and both together are the most expensive
   configuration available ([[cost-control]]).
+- `FETCH_FAILED` "The Zenrows API answered HTTP 502/503/504 after N retries" →
+  the CLI already retried a busy answer (`--retries`). Wait a minute and run the
+  same command again; do not escalate, a costlier configuration does not fix
+  it. A 429 ("Rate limited") keeps its advice with or without `--retries`: wait
+  a few seconds and retry; if it persists, lower concurrency.
 - `REQUEST_TIMEOUT` → the CLI stopped waiting; the API was reached. Raise
   `--timeout` (default 120000ms, above the API's own 90s budget), or drop
   `--wait-for` so the request finishes inside that budget and the API returns

src/adapters/protected-fetch.ts

@@ -51,6 +51,8 @@ export interface FetchOptions {
    * gateway's own request budget on purpose.
    */
   timeoutMs?: number;
+  /** Retries for a busy answer or a request that never reached the API. Not an API parameter. */
+  retries?: number;
 }
 
 const RESPONSE_TYPE: Partial<Record<ResponseFormat, string>> = {
@@ -152,6 +154,6 @@ export async function runFetch(
   validateAutoManual(opts, config);
   const params = buildParams(opts, config);
   const mode: "auto" | "manual" = params.mode === "auto" ? "auto" : "manual";
-  const result = await scrape(config.apiBase, apiKey, params, { timeoutMs: opts.timeoutMs, projectRoot });
+  const result = await scrape(config.apiBase, apiKey, params, { timeoutMs: opts.timeoutMs, retries: opts.retries, projectRoot });
   return { result, params, mode };
 }

src/cli/commands/extract.ts

@@ -21,6 +21,7 @@ import { formatRequestCost } from "../../core/http.ts";
 import { parse, asString, asNumber, type Command, type RunContext } from "../command.ts";
 import { printError, writeOut } from "../output.ts";
 import { refreshClaimAdvice } from "../../core/claim-link.ts";
+import { normalizeRetries } from "./fetch.ts";
 import { PREPARE_HELP, PREPARE_USAGE, runPrepare } from "./extract-prepare.ts";
 
 export const extract: Command = {
@@ -42,6 +43,7 @@ export const extract: Command = {
     "Shared:",
     "  --manual --js-render --premium-proxy   manual fetch controls",
     "  --validate             fail if the result is not valid JSON",
+    "  --retries <n>          retry 429/502/503/504 and failed connects up to n times (default 0)",
     "  --out <file>           write output to a file",
     "  --no-signup            do not auto-create a Free plan account if no key exists",
     "  --json                 structured result",
@@ -66,6 +68,7 @@ export const extract: Command = {
       wait: { type: "string" },
       "wait-for": { type: "string" },
       validate: { type: "boolean" },
+      retries: { type: "string" },
       out: { type: "string" },
       "no-signup": { type: "boolean" },
       json: { type: "boolean" },
@@ -116,6 +119,7 @@ export const extract: Command = {
       wait: asNumber(values.wait),
       waitFor: asString(values["wait-for"]),
       validate: values.validate === true,
+      retries: normalizeRetries(values.retries),
     };
 
     assertUsable("extract");

src/cli/commands/fetch.ts

@@ -37,6 +37,7 @@ export const fetch_: Command = {
     "  --screenshot           capture an above-the-fold screenshot",
     "  --out <file>           write the response body to a file",
     "  --timeout <ms>         client-side timeout (default 120000; above the API's own 90s budget)",
+    "  --retries <n>          retry 429/502/503/504 and failed connects up to n times (default 0)",
     "  --no-signup            do not auto-create a Free plan account if no key exists",
     "  --json                 print a structured result",
     "",
@@ -59,6 +60,7 @@ export const fetch_: Command = {
       screenshot: { type: "boolean" },
       out: { type: "string" },
       timeout: { type: "string" },
+      retries: { type: "string" },
       "no-signup": { type: "boolean" },
       json: { type: "boolean" },
     });
@@ -101,6 +103,7 @@ export const fetch_: Command = {
       output: normalizeOutput(asString(values.output)),
       screenshot: values.screenshot === true,
       timeoutMs: normalizeTimeout(values.timeout),
+      retries: normalizeRetries(values.retries),
     };
 
     const runId = newRunId();
@@ -209,6 +212,20 @@ export function normalizeTimeout(v: unknown): number | undefined {
   return ms;
 }
 
+/** `--retries <n>`: a whole number, 0 or more. Like `--timeout`, a bad value is an error. */
+export function normalizeRetries(v: unknown): number | undefined {
+  if (v === undefined) return undefined;
+  if (typeof v !== "string" || !/^\d+$/.test(v)) {
+    throw new ToolkitError({
+      code: "INVALID_USAGE",
+      message: `Invalid --retries value '${String(v)}'.`,
+      likely_cause: "--retries takes a whole number, 0 or more.",
+      next_action: "Pass a count, e.g. --retries 2. 0, the default, sends the request once.",
+    });
+  }
+  return Number(v);
+}
+
 export function normalizeOutput(v?: string): ResponseFormat | undefined {
   if (!v) return undefined;
   const map: Record<string, ResponseFormat> = {

src/core/http.ts

@@ -9,7 +9,7 @@ import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "./
 import { storedClaimHint, usableClaimUrl } from "./claim-link.ts";
 import { readAccount } from "./agent-account.ts";
 import { ENV_KEY, resolveApiKey } from "./auth.ts";
-import { registerSecret } from "./logger.ts";
+import { log, registerSecret } from "./logger.ts";
 import { CLI_USER_AGENT } from "./config.ts";
 import { agentClientHeader } from "./agent-client.ts";
 import { normalizeDomain, PAGE_NOT_RECOGNISED_STEPS, prepareCommand } from "./prepared-domains-api.ts";
@@ -68,6 +68,47 @@ export const DEFAULT_TIMEOUT_MS = 120_000;
 /** The gateway's own request budget. Kept here only to justify the default above. */
 export const SERVER_BUDGET_MS = 90_000;
 
+/** No retry starts later than this after the first request. */
+const RETRY_BUDGET_MS = 90_000;
+/** Busy answers from the API, retried with `--retries`. */
+const RETRY_STATUSES: ReadonlySet<number> = new Set([429, 502, 503, 504]);
+/** Network error codes that mean the request never reached the API. */
+const NEVER_SENT_CODES: ReadonlySet<string> = new Set([
+  "ECONNREFUSED",
+  "ENOTFOUND",
+  "EAI_AGAIN",
+  "ENETUNREACH",
+  "EHOSTUNREACH",
+  "UND_ERR_CONNECT_TIMEOUT",
+]);
+
+/** `Retry-After` in whole seconds, clamped to 0.25-30 s; undefined for any other value. */
+export function retryAfterMs(value: string | null): number | undefined {
+  if (!value || !/^\d+$/.test(value.trim())) return undefined;
+  return Math.min(Math.max(Number(value) * 1000, 250), 30_000);
+}
+
+/** Full jitter before retry `n` (0-based): uniform(0.25 s, min(30 s, 2^n s)). */
+export function jitterMs(n: number, random: () => number = Math.random): number {
+  return 250 + random() * (Math.min(30_000, 1000 * 2 ** n) - 250);
+}
+
+/**
+ * True when a fetch error's `cause` means the request never reached the API.
+ * ETIMEDOUT counts only on connect. A happy-eyeballs AggregateError counts
+ * only when every address failed to connect.
+ */
+export function neverSent(cause: unknown): boolean {
+  if (cause instanceof AggregateError) return cause.errors.length > 0 && cause.errors.every(neverSent);
+  const { code, syscall } = (cause ?? {}) as { code?: unknown; syscall?: unknown };
+  return code === "ETIMEDOUT" ? syscall === "connect" : NEVER_SENT_CODES.has(String(code));
+}
+
+/** A busy answer from the API. A status the target site sent is never retried. */
+function isRetryable(status: number, params: ScraperParams): boolean {
+  return RETRY_STATUSES.has(status) && !params.original_status && params.allowed_status_codes === undefined;
+}
+
 function buildUrl(apiBase: string, apiKey: string, params: ScraperParams): { full: string; redacted: string } {
   const u = new URL(apiBase);
   u.searchParams.set("apikey", apiKey);
@@ -86,48 +127,77 @@ export async function scrape(
   apiBase: string,
   apiKey: string,
   params: ScraperParams,
-  opts: { timeoutMs?: number; projectRoot?: string } = {},
+  opts: {
+    timeoutMs?: number;
+    projectRoot?: string;
+    /** Retries after the first request (`--retries`). 0, the default, sends it once. */
+    retries?: number;
+    /** Test seams for the waits and the retry budget. */
+    sleep?: (ms: number) => Promise<void>;
+    now?: () => number;
+  } = {},
 ): Promise<ScraperResult> {
   registerSecret(apiKey);
   const { full, redacted } = buildUrl(apiBase, apiKey, params);
   const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
-  const controller = new AbortController();
-  // Our own timer is the only thing that aborts this controller, so this flag —
-  // not `err.name === "AbortError"` — is what tells a client-side give-up apart
-  // from a genuine transport failure. The two must never share an error code.
-  let timedOut = false;
-  const timeout = setTimeout(() => {
-    timedOut = true;
-    controller.abort();
-  }, timeoutMs);
-  const startedAt = Date.now();
+  const retries = opts.retries ?? 0;
+  const now = opts.now ?? Date.now;
+  const firstSentAt = now();
+  /** Waits before retry `n` and returns true, unless no retry is left or it would start after the budget. */
+  const retry = async (n: number, delayMs: number, reason: string): Promise<boolean> => {
+    if (n >= retries || now() - firstSentAt + delayMs > RETRY_BUDGET_MS) return false;
+    log.dim(`${reason}; retry ${n + 1}/${retries} in ${formatMs(delayMs)}`);
+    await (opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms))))(delayMs);
+    return true;
+  };
 
   let res: Response;
-  try {
-    res = await fetch(full, {
-      method: "GET",
-      headers: { "User-Agent": CLI_USER_AGENT, "Accept-Encoding": "gzip, deflate", ...agentClientHeader({ projectRoot: opts.projectRoot }) },
-      signal: controller.signal,
-    });
-  } catch (err) {
+  let raw: Buffer;
+  let retriesUsed = 0;
+  for (; ; retriesUsed++) {
+    const controller = new AbortController();
+    // Our own timer is the only thing that aborts this controller, so this flag —
+    // not `err.name === "AbortError"` — is what tells a client-side give-up apart
+    // from a genuine transport failure. The two must never share an error code.
+    let timedOut = false;
+    const timeout = setTimeout(() => {
+      timedOut = true;
+      controller.abort();
+    }, timeoutMs);
+    const startedAt = Date.now();
+
+    try {
+      res = await fetch(full, {
+        method: "GET",
+        headers: { "User-Agent": CLI_USER_AGENT, "Accept-Encoding": "gzip, deflate", ...agentClientHeader({ projectRoot: opts.projectRoot }) },
+        signal: controller.signal,
+      });
+    } catch (err) {
+      clearTimeout(timeout);
+      const elapsedMs = Date.now() - startedAt;
+      // A timeout, or an error after the request was sent, is never retried:
+      // the API may still finish and charge that request.
+      if (timedOut) throw requestTimeout(params.url, timeoutMs, elapsedMs);
+      const retryable = err instanceof Error && neverSent(err.cause);
+      if (retryable && (await retry(retriesUsed, jitterMs(retriesUsed), "Network error before the request reached the API"))) continue;
+      const cause = err instanceof Error ? err.message : String(err);
+      throw new ToolkitError({
+        code: "BACKEND_UNAVAILABLE",
+        message: `Could not reach the Zenrows API.`,
+        likely_cause: `Network error after ${formatMs(elapsedMs)}: ${cause}`,
+        next_action: "Check connectivity and retry. Verify api base in `zenrows config show`.",
+        suggested_commands: ["zenrows status", "zenrows config show"],
+      });
+    }
     clearTimeout(timeout);
-    const elapsedMs = Date.now() - startedAt;
-    if (timedOut) throw requestTimeout(params.url, timeoutMs, elapsedMs);
-    const cause = err instanceof Error ? err.message : String(err);
-    throw new ToolkitError({
-      code: "BACKEND_UNAVAILABLE",
-      message: `Could not reach the Zenrows API.`,
-      likely_cause: `Network error after ${formatMs(elapsedMs)}: ${cause}`,
-      next_action: "Check connectivity and retry. Verify api base in `zenrows config show`.",
-      suggested_commands: ["zenrows status", "zenrows config show"],
-    });
-  }
-  clearTimeout(timeout);
 
-  // Read the raw bytes once. `body` (UTF-8) drives all error detection and text
-  // output; `raw` preserves the exact bytes so binary responses (screenshot /
-  // PDF) can be written without the UTF-8 round-trip that corrupts them.
-  const raw = Buffer.from(await res.arrayBuffer());
+    // Read the raw bytes once. `body` (UTF-8) drives all error detection and text
+    // output; `raw` preserves the exact bytes so binary responses (screenshot /
+    // PDF) can be written without the UTF-8 round-trip that corrupts them.
+    raw = Buffer.from(await res.arrayBuffer());
+    const delayMs = retryAfterMs(res.headers.get("retry-after")) ?? jitterMs(retriesUsed);
+    if (!isRetryable(res.status, params) || !(await retry(retriesUsed, delayMs, `HTTP ${res.status}`))) break;
+  }
   const body = raw.toString("utf8");
   // A binary body is only ever produced by a successful screenshot/PDF request.
   // Zenrows error responses are always a JSON envelope (text), so exclude those.
@@ -280,6 +350,16 @@ export async function scrape(
         : "Verify the URL. Rendering or premium proxies will not turn a 404 into content — do not retry; it will fail identically and still cost credits.",
     });
   }
+  if (retriesUsed > 0 && isRetryable(res.status, params)) {
+    // A busy answer that outlasted the retries: a costlier configuration does not fix it.
+    throw new ToolkitError({
+      code: "FETCH_FAILED",
+      message: `The Zenrows API answered HTTP ${res.status} after ${retriesUsed} ${retriesUsed === 1 ? "retry" : "retries"}.`,
+      likely_cause: snippet(body) || "The Zenrows API is busy.",
+      next_action:
+        "Wait a minute, then run the same command again. Do not add --js-render or --premium-proxy: they cost more and do not help here.",
+    });
+  }
   if (res.status === 422 || res.status >= 500 || (res.status >= 400 && !looksLikeContent(result))) {
     throw new ToolkitError({
       code: "FETCH_FAILED",