From 23bab586f12c81abbef8fe731a6aee85db879079 Mon Sep 17 00:00:00 2001 From: gonzalo-zenrows Date: Tue, 3 Nov 2026 15:55:21 +0100 Subject: [PATCH] feat(client): follow the shared retry policy when retries are turned on Retries stay off by default: a client that never sets `retries` sends each request once, as before. With `retries` set, a GET is retried on a 429, a 5xx or a 422 other than `no_data_extracted`, and on a failure to connect. Waits use `Retry-After` in whole seconds (at most 30 s) or full jitter, within a 90 s budget (new `retry_max_wait` and `retry_budget` options). A POST or PUT, a read timeout, a stream body and a status from the target site are never retried. Every `retries` value 1.12.0 accepted still works, a non-int one with a DeprecationWarning, and setting it later takes effect. --- README.md | 42 ++- src/zenrows/client.py | 186 ++++++++++--- tests/test_retries.py | 592 +++++++++++++++++++++++++++++++++++++----- 3 files changed, 709 insertions(+), 111 deletions(-) diff --git a/README.md b/README.md index 994b5f9..64f1008 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ This package ships three clients: - [Concurrency](#concurrency) - [Extract](#extract) - [Preparing a domain](#preparing-a-domain) + - [Retries](#retries) - [Quickstart — Batch API (`ZenRowsBatchClient`)](#quickstart--batch-api-zenrowsbatchclient) - [Extract in a batch](#extract-in-a-batch) - [Upload URLs from a CSV](#upload-urls-from-a-csv) @@ -59,7 +60,7 @@ Start using the API by [creating your API Key](https://www.zenrows.com/register? The SDK uses [requests](https://docs.python-requests.org/) for HTTP requests. The client's response will be a requests `Response`. -It also uses [Retry](https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html) to automatically retry failed requests (status codes 422, 429, 500, 502, 503, and 504, except Extract's `422 no_data_extracted`). Retries are not active by default; you need to specify the number of retries, as shown below. It already includes an exponential back-off retry delay between failed requests. +Retries are off by default: the client sends each request once. Pass `retries` to retry a failed GET, as shown below. See [Retries](#retries) for what is retried and how long the client waits. `client.fetch()` is the primary method for the main page-scraping product; `client.get()` still works and is kept as a deprecated alias. @@ -132,7 +133,7 @@ The SDK also offers POST requests by calling the `client.post` method. It can re ```python from zenrows import ZenRowsClient -client = ZenRowsClient("YOUR-API-KEY", retries=1) +client = ZenRowsClient("YOUR-API-KEY") url = "https://httpbin.org/anything" response = client.post(url, data={ @@ -150,7 +151,7 @@ The SDK also offers PUT requests by calling the `client.put` method. It can rece ```python from zenrows import ZenRowsClient -client = ZenRowsClient("YOUR-API-KEY", retries=1) +client = ZenRowsClient("YOUR-API-KEY") url = "https://httpbin.org/anything" response = client.put(url, data={ @@ -271,6 +272,41 @@ Both methods raise `PreparedDomainError` on an error answer, with `status_code`, A `422` with `code: "no_data_extracted"` means Extract knows this kind of page but found no data on it. It isn't charged, and the client never retries it, even with `retries` set. `is_no_data_extracted(response)` tells you when you got one. See the [Extract docs](https://docs.zenrows.com/extract/setup). +### Retries + +Retries are off by default (`retries=0`): the client sends each request once. Pass `retries` to turn them on: + +```python +from zenrows import ZenRowsClient + +client = ZenRowsClient("YOUR-API-KEY", retries=3, retry_max_wait=10, retry_budget=60) +``` + +With `retries` set, the client retries a GET, up to `retries` times, after: + +1. A `429`, `500`, `502`, `503` or `504`, or a `422` other than Extract's `no_data_extracted`. +2. A failure to connect: connection refused, a name that does not resolve, or a connect timeout. The request never reached the server. + +It never retries: + +1. A POST or PUT. The API does not deduplicate requests, so a second try could run twice. Write your own retry if you need one. +2. A request with `files`, or whose `data` is a file, a generator or another stream: the first try uses it up. +3. A read timeout (it raises `requests.ReadTimeout`), a connection closed after the request was sent, a TLS error, or any other status. The server may still be working on that request. +4. A status from the target site: any status when `params` sets `original_status`, and a status listed in `allowed_status_codes`. +5. `prepare_domain()` and `prepared_domain_status()`: their `429` and `503` are answers to read. + +Between tries, the client waits the `Retry-After` header when it is whole seconds (at least 0.25 s, at most 30 s). Otherwise it waits a random time between 0.25 s and `min(retry_max_wait, 2 ** attempt)` seconds, where `attempt` counts retries from 0 and `retry_max_wait` defaults to 30. No retry starts after `retry_budget` seconds (default 90) from the call's first try; a running try is not cut short. A wait runs to its end, also in the `_async` methods. `retry_max_wait` and `retry_budget` must be numbers above 0, or the client raises `ValueError`. Setting `client.retries` later applies from the next call. + +**If you already set `retries`.** Up to 1.12.0 the client retried more. Now: + +1. A PUT is no longer retried on a `429` or `5xx`, and a POST or PUT is no longer retried on a `422` or a failure to connect. +2. A read timeout, a dropped connection, a TLS error and a `413` with `Retry-After` are no longer retried. A read timeout raises `requests.ReadTimeout` instead of `requests.ConnectionError`. +3. A status from the target site, a request with `files` and a stream body are sent once. +4. `Retry-After` counts only as whole seconds, capped at 30 s: an HTTP date is ignored, and `1.5` no longer raises `InvalidHeader`. The `422` retry now honours it too. +5. Waits are jittered instead of 0, 1, 2, 4... seconds, within the 90 s budget, and a call sends at most `retries + 1` requests: a `422` retry no longer adds to the `5xx` retries. +6. A `bool` or non-integer `retries` still works with a `DeprecationWarning`: `True` is 1, `2.5` is 2, 0 or less is off. +7. Setting `client.retries` after creating the client now applies to every retry, not only the `422` retry. + ## Quickstart — Batch API (`ZenRowsBatchClient`) > Batch is open to every account and spends the same credits as a regular diff --git a/src/zenrows/client.py b/src/zenrows/client.py index 61df0c7..851ab4e 100644 --- a/src/zenrows/client.py +++ b/src/zenrows/client.py @@ -13,9 +13,13 @@ import asyncio import math +import numbers import os +import random +import sys import time import warnings +from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor from functools import partial from typing import Any @@ -23,19 +27,23 @@ import requests import urllib3 -from requests.adapters import HTTPAdapter -from urllib3.util.retry import Retry +from urllib3.exceptions import ConnectTimeoutError from zenrows.__version__ import __version__ DEFAULT_SCRAPER_URL = "https://api.zenrows.com/v1/" DEFAULT_USER_AGENT = f"zenrows/{__version__} python" -# Status codes the urllib3 retry layer treats as transient. 422 is retried -# too, but in `_worker`, because whether it is worth retrying depends on the -# response body (see `is_no_data_extracted`). -_RETRY_STATUSES = [429, 500, 502, 503, 504] -_RETRY_BACKOFF_FACTOR = 0.5 +# Statuses worth another try. A 422 is retried too, unless its body is +# Extract's `no_data_extracted` (see `is_no_data_extracted`). +_RETRY_STATUSES = (429, 500, 502, 503, 504) +_RETRY_MIN_WAIT = 0.25 +_RETRY_AFTER_CAP = 30.0 +DEFAULT_RETRY_MAX_WAIT = 30.0 +DEFAULT_RETRY_BUDGET = 90.0 + +# Clock for the retry budget; tests replace it. +_monotonic = time.monotonic # Extract's answer when it knows the kind of page but none of the fields came # back filled. Retrying returns the same answer, so it is never retried. @@ -59,6 +67,83 @@ def is_no_data_extracted(response: requests.Response) -> bool: return response.status_code == 422 and code is not None and code.lower() == _NO_DATA_EXTRACTED +def _retry_count(retries: Any) -> int: + """`retries` as a count of retries; 0 or less means off. A bool or a + non-int number still works as in 1.12.0 (True is 1, 2.5 is 2), with a + DeprecationWarning.""" + if isinstance(retries, int) and not isinstance(retries, bool): + return max(retries, 0) + if not isinstance(retries, numbers.Real): + raise TypeError(f"ZenRowsClient: retries must be an int, got {retries!r}.") + count = int(min(retries, sys.maxsize)) if retries > 0 else 0 + warnings.warn( + f"ZenRowsClient: pass retries as an int; {retries!r} is read as {count}.", + DeprecationWarning, + stacklevel=3, + ) + return count + + +def _seconds(name: str, value: Any) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)) or not 0 < value < math.inf: + raise ValueError( + f"ZenRowsClient: {name} must be a number of seconds above 0, got {value!r}." + ) + return float(value) + + +def _retry_after_seconds(value: str | None) -> float | None: + """The wait a `Retry-After` header asks for, clamped to [0.25, 30] s, or + None when it is not whole seconds (an HTTP date, a fraction, a sign).""" + value = (value or "").strip() + if not (value.isascii() and value.isdigit()): + return None + # float(), not int(): int() refuses a string of thousands of digits. + return min(max(float(value), _RETRY_MIN_WAIT), _RETRY_AFTER_CAP) + + +def _jitter(attempt: int, retry_max_wait: float) -> float: + """Full jitter: a random wait between 0.25 s and + `min(retry_max_wait, 2 ** attempt)` s, `attempt` counting retries from 0.""" + ceiling = max(_RETRY_MIN_WAIT, min(retry_max_wait, 2**attempt)) + return _RETRY_MIN_WAIT + (ceiling - _RETRY_MIN_WAIT) * random.random() + + +def _can_send_again(method: str, data: Any, kwargs: dict) -> bool: + """True for a GET whose body `requests` builds the same way on every try. + A POST or PUT could run twice (the API does not deduplicate requests), + and a file or stream body is used up by the first try.""" + replayable = data is None or isinstance(data, (str, bytes, Mapping, list, tuple)) + return method == "GET" and replayable and not kwargs.get("files") + + +def _failed_to_connect(error: requests.ConnectionError) -> bool: + """True when the request never reached the server: connection refused, + a name that does not resolve, or a connect timeout.""" + reason = getattr(error.args[0] if error.args else None, "reason", None) + reason = getattr(reason, "original_error", reason) # behind a proxy + return isinstance(reason, ConnectTimeoutError) # NewConnectionError is one too + + +def _from_target(status: int, params: dict) -> bool: + """True when `status` may be the target site's own answer, which another + try cannot change: any status with `original_status`, or one listed in + `allowed_status_codes`.""" + if str(params.get("original_status", "")).strip().lower() in ("true", "1"): + return True + allowed = params.get("allowed_status_codes", "") + if isinstance(allowed, (list, tuple)): + allowed = ",".join(map(str, allowed)) + return str(status) in (code.strip() for code in str(allowed).split(",")) + + +def _is_retryable(response: requests.Response, params: dict) -> bool: + status = response.status_code + if status not in _RETRY_STATUSES and (status != 422 or is_no_data_extracted(response)): + return False + return not _from_target(status, params) + + _FALLBACK_DEPRECATED = ( "fallback_to_autoparse is deprecated and has no effect: extract() no longer " "retries with autoparse. On a 403 REQS007, check `reason` in the problem " @@ -182,6 +267,14 @@ class ZenRowsClient: `base_url` defaults to the public production endpoint; override via constructor arg or `ZENROWS_SCRAPER_BASE_URL` env var. + + Retries are off by default (`retries=0`): each request is sent once. + With `retries` above 0, a GET is retried on a 429, 500, 502, 503 or 504, + a 422 other than `no_data_extracted`, or a failure to connect. Waits + honour `Retry-After` in whole seconds (0.25 to 30 s), or else are + jittered up to `retry_max_wait`, and no retry starts after + `retry_budget` seconds from the first try. See the README for the full + rules. """ # Kept as a class attribute for compatibility — pre-1.4 callers @@ -195,25 +288,29 @@ def __init__( concurrency: int = 5, *, base_url: str | None = None, + retry_max_wait: float = DEFAULT_RETRY_MAX_WAIT, + retry_budget: float = DEFAULT_RETRY_BUDGET, ): if not apikey: raise ValueError("ZenRowsClient: apikey is required.") self.apikey = apikey self.api_url = base_url or os.environ.get("ZENROWS_SCRAPER_BASE_URL") or DEFAULT_SCRAPER_URL - self.retries = retries + self._retries = _retry_count(retries) + self.retry_max_wait = _seconds("retry_max_wait", retry_max_wait) + self.retry_budget = _seconds("retry_budget", retry_budget) self.executor = ThreadPoolExecutor(max_workers=concurrency) self.requests_session = requests.Session() - if retries > 0: - max_retries = Retry( - total=retries, - backoff_factor=_RETRY_BACKOFF_FACTOR, - status_forcelist=_RETRY_STATUSES, - raise_on_status=False, - ) - adapter = HTTPAdapter(max_retries=max_retries) - self.requests_session.mount("https://", adapter) - self.requests_session.mount("http://", adapter) + + @property + def retries(self) -> int: + """How many times a call may be retried; 0 (the default) is off. Read + on every call, so setting it after construction takes effect.""" + return self._retries + + @retries.setter + def retries(self, value: int) -> None: + self._retries = _retry_count(value) # ---- sync HTTP verbs ---- @@ -510,7 +607,8 @@ def _worker( final_headers["Connection"] = None final_headers.update(headers) - response = self.requests_session.request( + send = partial( + self.requests_session.request, method, self.api_url, params=final_params, @@ -518,25 +616,31 @@ def _worker( data=data, **kwargs, ) - # Retry a 422 here rather than in urllib3, which can't read the body: - # Extract's `no_data_extracted` is a final answer, any other 422 is a - # failure to get the target's content and is worth another try. - attempt = 0 - while ( - response.status_code == 422 - and attempt < self.retries - and not is_no_data_extracted(response) - ): - attempt += 1 - # Same schedule as urllib3's Retry: no wait before the first retry. - if attempt > 1: - time.sleep(_RETRY_BACKOFF_FACTOR * 2 ** (attempt - 1)) - response = self.requests_session.request( - method, - self.api_url, - params=final_params, - headers=final_headers, - data=data, - **kwargs, - ) - return response + if not self.retries or not _can_send_again(method, data, kwargs): + return send() + started = _monotonic() + for attempt in range(self.retries): + try: + response = send() + except requests.ConnectionError as error: + if not (_failed_to_connect(error) and self._wait_to_retry(attempt, started, None)): + raise + else: + retry_after = response.headers.get("Retry-After") + if not ( + _is_retryable(response, final_params) + and self._wait_to_retry(attempt, started, retry_after) + ): + return response + return send() + + def _wait_to_retry(self, attempt: int, started: float, retry_after: str | None) -> bool: + """Sleep before retry number `attempt` (from 0) and return True, or + return False when the wait would end past the retry budget.""" + delay = _retry_after_seconds(retry_after) + if delay is None: + delay = _jitter(attempt, self.retry_max_wait) + if _monotonic() - started + delay > self.retry_budget: + return False + time.sleep(delay) + return True diff --git a/tests/test_retries.py b/tests/test_retries.py index 0fb37d5..87fa20e 100644 --- a/tests/test_retries.py +++ b/tests/test_retries.py @@ -1,105 +1,563 @@ +"""The ZenRowsClient retry policy, against a local HTTP server. + +Retries are off by default. Sleeps are patched out, and the retry budget runs +on a fake clock. `tests/test_prepared_domains.py` checks that the +prepared-domains calls are never retried. +""" + import asyncio +import io +import math +import socket +import sys +import threading +import warnings +from collections.abc import Callable, Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from unittest import TestCase from unittest.mock import patch -from requests import Response, Session +import pytest +import requests +from requests import Response +from requests.adapters import HTTPAdapter from zenrows import ZenRowsClient, is_no_data_extracted -from zenrows.client import _RETRY_STATUSES +from zenrows.client import _jitter, _retry_after_seconds -apikey = "APIKEY" -url = "http://example.com" -api_url_base = "https://api.zenrows.com/v1/" +APIKEY = "APIKEY" +TARGET = "http://example.com" +KEY = {"Idempotency-Key": "a-unique-key"} +NO_DATA = b'{"code": "no_data_extracted", "title": "No data extracted"}' +NO_CONTENT = b'{"code": "RESP001", "title": "Could not get content"}' -class TestZenRowsClientRetries(TestCase): - @patch("zenrows.client.Retry") - @patch.object(Session, "mount") - def test_custom_session_not_initiated(self, mock_mount, mock_retry): - ZenRowsClient(apikey, retries=0) +# Scripted answers: (status, headers, body). HANG answers after the client's +# read timeout; DROP closes the connection without an answer. +HANG = (0, {}, b"") +DROP = (-1, {}, b"") + + +class _Server: + """Answers each request with the next scripted answer (the last one + repeats) and records the body of every request.""" + + def __init__(self) -> None: + self.script: list[tuple[int, dict[str, str], bytes]] = [(200, {}, b"ok")] + self.bodies: list[bytes] = [] + self.on_request = lambda: None + server = self + + class Handler(BaseHTTPRequestHandler): + def _answer(self) -> None: + server.bodies.append(self._body()) + server.on_request() + status, headers, body = ( + server.script.pop(0) if len(server.script) > 1 else server.script[0] + ) + if status == -1: + self.close_connection = True + return + if status == 0: + threading.Event().wait(0.5) # not time.sleep: tests patch it + status = 200 + try: + self.send_response(status) + for name, value in headers.items(): + self.send_header(name, value) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + except OSError: + pass # the client gave up first + + def _body(self) -> bytes: + if self.headers.get("Transfer-Encoding", "").lower() != "chunked": + return self.rfile.read(int(self.headers.get("Content-Length") or 0)) + body = b"" + while size := int(self.rfile.readline().strip() or b"0", 16): + body += self.rfile.read(size) + self.rfile.readline() + self.rfile.readline() + return body + + do_GET = do_POST = do_PUT = _answer + + def log_message(self, *args: object) -> None: + pass + + self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.url = f"http://127.0.0.1:{self.httpd.server_address[1]}/" + threading.Thread( + target=self.httpd.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True + ).start() + + @property + def calls(self) -> int: + return len(self.bodies) + + +@pytest.fixture +def server() -> Iterator[_Server]: + srv = _Server() + yield srv + srv.httpd.shutdown() + srv.httpd.server_close() + + +@pytest.fixture +def sleeps() -> Iterator[list[float]]: + """Every wait between tries, in seconds; none of them actually waits.""" + waited: list[float] = [] + with patch("time.sleep", side_effect=waited.append): + yield waited + + +def client(server: _Server, **kwargs) -> ZenRowsClient: + return ZenRowsClient(APIKEY, base_url=server.url, **kwargs) + + +def closed_port_url() -> str: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + return f"http://127.0.0.1:{port}/" # nothing listens there + + +# ---- off by default ---- + + +@pytest.mark.parametrize("options", [{}, {"retries": 0}]) +@pytest.mark.parametrize("status", [503, 422]) +def test_default_sends_once(server: _Server, sleeps: list[float], options: dict, status: int): + server.script = [(status, {}, NO_CONTENT), (200, {}, b"ok")] + zr = client(server, **options) + + response = zr.fetch(TARGET) + + assert response.status_code == status + assert server.calls == 1 + assert sleeps == [] + assert type(zr.requests_session.get_adapter(server.url)) is HTTPAdapter + assert zr.requests_session.get_adapter(server.url).max_retries.total == 0 + + +# ---- the options ---- + + +@pytest.mark.parametrize("set_later", [False, True]) +@pytest.mark.parametrize( + ("value", "count", "warns"), + [ + (3, 3, False), + (0, 0, False), + (-1, 0, False), + (False, 0, True), + (0.0, 0, True), + (True, 1, True), + (2.5, 2, True), + (math.nan, 0, True), + (math.inf, sys.maxsize, True), + ], +) +def test_legacy_retries_values_still_work(value: object, count: int, warns: bool, set_later: bool): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + if set_later: + zr = ZenRowsClient(APIKEY) + zr.retries = value + else: + zr = ZenRowsClient(APIKEY, retries=value) - mock_retry.assert_not_called() - self.assertEqual(mock_mount.call_count, 2) # called internally + assert zr.retries == count + deprecations = [w for w in caught if w.category is DeprecationWarning] + assert len(deprecations) == (1 if warns else 0) + assert all(w.filename == __file__ for w in deprecations) # points at the caller - @patch("zenrows.client.Retry") - def test_retry_parameters(self, mock_retry): - ZenRowsClient(apikey, retries=2) - mock_retry.assert_called_once_with( - total=2, - backoff_factor=0.5, - status_forcelist=list(_RETRY_STATUSES), - raise_on_status=False, - ) +@pytest.mark.parametrize("value", [None, "2"]) +def test_retries_that_is_not_a_number_raises_type_error(value: object): + with pytest.raises(TypeError, match="retries"): + ZenRowsClient(APIKEY, retries=value) - @patch.object(Session, "mount") - def test_adapter_is_mount(self, mock_mount): - ZenRowsClient(apikey, retries=2) - self.assertEqual(mock_mount.call_count, 4) +def test_setting_retries_later_turns_retries_on_and_off(server: _Server, sleeps: list[float]): + zr = client(server) + zr.retries = 2 + server.script = [(503, {}, b""), (422, {}, NO_CONTENT), (200, {}, b"ok")] + assert zr.fetch(TARGET).status_code == 200 + assert server.calls == 3 -def _fake_response(status_code: int, body: bytes = b"") -> Response: - response = Response() - response.status_code = status_code - response._content = body - return response + zr.retries = 0 + server.script = [(503, {}, b""), (200, {}, b"ok")] + assert zr.fetch(TARGET).status_code == 503 + assert server.calls == 4 -NO_DATA = b'{"code": "no_data_extracted", "title": "No data extracted"}' -NO_CONTENT = b'{"code": "RESP001", "title": "Could not get content"}' +@pytest.mark.parametrize("option", ["retry_max_wait", "retry_budget"]) +@pytest.mark.parametrize("value", [0, -1.0, math.inf, math.nan, "5", None, True]) +def test_invalid_wait_option_is_rejected(option: str, value: object): + with pytest.raises(ValueError, match=option): + ZenRowsClient(APIKEY, **{option: value}) + + +# ---- what is retried ---- + + +@pytest.mark.parametrize( + ("status", "body"), + [(429, b""), (500, b""), (502, b""), (503, b""), (504, b""), (422, NO_CONTENT), (422, b"x")], +) +def test_retryable_status_is_retried( + server: _Server, sleeps: list[float], status: int, body: bytes +): + server.script = [(status, {}, body), (200, {}, b"ok")] + + response = client(server, retries=2).fetch(TARGET) + + assert response.status_code == 200 + assert server.calls == 2 + assert len(sleeps) == 1 + + +@pytest.mark.parametrize( + ("status", "headers", "body"), + [ + (400, {}, b""), + (401, {}, b""), + (403, {"Retry-After": "1"}, b""), + (404, {}, b""), + (413, {"Retry-After": "1"}, b""), + (422, {}, NO_DATA), + ], +) +def test_other_status_is_not_retried( + server: _Server, sleeps: list[float], status: int, headers: dict, body: bytes +): + server.script = [(status, headers, body), (200, {}, b"ok")] + + response = client(server, retries=3).fetch(TARGET) + + assert response.status_code == status + assert server.calls == 1 + assert sleeps == [] + + +@pytest.mark.parametrize("use_async", [False, True]) +def test_extract_does_not_retry_no_data_extracted( + server: _Server, sleeps: list[float], use_async: bool +): + server.script = [(422, {}, NO_DATA), (200, {}, b"{}")] + zr = client(server, retries=3) + + response = asyncio.run(zr.extract_async(TARGET)) if use_async else zr.extract(TARGET) + + assert is_no_data_extracted(response) + assert server.calls == 1 + + +@pytest.mark.parametrize("status", [503, 422]) +def test_last_response_is_returned_when_retries_run_out( + server: _Server, sleeps: list[float], status: int +): + server.script = [(status, {}, b"first"), (status, {}, b"second"), (status, {}, b"last")] + response = client(server, retries=2).fetch(TARGET) -@patch("zenrows.client.time.sleep") -class TestUnprocessableRetries(TestCase): - """A 422 is retried in the client, not in urllib3, because only the body - says whether another try can help.""" + assert response.content == b"last" + assert server.calls == 3 + assert len(sleeps) == 2 - @patch.object(Session, "request") - def test_no_data_extracted_is_not_retried(self, mock_request, mock_sleep): - mock_request.return_value = _fake_response(422, NO_DATA) - response = ZenRowsClient(apikey, retries=3).extract(url) +def test_422_and_5xx_share_one_retry_count(server: _Server, sleeps: list[float]): + server.script = [(503, {}, b""), (422, {}, NO_CONTENT), (503, {}, b""), (200, {}, b"ok")] - self.assertEqual(response.status_code, 422) - self.assertEqual(mock_request.call_count, 1) - mock_sleep.assert_not_called() + with patch("zenrows.client.random.random", return_value=1.0): + response = client(server, retries=2).fetch(TARGET) - @patch.object(Session, "request") - def test_other_422_is_retried_up_to_retries(self, mock_request, mock_sleep): - mock_request.return_value = _fake_response(422, NO_CONTENT) + assert response.status_code == 503 + assert server.calls == 3 + assert sleeps == [1.0, 2.0] # the attempt index runs on across both kinds - response = ZenRowsClient(apikey, retries=3).fetch(url) - self.assertEqual(response.status_code, 422) - self.assertEqual(mock_request.call_count, 4) # 1 + 3 retries - self.assertEqual([c.args[0] for c in mock_sleep.call_args_list], [1.0, 2.0]) +def test_read_timeout_is_not_retried(server: _Server, sleeps: list[float]): + server.script = [HANG] - @patch.object(Session, "request") - def test_other_422_stops_retrying_on_success(self, mock_request, mock_sleep): - mock_request.side_effect = [_fake_response(422, NO_CONTENT), _fake_response(200, b"ok")] + with pytest.raises(requests.ReadTimeout): + client(server, retries=3).fetch(TARGET, timeout=0.1) - response = ZenRowsClient(apikey, retries=3).fetch(url) + assert server.calls == 1 - self.assertEqual(response.status_code, 200) - self.assertEqual(mock_request.call_count, 2) - @patch.object(Session, "request") - def test_422_not_retried_without_retries(self, mock_request, mock_sleep): - mock_request.return_value = _fake_response(422, NO_CONTENT) +def test_connection_dropped_after_sending_is_not_retried(server: _Server, sleeps: list[float]): + server.script = [DROP, (200, {}, b"ok")] - ZenRowsClient(apikey).fetch(url) + with pytest.raises(requests.ConnectionError): + client(server, retries=3).fetch(TARGET) - self.assertEqual(mock_request.call_count, 1) + assert server.calls == 1 - @patch.object(Session, "request") - def test_no_data_extracted_is_not_retried_async(self, mock_request, mock_sleep): - mock_request.return_value = _fake_response(422, NO_DATA) - response = asyncio.run(ZenRowsClient(apikey, retries=3).extract_async(url)) +@pytest.mark.parametrize( + ("verb", "params", "retried"), + [ + ("fetch", None, 2), + ("fetch", {"original_status": "true", "allowed_status_codes": "503"}, 2), + ("post", None, 0), + ("put", None, 0), + ], +) +def test_failure_to_connect_is_retried_for_a_get_only( + sleeps: list[float], verb: str, params: dict | None, retried: int +): + zr = ZenRowsClient(APIKEY, base_url=closed_port_url(), retries=2) - self.assertEqual(response.status_code, 422) - self.assertEqual(mock_request.call_count, 1) + with pytest.raises(requests.ConnectionError): + getattr(zr, verb)(TARGET, params=params) + + assert len(sleeps) == retried + + +# ---- which requests are sent again ---- + + +@pytest.mark.parametrize("status", [429, 500, 503, 422]) +@pytest.mark.parametrize("headers", [None, KEY]) +@pytest.mark.parametrize("verb", ["post", "put"]) +def test_post_and_put_are_sent_once( + server: _Server, sleeps: list[float], verb: str, headers: dict | None, status: int +): + server.script = [(status, {}, NO_CONTENT), (200, {}, b"ok")] + + response = getattr(client(server, retries=3), verb)(TARGET, headers=headers, data="payload") + + # An `Idempotency-Key` changes nothing: the API does not deduplicate on it. + assert response.status_code == status + assert server.bodies == [b"payload"] + assert sleeps == [] + + +@pytest.mark.parametrize("status", [503, 422]) +@pytest.mark.parametrize( + "body", + [ + lambda: {"data": io.BytesIO(b"chunk1-chunk2")}, + lambda: {"data": (c for c in [b"chunk1-", b"chunk2"])}, + lambda: {"data": iter([b"chunk1-", b"chunk2"])}, + lambda: {"files": {"f": ("f.txt", io.BytesIO(b"chunk1-chunk2"))}}, + ], + ids=["file", "generator", "iterator", "files"], +) +def test_get_with_a_one_shot_body_is_sent_once( + server: _Server, sleeps: list[float], body: Callable[[], dict], status: int +): + server.script = [(status, {}, NO_CONTENT), (200, {}, b"ok")] + + response = client(server, retries=2).fetch(TARGET, **body()) + + assert response.status_code == status + assert len(server.bodies) == 1 + assert b"chunk1-chunk2" in server.bodies[0] + assert sleeps == [] + + +@pytest.mark.parametrize("status", [503, 422]) +@pytest.mark.parametrize( + ("data", "sent"), [("payload", b"payload"), (b"payload", b"payload"), ({"a": "1"}, b"a=1")] +) +def test_get_with_a_string_bytes_or_dict_body_is_retried( + server: _Server, sleeps: list[float], data: object, sent: bytes, status: int +): + server.script = [(status, {}, NO_CONTENT), (200, {}, b"ok")] + + response = client(server, retries=2).fetch(TARGET, data=data) + + assert response.status_code == 200 + assert server.bodies == [sent, sent] + + +# ---- statuses from the target site ---- + + +@pytest.mark.parametrize( + ("params", "status", "retried"), + [ + ({"original_status": True}, 503, False), + ({"original_status": "true"}, 422, False), + ({"original_status": "TRUE"}, 429, False), + ({"original_status": 1}, 500, False), + ({"original_status": "1"}, 504, False), + ({"original_status": False}, 503, True), + ({"original_status": "false"}, 422, True), + ({"original_status": 0}, 503, True), + ({"original_status": ""}, 503, True), + ({"allowed_status_codes": "503"}, 503, False), + ({"allowed_status_codes": " 404 , 503 "}, 503, False), + ({"allowed_status_codes": ["404", "503"]}, 503, False), + ({"allowed_status_codes": [404, 422]}, 422, False), + ({"allowed_status_codes": "404,503"}, 502, True), + ({"allowed_status_codes": "404,503"}, 422, True), + ], +) +def test_status_from_the_target_site_is_not_retried( + server: _Server, sleeps: list[float], params: dict, status: int, retried: bool +): + server.script = [(status, {}, NO_CONTENT), (200, {}, b"ok")] + + response = client(server, retries=2).fetch(TARGET, params=params) + + assert response.status_code == (200 if retried else status) + assert server.calls == (2 if retried else 1) + + +# ---- how long it waits ---- + + +@pytest.mark.parametrize( + ("value", "millis"), + [ + ("5", 5000), + ("0", 250), + (" 7 ", 7000), + ("99999", 30000), + pytest.param("9" * 5000, 30000, id="5000-nines-30000"), + ("100", 30000), + ("0005", 5000), + ("000", 250), + ("1.5", None), + ("-1", None), + ("1e3", None), + ("0x10", None), + ("", None), + ("abc", None), + ("Wed, 21 Oct 2026 07:28:00 GMT", None), + ], +) +def test_retry_after_conformance(value: str, millis: int | None): + seconds = _retry_after_seconds(value) + + assert (None if seconds is None else round(seconds * 1000)) == millis + + +@pytest.mark.parametrize( + ("attempt", "retry_max_wait", "low", "high"), + [ + (0, 30.0, 250, 1000), + (1, 30.0, 250, 2000), + (2, 30.0, 250, 4000), + (5, 30.0, 250, 30000), + (9, 30.0, 250, 30000), + (4, 5.0, 250, 5000), + (0, 0.1, 250, 250), + (6, 0.1, 250, 250), + ], +) +def test_jitter_conformance(attempt: int, retry_max_wait: float, low: int, high: int): + with patch("zenrows.client.random.random", return_value=0.0): + assert round(_jitter(attempt, retry_max_wait) * 1000) == low + with patch("zenrows.client.random.random", return_value=1.0): + assert round(_jitter(attempt, retry_max_wait) * 1000) == high + + +def test_jitter_ceiling_doubles_per_retry(server: _Server, sleeps: list[float]): + server.script = [(503, {}, b"")] * 4 + [(200, {}, b"ok")] + + with patch("zenrows.client.random.random", return_value=1.0): + client(server, retries=4).fetch(TARGET) + + assert sleeps == [1.0, 2.0, 4.0, 8.0] + + +@pytest.mark.parametrize( + ("status", "value", "waited"), + [ + (503, "20", 20.0), # above retry_max_wait=5: Retry-After has its own cap + (429, "45", 30.0), + (422, "4", 4.0), + (503, "1.5", 0.625), # not whole seconds: the jittered wait instead + (503, "Wed, 21 Oct 2099 07:28:00 GMT", 0.625), + ], +) +def test_retry_after_sets_the_wait( + server: _Server, sleeps: list[float], status: int, value: str, waited: float +): + server.script = [(status, {"Retry-After": value}, NO_CONTENT), (200, {}, b"ok")] + + with patch("zenrows.client.random.random", return_value=0.5): + response = client(server, retries=2, retry_max_wait=5).fetch(TARGET) + + assert response.status_code == 200 + assert sleeps == [waited] + + +# ---- the retry budget ---- + + +@pytest.fixture +def clock() -> Iterator[tuple[list[float], list[float]]]: + """A fake clock for the budget that every wait moves on: ([now], waits).""" + now = [0.0] + waited: list[float] = [] + + def wait(seconds: float) -> None: + waited.append(seconds) + now[0] += seconds + + with patch("zenrows.client._monotonic", lambda: now[0]), patch("time.sleep", wait): + yield now, waited + + +@pytest.mark.parametrize( + ("retry_budget", "first_try_takes", "waits"), + [ + (90.0, 0.0, [20.0] * 4), # a fifth wait would end at 100 s + (90.0, 75.0, []), # the budget counts the first try + (15.0, 0.0, []), + ], +) +def test_no_retry_starts_past_the_budget( + server: _Server, clock: tuple, retry_budget: float, first_try_takes: float, waits: list +): + now, waited = clock + server.script = [(503, {"Retry-After": "20"}, b"")] + + def slow_first_try() -> None: + if server.calls == 1: + now[0] += first_try_takes + + server.on_request = slow_first_try + + response = client(server, retries=10, retry_budget=retry_budget).fetch(TARGET) + + assert response.status_code == 503 + assert waited == waits + assert server.calls == len(waits) + 1 + + +def test_each_async_call_has_its_own_budget(server: _Server, clock: tuple): + _, waited = clock + zr = client(server, retries=1, retry_budget=40.0) + + async def two_calls() -> list[int]: + statuses = [] + for _ in range(2): + server.script = [(503, {"Retry-After": "30"}, b""), (200, {}, b"ok")] + statuses.append((await zr.fetch_async(TARGET)).status_code) + return statuses + + assert asyncio.run(two_calls()) == [200, 200] + assert waited == [30.0, 30.0] + + +# ---- is_no_data_extracted ---- + + +def _fake_response(status_code: int, body: bytes = b"") -> Response: + response = Response() + response.status_code = status_code + response._content = body + return response class TestIsNoDataExtracted(TestCase):