From 6be6bad48de2938839a18f22e8badc359d238a89 Mon Sep 17 00:00:00 2001 From: matthewberry Date: Wed, 4 Nov 2026 10:59:53 -0600 Subject: [PATCH 1/2] feat: return full CSV results up to 100,000 rows; cap JSON limit at 5,000 Same behavior as CLEANDB-api#20. CSV is not paginated: without a limit it returns every match, streamed through a server-side cursor into a spooled temp file, and a query matching more than 100,000 records gets a 400 instead of a partial file. An explicit JSON limit above the 5,000-record page size gets a 422. CSV skips the COUNT query. This replaces #8's silent cap on both formats and its CSV Link/X-Total-Count headers. --- app/db/database.py | 11 +- app/db/queries.py | 32 ++++-- app/main.py | 13 ++- app/routers/data.py | 186 ++++++++++++++++++++------------- tests/conftest.py | 5 + tests/test_auto_pagination.py | 63 ++---------- tests/test_data_csv.py | 187 ++++++++++++++++++++++++++++++++++ 7 files changed, 358 insertions(+), 139 deletions(-) create mode 100644 tests/test_data_csv.py diff --git a/app/db/database.py b/app/db/database.py index 93c1eb0..79b762e 100644 --- a/app/db/database.py +++ b/app/db/database.py @@ -1,4 +1,4 @@ -from typing import Any, AsyncGenerator, Dict, List, Optional +from typing import Any, AsyncGenerator, AsyncIterator, Dict, List, Optional import asyncpg from loguru import logger @@ -66,6 +66,15 @@ async def fetch(self, query: str, *args, **kwargs) -> List[Dict[str, Any]]: records = await self.pool.fetch(query, *args, **kwargs) return [dict(record) for record in records] + async def iterate(self, query: str, *args) -> AsyncIterator[Dict[str, Any]]: + """Stream rows from a query through a server-side cursor.""" + if not self.pool: + await self.connect() + # asyncpg cursors only work inside a transaction + async with self.pool.acquire() as conn, conn.transaction(): + async for record in conn.cursor(query, *args, prefetch=1000): + yield dict(record) + async def fetchval(self, query: str, *args, **kwargs) -> Any: """Fetch a single value from a query.""" if not self.pool: diff --git a/app/db/queries.py b/app/db/queries.py index 9cf96c4..1cb9fe5 100644 --- a/app/db/queries.py +++ b/app/db/queries.py @@ -1,4 +1,5 @@ -from typing import Any, Dict, List, Tuple +from contextlib import aclosing +from typing import Any, AsyncIterator, Dict, List, Tuple from app.db.database import Database from app.models.oed_data import OEDColumn @@ -101,10 +102,8 @@ async def build_query_conditions( return where_clause, query_params -async def get_filtered_data( - db: Database, params: OEDDataQueryParams -) -> List[Dict[str, Any]]: - """Get filtered data from the database.""" +async def build_data_query(params: OEDDataQueryParams) -> Tuple[str, List[Any]]: + """Build the filtered data query and its arguments.""" where_clause, query_params = await build_query_conditions(params) # Determine which columns to select, default to all columns @@ -136,9 +135,26 @@ async def get_filtered_data( # Extract query parameters from the dictionary query_args = list(query_params.values()) - # Execute the query - records = await db.fetch(query, *query_args) - return records + return query, query_args + + +async def get_filtered_data( + db: Database, params: OEDDataQueryParams +) -> List[Dict[str, Any]]: + """Get filtered data from the database.""" + query, query_args = await build_data_query(params) + return await db.fetch(query, *query_args) + + +async def iter_filtered_data( + db: Database, params: OEDDataQueryParams +) -> AsyncIterator[Dict[str, Any]]: + """Stream filtered data from the database.""" + query, query_args = await build_data_query(params) + # Closing this generator early closes the cursor too, releasing its connection + async with aclosing(db.iterate(query, *query_args)) as records: + async for record in records: + yield record async def get_total_count(db: Database, params: OEDDataQueryParams) -> int: diff --git a/app/main.py b/app/main.py index bb46b4f..4803aeb 100644 --- a/app/main.py +++ b/app/main.py @@ -46,11 +46,14 @@ async def lifespan(app: FastAPI): ### Technical Note: Automatic Pagination -When a query would return more than {settings.AUTO_PAGINATION_THRESHOLD} records and no explicit limit, or one of -at least {settings.AUTO_PAGINATION_THRESHOLD}, is provided, the API will automatically paginate results to return -{settings.AUTO_PAGINATION_THRESHOLD} records at a time. A JSON response will include pagination metadata with links to -navigate to next and previous pages. A CSV response carries the same links in a `Link` header, and the total -number of matching records in an `X-Total-Count` header. +When a JSON query would return more than {settings.AUTO_PAGINATION_THRESHOLD} records and no explicit limit is +provided, the API will automatically paginate results to return {settings.AUTO_PAGINATION_THRESHOLD} records +at a time. The response will include pagination metadata with links to navigate to next and previous pages. +An explicit JSON limit can be at most {settings.AUTO_PAGINATION_THRESHOLD}. + +CSV responses are not paginated. Without an explicit limit, a CSV query returns every matching +record, up to {data.MAX_DATA_LIMIT}; a query that matches more is rejected with a 400 asking +you to narrow the filters or page with `limit` and `offset`. ## Data Download diff --git a/app/routers/data.py b/app/routers/data.py index 59a8484..aff84d1 100644 --- a/app/routers/data.py +++ b/app/routers/data.py @@ -1,20 +1,75 @@ import csv -from io import StringIO -from typing import Any, List, Optional +from contextlib import aclosing +from tempfile import SpooledTemporaryFile +from typing import Any, AsyncIterator, Dict, Iterator, List, Optional from urllib.parse import urlencode from fastapi import APIRouter, Depends, HTTPException, Query, Request +from fastapi.exceptions import RequestValidationError from fastapi.responses import StreamingResponse from loguru import logger +from starlette.background import BackgroundTask from app.core.config import settings from app.db.database import Database, get_db -from app.db.queries import get_filtered_data, get_total_count +from app.db.queries import get_filtered_data, get_total_count, iter_filtered_data from app.models.query_params import OEDDataQueryParams, ResponseFormat # Largest value Postgres accepts for LIMIT and OFFSET MAX_BIGINT = 2**63 - 1 +# Most records one /data response may contain, as an explicit limit or a whole CSV +MAX_DATA_LIMIT = 100_000 + +# CSV output stays in memory up to this size, then spills to a temp file on disk +CSV_SPOOL_MAX_MEMORY = 8 * 1024 * 1024 + + +async def write_csv( + rows: AsyncIterator[Dict[str, Any]], + fieldnames: Optional[List[str]], + max_rows: Optional[int] = None, +) -> SpooledTemporaryFile: + """Write rows as CSV to a spooled temp file, rewound for reading. + + The columns are fieldnames, or the first row's keys if fieldnames is None. The whole + result is written before the response starts, so the DB connection is released at + the speed of the DB, not the speed of the client reading the response. Raises a 400 + if rows has more than max_rows rows. + """ + output = SpooledTemporaryFile(max_size=CSV_SPOOL_MAX_MEMORY, mode="w+", newline="") + try: + writer = None + if fieldnames: + writer = csv.DictWriter(output, fieldnames=fieldnames, extrasaction="ignore") + writer.writeheader() + # aclosing releases the DB connection now if we stop early, not at garbage collection + async with aclosing(rows): + count = 0 + async for row in rows: + count += 1 + if max_rows is not None and count > max_rows: + raise HTTPException( + status_code=400, + detail=f"Query matches more than {max_rows} records, the most a CSV " + "response can contain. Narrow the filters, or page with limit and offset.", + ) + if writer is None: + writer = csv.DictWriter(output, fieldnames=list(row.keys()), extrasaction="ignore") + writer.writeheader() + writer.writerow(row) + except BaseException: + output.close() + raise + output.seek(0) + return output + + +def read_chunks(file: SpooledTemporaryFile, size: int = 64 * 1024) -> Iterator[str]: + """Read a file in chunks.""" + while chunk := file.read(size): + yield chunk + router = APIRouter(tags=["Data"]) @@ -51,13 +106,28 @@ def parse_query_params( None, description="Columns to include in the response" ), limit: Optional[int] = Query( - None, ge=1, le=MAX_BIGINT, description="Maximum number of records to return" + None, ge=1, le=MAX_DATA_LIMIT, + description="Maximum number of records to return " + f"(at most {settings.AUTO_PAGINATION_THRESHOLD} for JSON, {MAX_DATA_LIMIT} for CSV)", ), offset: Optional[int] = Query( 0, ge=0, le=MAX_BIGINT, description="Number of records to skip" ), ) -> OEDDataQueryParams: """Parse and validate query parameters.""" + # JSON is paginated, so a JSON page is never larger than the default page size. + # Shaped like the le= error on limit, so both caps fail the same way. + max_json_limit = settings.AUTO_PAGINATION_THRESHOLD + if format == ResponseFormat.JSON and limit is not None and limit > max_json_limit: + raise RequestValidationError([{ + "type": "less_than_equal", + "loc": ("query", "limit"), + "msg": f"Input should be less than or equal to {max_json_limit} for JSON responses; " + "page with offset, or use format=csv", + "input": str(limit), + "ctx": {"le": max_json_limit}, + }]) + try: # Validate format if format not in [fmt for fmt in ResponseFormat]: @@ -98,9 +168,10 @@ async def get_data( The response format can be either JSON (default) or CSV. -Results are automatically paginated when they exceed the configured threshold, unless -an explicit limit below the threshold is provided. CSV responses carry the pagination -links in a `Link` header and the total in an `X-Total-Count` header. +JSON results are automatically paginated when they exceed the configured threshold, unless +an explicit limit is provided; an explicit JSON limit can be at most the threshold. CSV +results are not paginated: without an explicit limit, every matching record is returned, +up to 100,000, and a query matching more gets a 400. ### URL examples @@ -195,13 +266,30 @@ async def get_data( """ try: + # CSV is not paginated: without an explicit limit, return every match up to the cap + if params.format == ResponseFormat.CSV: + if params.limit is None: + # Fetch one row past the cap, so a result that is too large is detected + # while writing instead of silently cut off + rows = iter_filtered_data(db, params.model_copy(update={"limit": MAX_DATA_LIMIT + 1})) + csv_file = await write_csv(rows, params.columns, max_rows=MAX_DATA_LIMIT) + else: + csv_file = await write_csv(iter_filtered_data(db, params), params.columns) + return StreamingResponse( + read_chunks(csv_file), + media_type="text/csv", + headers={"Content-Disposition": "attachment; filename=oed_data.csv"}, + # Runs after the response ends, including when the client disconnects + background=BackgroundTask(csv_file.close), + ) + # Get total count for the query (without pagination) total_count = await get_total_count(db, params) # Apply automatic pagination if results exceed threshold and no explicit limit, - # or one at least as large, was provided. Without the cap one request could load - # the whole table into memory. ">=" because the `next` links below carry - # limit=threshold, and those pages need links of their own. + # or one equal to it, was provided. A larger one was rejected when parsing. + # Equal counts because the `next` links below carry limit=threshold, and those + # pages need links of their own. if total_count > settings.AUTO_PAGINATION_THRESHOLD and ( params.limit is None or params.limit >= settings.AUTO_PAGINATION_THRESHOLD ): @@ -214,7 +302,7 @@ async def get_data( # Get data from database (now with potential auto-pagination applied) data = await get_filtered_data(db, params) - # Pagination links, for both response formats + # Pagination links next_url = None previous_url = None if params.auto_paginated and request: @@ -229,10 +317,6 @@ async def get_data( and v is not None } - # Set format explicitly if it was provided - if params.format != ResponseFormat.JSON: - query_params["format"] = params.format - current_offset = params.offset or 0 current_limit = params.limit or total_count @@ -257,66 +341,26 @@ async def get_data( } previous_url = f"{base_url}?{urlencode(prev_params, doseq=True)}" - # Handle response format - if params.format == ResponseFormat.CSV: - # Create CSV response - output = StringIO() - - # Determine columns to include in CSV - if params.columns: - fieldnames = params.columns - elif data and len(data) > 0: - fieldnames = list(data[0].keys()) - else: - fieldnames = [] - - # Write CSV - writer = csv.DictWriter( - output, fieldnames=fieldnames, extrasaction="ignore" - ) - writer.writeheader() - for row in data: - writer.writerow(row) - - # A CSV body can't carry pagination metadata, so it goes in headers - headers = { - "Content-Disposition": "attachment; filename=oed_data.csv", - "X-Total-Count": str(total_count), - } - links = [] + # JSON response with enhanced pagination info + response = { + "total": total_count, + "offset": params.offset, + "limit": params.limit if params.limit is not None else total_count, + "data": data, + } + + if params.auto_paginated: + # Add flag indicating automatic pagination was applied + response["auto_paginated"] = True if next_url: - links.append(f'<{next_url}>; rel="next"') + response["next"] = next_url if previous_url: - links.append(f'<{previous_url}>; rel="prev"') - if links: - headers["Link"] = ", ".join(links) - - # Return streaming response - output.seek(0) - return StreamingResponse( - iter([output.getvalue()]), - media_type="text/csv", - headers=headers, - ) - else: - # JSON response with enhanced pagination info - response = { - "total": total_count, - "offset": params.offset, - "limit": params.limit if params.limit is not None else total_count, - "data": data, - } - - if params.auto_paginated: - # Add flag indicating automatic pagination was applied - response["auto_paginated"] = True - if next_url: - response["next"] = next_url - if previous_url: - response["previous"] = previous_url + response["previous"] = previous_url - return response + return response + except HTTPException: + raise except Exception as e: logger.error(f"Error getting data: {e}") raise HTTPException(status_code=500, detail=f"Error retrieving data: {str(e)}") diff --git a/tests/conftest.py b/tests/conftest.py index 5ad7e22..8a14bd5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -44,6 +44,11 @@ async def fetch(self, query: str, *args, **kwargs) -> List[Dict[str, Any]]: return [] # Return empty list as a default response + async def iterate(self, query: str, *args): + """Capture query and stream the same result set fetch would return.""" + for record in await self.fetch(query, *args): + yield record + async def fetchval(self, query: str, *args, **kwargs) -> Any: """Capture query and return a default value.""" self.executed_queries.append({"query": query, "args": args}) diff --git a/tests/test_auto_pagination.py b/tests/test_auto_pagination.py index 43cb997..50c9eee 100644 --- a/tests/test_auto_pagination.py +++ b/tests/test_auto_pagination.py @@ -1,5 +1,3 @@ -import re - import pytest from fastapi.testclient import TestClient @@ -142,37 +140,19 @@ async def test_pagination_navigation(monkeypatch, test_client, mock_db): assert offsets == [0, 10, 20] assert "offset=10" in data["previous"] -@pytest.mark.asyncio -@pytest.mark.parametrize( - "mock_db", - [ - { - "fetchval": lambda self, query, *args: 100, - "fetch": lambda self, query, *args: [{"id": i, "ec": f"1.1.1.{i}", "organism": f"Test Organism {i}"} for i in range(10)] - } - ], - indirect=True -) -async def test_explicit_limit_above_threshold_is_capped(monkeypatch, test_client, mock_db, sql_capture): - """Test that an explicit limit larger than the threshold is reduced to the threshold. +@pytest.mark.parametrize("limit, status", [(10, 200), (11, 422)]) +def test_json_limit_is_at_most_the_page_size(monkeypatch, test_client, sql_capture, limit, status): + """Test that a JSON limit above the page size is rejected before any SQL runs. - Without the cap, one request can load the whole table into memory.""" + JSON is paginated, so nothing becomes unreachable; CSV takes larger limits.""" monkeypatch.setattr(settings, "AUTO_PAGINATION_THRESHOLD", 10) - response = test_client.get("/api/v1/data?limit=1000") - - assert response.status_code == 200 - data = response.json() + response = test_client.get(f"/api/v1/data?limit={limit}") - # The query itself is limited, not just the response - data_query = [q["query"] for q in sql_capture if "COUNT(*)" not in q["query"]][0] - assert re.search(r"LIMIT 10\b", data_query) - - assert data["auto_paginated"] is True - assert data["limit"] == 10 - assert data["total"] == 100 - assert "offset=10" in data["next"] - assert "limit=10" in data["next"] + assert response.status_code == status, response.text + if status == 422: + assert [e["loc"] for e in response.json()["detail"]] == [["query", "limit"]] + assert sql_capture == [] @pytest.mark.asyncio @@ -195,28 +175,3 @@ async def test_results_are_not_printed(test_client, mock_db, capsys): assert response.status_code == 200 assert "Distinctive organism" not in capsys.readouterr().out - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "mock_db", - [ - { - "fetchval": lambda self, query, *args: 100, - "fetch": lambda self, query, *args: [{"id": i, "ec": f"1.1.1.{i}", "organism": f"Test Organism {i}"} for i in range(10)] - } - ], - indirect=True -) -async def test_capped_csv_says_there_is_more(monkeypatch, test_client, mock_db): - """Test that a CSV response cut off by the cap says so in its headers. - - A CSV body has nowhere to carry pagination metadata.""" - monkeypatch.setattr(settings, "AUTO_PAGINATION_THRESHOLD", 10) - - response = test_client.get("/api/v1/data?format=csv&limit=1000") - - assert response.status_code == 200 - assert response.headers["x-total-count"] == "100" - assert 'rel="next"' in response.headers["link"] - assert "offset=10" in response.headers["link"] - assert "format=csv" in response.headers["link"] diff --git a/tests/test_data_csv.py b/tests/test_data_csv.py new file mode 100644 index 0000000..efd9fc6 --- /dev/null +++ b/tests/test_data_csv.py @@ -0,0 +1,187 @@ +import csv +from io import StringIO +from unittest.mock import AsyncMock, patch + +import pytest + +import app.routers.data as data_router +from app.core.config import settings + + +@pytest.fixture +def db_calls(): + """Mock the DB query functions; iter_filtered_data yields the rows in db_calls.rows.""" + class Calls: + rows = [] + streamed_params = [] + rows_closed = False + + async def iter_filtered_data(db, params): + Calls.streamed_params.append(params.model_copy()) + try: + for row in Calls.rows: + yield row + finally: + Calls.rows_closed = True + + with ( + patch.object(data_router, "iter_filtered_data", iter_filtered_data), + patch.object(data_router, "get_total_count", AsyncMock()) as count, + patch.object(data_router, "get_filtered_data", AsyncMock()) as fetch, + ): + Calls.get_total_count, Calls.get_filtered_data = count, fetch + yield Calls + + +def get_csv(client, query="ec=1.1.1.1"): + return client.get(f"/api/v1/data?{query}&format=csv") + + +def parse(response): + assert response.status_code == 200, response.text + return list(csv.reader(StringIO(response.text))) + + +def make_row(**overrides): + row = {"ec": "1.1.1.1", "substrate": 'ethanol, "absolute"', "organism": "Homo sapiens", "kcat_value": 7.5} + row.update(overrides) + return row + + +def test_csv_header_comes_from_the_rows(test_client, db_calls): + db_calls.rows = [make_row()] + + assert parse(get_csv(test_client)) == [ + ["ec", "substrate", "organism", "kcat_value"], + ["1.1.1.1", 'ethanol, "absolute"', "Homo sapiens", "7.5"], + ] + + +def test_csv_header_follows_requested_columns(test_client, db_calls): + db_calls.rows = [{"organism": "Homo sapiens", "ec": "1.1.1.1"}] + + assert parse(get_csv(test_client, "columns=ec&columns=organism")) == [ + ["ec", "organism"], ["1.1.1.1", "Homo sapiens"], + ] + + +def test_csv_has_header_when_no_rows_match(test_client, db_calls): + assert parse(get_csv(test_client, "columns=ec&columns=organism")) == [["ec", "organism"]] + + +def test_csv_without_limit_returns_all_rows_up_to_cap(test_client, db_calls, monkeypatch): + monkeypatch.setattr(settings, "AUTO_PAGINATION_THRESHOLD", 2) + monkeypatch.setattr(data_router, "MAX_DATA_LIMIT", 3) + db_calls.rows = [make_row(ec=f"1.1.1.{i}") for i in range(3)] + rows = parse(get_csv(test_client, "offset=7")) + + # More rows than the JSON page size, all in one file + assert [r[0] for r in rows[1:]] == ["1.1.1.0", "1.1.1.1", "1.1.1.2"] + # One row past the cap is requested, so an oversized result can be detected + assert db_calls.streamed_params[0].limit == 4 + assert db_calls.streamed_params[0].offset == 7 + db_calls.get_total_count.assert_not_awaited() + db_calls.get_filtered_data.assert_not_awaited() + + +def test_csv_without_limit_rejects_more_rows_than_cap(test_client, db_calls, monkeypatch): + monkeypatch.setattr(data_router, "MAX_DATA_LIMIT", 3) + db_calls.rows = [make_row() for _ in range(4)] + response = get_csv(test_client) + + assert response.status_code == 400 + assert response.json()["detail"].startswith("Query matches more than 3 records") + # The row iterator, and so the DB cursor, is closed as soon as the cap is passed + assert db_calls.rows_closed + + +def test_csv_with_limit_is_not_capped(test_client, db_calls, monkeypatch): + monkeypatch.setattr(data_router, "MAX_DATA_LIMIT", 3) + db_calls.rows = [make_row() for _ in range(4)] + rows = parse(get_csv(test_client, "limit=50&offset=100")) + + assert len(rows) == 5 + assert db_calls.streamed_params[0].limit == 50 + assert db_calls.streamed_params[0].offset == 100 + + +def test_csv_accepts_limits_above_json_cap(test_client, db_calls): + for limit in (settings.AUTO_PAGINATION_THRESHOLD + 1, data_router.MAX_DATA_LIMIT): + parse(get_csv(test_client, f"limit={limit}")) + + assert [p.limit for p in db_calls.streamed_params] == [ + settings.AUTO_PAGINATION_THRESHOLD + 1, data_router.MAX_DATA_LIMIT, + ] + + +def test_csv_rejects_limit_above_cap(test_client, db_calls): + response = get_csv(test_client, f"limit={data_router.MAX_DATA_LIMIT + 1}") + + assert response.status_code == 422 + assert [e["loc"] for e in response.json()["detail"]] == [["query", "limit"]] + assert db_calls.streamed_params == [] + + +def test_csv_db_error_mid_result_returns_500(test_client): + async def failing_rows(db, params): + yield make_row() + raise RuntimeError("connection lost") + + with patch.object(data_router, "iter_filtered_data", failing_rows): + response = get_csv(test_client) + + assert response.status_code == 500 + assert "connection lost" in response.json()["detail"] + + +def test_csv_spills_large_output_to_disk(test_client, db_calls): + db_calls.rows = [make_row(substrate="C" * 100_000) for _ in range(100)] + rows = parse(get_csv(test_client)) + + assert len(rows) == 101 + assert all(len(r[1]) == 100_000 for r in rows[1:]) + + +async def test_closing_iter_filtered_data_closes_the_cursor(): + from contextlib import aclosing + + from app.db.database import Database + from app.db.queries import iter_filtered_data + from app.models.query_params import OEDDataQueryParams + + closed = [] + + async def iterate(query, *args): + try: + yield make_row() + yield make_row() + finally: + closed.append(True) + + db = Database() + db.iterate = iterate + async with aclosing(iter_filtered_data(db, OEDDataQueryParams(limit=2))) as rows: + await anext(rows) + + assert closed == [True] + + +async def test_write_csv_closes_rows_when_over_cap(): + from fastapi import HTTPException + + closed = [] + + async def rows(): + try: + for _ in range(5): + yield make_row() + finally: + closed.append(True) + + # Holding a reference means only an explicit close, not garbage collection, can close it + generator = rows() + with pytest.raises(HTTPException) as error: + await data_router.write_csv(generator, fieldnames=None, max_rows=3) + + assert error.value.status_code == 400 + assert closed == [True] From 8605b9e0669f6097099b943ce0225f1506af9bf8 Mon Sep 17 00:00:00 2001 From: matthewberry Date: Wed, 4 Nov 2026 11:08:15 -0600 Subject: [PATCH 2/2] docs: say how to page CSV without a total --- app/main.py | 3 ++- app/routers/data.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/main.py b/app/main.py index 4803aeb..b05e330 100644 --- a/app/main.py +++ b/app/main.py @@ -53,7 +53,8 @@ async def lifespan(app: FastAPI): CSV responses are not paginated. Without an explicit limit, a CSV query returns every matching record, up to {data.MAX_DATA_LIMIT}; a query that matches more is rejected with a 400 asking -you to narrow the filters or page with `limit` and `offset`. +you to narrow the filters or page with `limit` and `offset`. A CSV response carries no total, +so when paging CSV with `limit`, stop at the first response with fewer than `limit` records. ## Data Download diff --git a/app/routers/data.py b/app/routers/data.py index 64daaaa..3ca5417 100644 --- a/app/routers/data.py +++ b/app/routers/data.py @@ -171,7 +171,9 @@ async def get_data( JSON results are automatically paginated when they exceed the configured threshold, unless an explicit limit is provided; an explicit JSON limit can be at most the threshold. CSV results are not paginated: without an explicit limit, every matching record is returned, -up to 100,000, and a query matching more gets a 400. +up to 100,000, and a query matching more gets a 400. A CSV response carries no total, +so when paging CSV with an explicit limit, stop at the first response with fewer than +limit records. ### URL examples