From 8626e595ad2fed49d779fc12da029e161ea821a5 Mon Sep 17 00:00:00 2001 From: Presend Date: Sun, 27 Sep 2026 20:00:30 +0200 Subject: [PATCH 1/2] fix(ci): repair the weekly trusted-packages download --- .github/workflows/weekly_download.yml | 1 - dependencies/scripts/download_packages.py | 3 +-- dependencies/scripts/utils.py | 5 ++--- dependencies/tests/test_download_packages.py | 4 +--- 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/weekly_download.yml b/.github/workflows/weekly_download.yml index daa561b..7be32ad 100644 --- a/.github/workflows/weekly_download.yml +++ b/.github/workflows/weekly_download.yml @@ -37,7 +37,6 @@ jobs: run: uv sync --locked --only-group download --python 3.14 - name: Download ${{ matrix.name }} packages - continue-on-error: true run: | PYTHONPATH=dependencies/ uv run --no-project dependencies/scripts/download_packages.py download ${{ matrix.name }} diff --git a/dependencies/scripts/download_packages.py b/dependencies/scripts/download_packages.py index 851a64e..7d08407 100644 --- a/dependencies/scripts/download_packages.py +++ b/dependencies/scripts/download_packages.py @@ -9,9 +9,8 @@ import click import httpx import stamina -from requests.exceptions import InvalidJSONError -from scripts.exceptions import ServerError +from scripts.exceptions import InvalidJSONError, ServerError from scripts.utils import ( DEPENDENCIES_DIR, ECOSYSTEMS, diff --git a/dependencies/scripts/utils.py b/dependencies/scripts/utils.py index 9ec62c9..ef13baa 100644 --- a/dependencies/scripts/utils.py +++ b/dependencies/scripts/utils.py @@ -3,15 +3,14 @@ from typing import Any import httpx -from requests.exceptions import InvalidJSONError -from scripts.exceptions import ServerError +from scripts.exceptions import InvalidJSONError, ServerError DEPENDENCIES_DIR = "dependencies" """Directory name where dependency files will be saved.""" # Sources -TOP_PYPI_SOURCE = "https://hugovk.github.io/top-pypi-packages/top-pypi-packages.min.json" +TOP_PYPI_SOURCE = "https://hugovk.dev/top-pypi-packages/top-pypi-packages.min.json" """URL for fetching top PyPI packages data.""" TOP_NPM_SOURCE = "https://packages.ecosyste.ms/api/v1/registries/npmjs.org/packages" diff --git a/dependencies/tests/test_download_packages.py b/dependencies/tests/test_download_packages.py index d026cc4..4ffb510 100644 --- a/dependencies/tests/test_download_packages.py +++ b/dependencies/tests/test_download_packages.py @@ -100,9 +100,7 @@ def test_pypi_download(self) -> None: # Check the HTTP request with its parameters assert m_client.call_count == 1 - assert m_client.call_args == call( - "https://hugovk.github.io/top-pypi-packages/top-pypi-packages.min.json", params={} - ) + assert m_client.call_args == call("https://hugovk.dev/top-pypi-packages/top-pypi-packages.min.json", params={}) # Check the file path assert m_open.call_args_list[0] == call(str(Path(DEPENDENCIES_DIR) / "pypi.json"), "w") From f5a2c4fd0c1f04e2e52c4c817785515b7c3842a9 Mon Sep 17 00:00:00 2001 From: Presend Date: Thu, 1 Oct 2026 15:15:58 +0200 Subject: [PATCH 2/2] fix(ci): follow redirects and raise non-5xx HTTP errors in the download script A source that moves (as top-pypi-packages did) is now followed, and any other non-success status fails with httpx's own message (status and URL) instead of falling through to response.json() on an empty body. Both new tests fail without the change. --- dependencies/scripts/download_packages.py | 3 +- dependencies/tests/test_download_packages.py | 37 ++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/dependencies/scripts/download_packages.py b/dependencies/scripts/download_packages.py index 7d08407..ee60e88 100644 --- a/dependencies/scripts/download_packages.py +++ b/dependencies/scripts/download_packages.py @@ -48,7 +48,7 @@ def _run(ecosystem: str) -> None: all_packages: set[str] = set() n_pages = selected_ecosystem.pages or 1 - with httpx.Client(timeout=TIMEOUT) as client: + with httpx.Client(timeout=TIMEOUT, follow_redirects=True) as client: for page in range(1, n_pages + 1): params = get_params(selected_ecosystem.params, page if selected_ecosystem.pages else None) all_packages.update(get_packages(client, selected_ecosystem.url, selected_ecosystem.parser, params)) @@ -100,6 +100,7 @@ def get_packages( except httpx.HTTPStatusError as e: if e.response.is_server_error: raise ServerError from e + raise try: json_data = response.json() except json.JSONDecodeError as e: diff --git a/dependencies/tests/test_download_packages.py b/dependencies/tests/test_download_packages.py index 4ffb510..53076a3 100644 --- a/dependencies/tests/test_download_packages.py +++ b/dependencies/tests/test_download_packages.py @@ -1,6 +1,7 @@ import json from collections.abc import Iterator from contextlib import contextmanager +from dataclasses import replace from pathlib import Path from typing import Any from unittest.mock import Mock, call, patch @@ -185,6 +186,42 @@ def test_retry_mechanism_with_server_errors(self) -> None: assert mock_client.call_count == RETRY_ATTEMPTS + def test_client_error_is_raised(self) -> None: + """Test that a non-5xx HTTP error is raised as is, without retrying or parsing the body.""" + mock_response = Mock() + mock_response.is_server_error = False + mock_response.status_code = 404 + + client_error = httpx.HTTPStatusError("Not Found", request=Mock(), response=mock_response) + + with patch_client_error(client_error) as mock_client: + with pytest.raises(httpx.HTTPStatusError): + _run("pypi") + + assert mock_client.call_count == 1 + mock_client.return_value.json.assert_not_called() + + def test_redirect_is_followed(self) -> None: + """Test that a source that moved is followed to its new location.""" + data = {"rows": [{"project": "requests"}, {"project": "numpy"}]} + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.host == "old.example": + return httpx.Response(301, headers={"Location": "https://new.example/top.json"}) + return httpx.Response(200, json=data) + + moved = replace(ECOSYSTEMS["pypi"], url="https://old.example/top.json") + with ( + patch.dict(ECOSYSTEMS, {"pypi": moved}), + patch("httpx.HTTPTransport.handle_request", side_effect=handler) as m_transport, + patch_save_to_file() as m_save, + patch_open_file(), + ): + _run("pypi") + + assert [c.args[0].url.host for c in m_transport.call_args_list] == ["old.example", "new.example"] + assert set(m_save.call_args[0][0]["packages"]) == {"requests", "numpy"} + def test_npm_download_with_multiple_pages(self) -> None: """Test that the script will iterate through pages if provided.""" page1_data = [