diff --git a/.github/workflows/weekly_download.yml b/.github/workflows/weekly_download.yml index daa561b8..7be32ad4 100644 --- a/.github/workflows/weekly_download.yml +++ b/.github/workflows/weekly_download.yml @@ -37,7 +37,6 @@ jobs: run: uv sync --locked --only-group download --python 3.14 - name: Download ${{ matrix.name }} packages - continue-on-error: true run: | PYTHONPATH=dependencies/ uv run --no-project dependencies/scripts/download_packages.py download ${{ matrix.name }} diff --git a/dependencies/scripts/download_packages.py b/dependencies/scripts/download_packages.py index 851a64e7..ee60e88d 100644 --- a/dependencies/scripts/download_packages.py +++ b/dependencies/scripts/download_packages.py @@ -9,9 +9,8 @@ import click import httpx import stamina -from requests.exceptions import InvalidJSONError -from scripts.exceptions import ServerError +from scripts.exceptions import InvalidJSONError, ServerError from scripts.utils import ( DEPENDENCIES_DIR, ECOSYSTEMS, @@ -49,7 +48,7 @@ def _run(ecosystem: str) -> None: all_packages: set[str] = set() n_pages = selected_ecosystem.pages or 1 - with httpx.Client(timeout=TIMEOUT) as client: + with httpx.Client(timeout=TIMEOUT, follow_redirects=True) as client: for page in range(1, n_pages + 1): params = get_params(selected_ecosystem.params, page if selected_ecosystem.pages else None) all_packages.update(get_packages(client, selected_ecosystem.url, selected_ecosystem.parser, params)) @@ -101,6 +100,7 @@ def get_packages( except httpx.HTTPStatusError as e: if e.response.is_server_error: raise ServerError from e + raise try: json_data = response.json() except json.JSONDecodeError as e: diff --git a/dependencies/scripts/utils.py b/dependencies/scripts/utils.py index 9ec62c97..ef13baa3 100644 --- a/dependencies/scripts/utils.py +++ b/dependencies/scripts/utils.py @@ -3,15 +3,14 @@ from typing import Any import httpx -from requests.exceptions import InvalidJSONError -from scripts.exceptions import ServerError +from scripts.exceptions import InvalidJSONError, ServerError DEPENDENCIES_DIR = "dependencies" """Directory name where dependency files will be saved.""" # Sources -TOP_PYPI_SOURCE = "https://hugovk.github.io/top-pypi-packages/top-pypi-packages.min.json" +TOP_PYPI_SOURCE = "https://hugovk.dev/top-pypi-packages/top-pypi-packages.min.json" """URL for fetching top PyPI packages data.""" TOP_NPM_SOURCE = "https://packages.ecosyste.ms/api/v1/registries/npmjs.org/packages" diff --git a/dependencies/tests/test_download_packages.py b/dependencies/tests/test_download_packages.py index d026cc44..53076a3e 100644 --- a/dependencies/tests/test_download_packages.py +++ b/dependencies/tests/test_download_packages.py @@ -1,6 +1,7 @@ import json from collections.abc import Iterator from contextlib import contextmanager +from dataclasses import replace from pathlib import Path from typing import Any from unittest.mock import Mock, call, patch @@ -100,9 +101,7 @@ def test_pypi_download(self) -> None: # Check the HTTP request with its parameters assert m_client.call_count == 1 - assert m_client.call_args == call( - "https://hugovk.github.io/top-pypi-packages/top-pypi-packages.min.json", params={} - ) + assert m_client.call_args == call("https://hugovk.dev/top-pypi-packages/top-pypi-packages.min.json", params={}) # Check the file path assert m_open.call_args_list[0] == call(str(Path(DEPENDENCIES_DIR) / "pypi.json"), "w") @@ -187,6 +186,42 @@ def test_retry_mechanism_with_server_errors(self) -> None: assert mock_client.call_count == RETRY_ATTEMPTS + def test_client_error_is_raised(self) -> None: + """Test that a non-5xx HTTP error is raised as is, without retrying or parsing the body.""" + mock_response = Mock() + mock_response.is_server_error = False + mock_response.status_code = 404 + + client_error = httpx.HTTPStatusError("Not Found", request=Mock(), response=mock_response) + + with patch_client_error(client_error) as mock_client: + with pytest.raises(httpx.HTTPStatusError): + _run("pypi") + + assert mock_client.call_count == 1 + mock_client.return_value.json.assert_not_called() + + def test_redirect_is_followed(self) -> None: + """Test that a source that moved is followed to its new location.""" + data = {"rows": [{"project": "requests"}, {"project": "numpy"}]} + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.host == "old.example": + return httpx.Response(301, headers={"Location": "https://new.example/top.json"}) + return httpx.Response(200, json=data) + + moved = replace(ECOSYSTEMS["pypi"], url="https://old.example/top.json") + with ( + patch.dict(ECOSYSTEMS, {"pypi": moved}), + patch("httpx.HTTPTransport.handle_request", side_effect=handler) as m_transport, + patch_save_to_file() as m_save, + patch_open_file(), + ): + _run("pypi") + + assert [c.args[0].url.host for c in m_transport.call_args_list] == ["old.example", "new.example"] + assert set(m_save.call_args[0][0]["packages"]) == {"requests", "numpy"} + def test_npm_download_with_multiple_pages(self) -> None: """Test that the script will iterate through pages if provided.""" page1_data = [