fix(ci): repair the weekly trusted-packages download

.github/workflows/weekly_download.yml

@@ -37,7 +37,6 @@ jobs:
         run: uv sync --locked --only-group download --python 3.14
 
       - name: Download ${{ matrix.name }} packages
-        continue-on-error: true
         run: |
           PYTHONPATH=dependencies/ uv run --no-project dependencies/scripts/download_packages.py download ${{ matrix.name }}
 

dependencies/scripts/download_packages.py

@@ -9,9 +9,8 @@
 import click
 import httpx
 import stamina
-from requests.exceptions import InvalidJSONError
 
-from scripts.exceptions import ServerError
+from scripts.exceptions import InvalidJSONError, ServerError
 from scripts.utils import (
     DEPENDENCIES_DIR,
     ECOSYSTEMS,
@@ -49,7 +48,7 @@ def _run(ecosystem: str) -> None:
     all_packages: set[str] = set()
 
     n_pages = selected_ecosystem.pages or 1
-    with httpx.Client(timeout=TIMEOUT) as client:
+    with httpx.Client(timeout=TIMEOUT, follow_redirects=True) as client:
         for page in range(1, n_pages + 1):
             params = get_params(selected_ecosystem.params, page if selected_ecosystem.pages else None)
             all_packages.update(get_packages(client, selected_ecosystem.url, selected_ecosystem.parser, params))
@@ -101,6 +100,7 @@ def get_packages(
             except httpx.HTTPStatusError as e:
                 if e.response.is_server_error:
                     raise ServerError from e
+                raise
     try:
         json_data = response.json()
     except json.JSONDecodeError as e:

dependencies/scripts/utils.py

@@ -3,15 +3,14 @@
 from typing import Any
 
 import httpx
-from requests.exceptions import InvalidJSONError
 
-from scripts.exceptions import ServerError
+from scripts.exceptions import InvalidJSONError, ServerError
 
 DEPENDENCIES_DIR = "dependencies"
 """Directory name where dependency files will be saved."""
 
 # Sources
-TOP_PYPI_SOURCE = "https://hugovk.github.io/top-pypi-packages/top-pypi-packages.min.json"
+TOP_PYPI_SOURCE = "https://hugovk.dev/top-pypi-packages/top-pypi-packages.min.json"
 """URL for fetching top PyPI packages data."""
 
 TOP_NPM_SOURCE = "https://packages.ecosyste.ms/api/v1/registries/npmjs.org/packages"

dependencies/tests/test_download_packages.py

@@ -1,6 +1,7 @@
 import json
 from collections.abc import Iterator
 from contextlib import contextmanager
+from dataclasses import replace
 from pathlib import Path
 from typing import Any
 from unittest.mock import Mock, call, patch
@@ -100,9 +101,7 @@ def test_pypi_download(self) -> None:
 
         # Check the HTTP request with its parameters
         assert m_client.call_count == 1
-        assert m_client.call_args == call(
-            "https://hugovk.github.io/top-pypi-packages/top-pypi-packages.min.json", params={}
-        )
+        assert m_client.call_args == call("https://hugovk.dev/top-pypi-packages/top-pypi-packages.min.json", params={})
 
         # Check the file path
         assert m_open.call_args_list[0] == call(str(Path(DEPENDENCIES_DIR) / "pypi.json"), "w")
@@ -187,6 +186,42 @@ def test_retry_mechanism_with_server_errors(self) -> None:
 
             assert mock_client.call_count == RETRY_ATTEMPTS
 
+    def test_client_error_is_raised(self) -> None:
+        """Test that a non-5xx HTTP error is raised as is, without retrying or parsing the body."""
+        mock_response = Mock()
+        mock_response.is_server_error = False
+        mock_response.status_code = 404
+
+        client_error = httpx.HTTPStatusError("Not Found", request=Mock(), response=mock_response)
+
+        with patch_client_error(client_error) as mock_client:
+            with pytest.raises(httpx.HTTPStatusError):
+                _run("pypi")
+
+            assert mock_client.call_count == 1
+            mock_client.return_value.json.assert_not_called()
+
+    def test_redirect_is_followed(self) -> None:
+        """Test that a source that moved is followed to its new location."""
+        data = {"rows": [{"project": "requests"}, {"project": "numpy"}]}
+
+        def handler(request: httpx.Request) -> httpx.Response:
+            if request.url.host == "old.example":
+                return httpx.Response(301, headers={"Location": "https://new.example/top.json"})
+            return httpx.Response(200, json=data)
+
+        moved = replace(ECOSYSTEMS["pypi"], url="https://old.example/top.json")
+        with (
+            patch.dict(ECOSYSTEMS, {"pypi": moved}),
+            patch("httpx.HTTPTransport.handle_request", side_effect=handler) as m_transport,
+            patch_save_to_file() as m_save,
+            patch_open_file(),
+        ):
+            _run("pypi")
+
+        assert [c.args[0].url.host for c in m_transport.call_args_list] == ["old.example", "new.example"]
+        assert set(m_save.call_args[0][0]["packages"]) == {"requests", "numpy"}
+
     def test_npm_download_with_multiple_pages(self) -> None:
         """Test that the script will iterate through pages if provided."""
         page1_data = [