From 2f3f7545aadbe6eaabc12e76802ae7af62020102 Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Thu, 13 Aug 2026 10:10:15 -0400 Subject: [PATCH 01/35] feat(packagesettings): activate `source` field and add `source_resolver` PBI property Uncomment the `source: SourceResolver | None` field on both `PackageSettings` and `VariantInfo`, add `SourceResolver` to the public API, and wire up a `PackageBuildInfo.source_resolver` property with variant-then-package fallback. This is the foundation for all downstream resolver-config wiring (Phases 2-9 of the new resolver config epic). Closes: #1291 Co-Authored-By: Claude Signed-off-by: Lalatendu Mohanty --- src/fromager/packagesettings/__init__.py | 2 + src/fromager/packagesettings/_models.py | 12 ++- src/fromager/packagesettings/_pbi.py | 14 ++++ tests/test_packagesettings.py | 100 +++++++++++++++++++++++ 4 files changed, 121 insertions(+), 7 deletions(-) diff --git a/src/fromager/packagesettings/__init__.py b/src/fromager/packagesettings/__init__.py index abca2f21d..484ef57fe 100644 --- a/src/fromager/packagesettings/__init__.py +++ b/src/fromager/packagesettings/__init__.py @@ -28,6 +28,7 @@ PyPIGitResolver, PyPIPrebuiltResolver, PyPISDistResolver, + SourceResolver, pep440_tag_matcher, ) from ._settings import Settings, SettingsFile @@ -84,6 +85,7 @@ "SbomSettings", "Settings", "SettingsFile", + "SourceResolver", "Template", "Variant", "VariantChangelog", diff --git a/src/fromager/packagesettings/_models.py b/src/fromager/packagesettings/_models.py index e87230613..5e646d898 100644 --- a/src/fromager/packagesettings/_models.py +++ b/src/fromager/packagesettings/_models.py @@ -16,7 +16,7 @@ from pydantic import AnyUrl, Field, PrivateAttr, StringConstraints from pydantic_core import core_schema -# from ._resolver import SourceResolver +from ._resolver import SourceResolver from ._typedefs import ( MODEL_CONFIG, BuildDirectory, @@ -480,9 +480,8 @@ class VariantInfo(pydantic.BaseModel): pre_built: bool = False """Use pre-built wheel from index server?""" - # TODO - # source: SourceResolver | None - # """Source resolver and downloader""" + source: SourceResolver | None = None + """Source resolver and downloader""" class GitOptions(pydantic.BaseModel): @@ -608,9 +607,8 @@ class PackageSettings(pydantic.BaseModel): project_override: ProjectOverride = Field(default_factory=ProjectOverride) """Patch project settings""" - # TODO - # source: SourceResolver | None - # """Source resolver and downloader""" + source: SourceResolver | None = None + """Source resolver and downloader""" variants: Mapping[Variant, VariantInfo] = Field(default_factory=dict) """Variant configuration""" diff --git a/src/fromager/packagesettings/_pbi.py b/src/fromager/packagesettings/_pbi.py index 2e2620047..ba62f9bf9 100644 --- a/src/fromager/packagesettings/_pbi.py +++ b/src/fromager/packagesettings/_pbi.py @@ -26,6 +26,7 @@ if typing.TYPE_CHECKING: from .. import build_environment + from ._resolver import SourceResolver from ._settings import Settings logger = logging.getLogger(__name__) @@ -176,6 +177,19 @@ def wheel_server_url(self) -> str | None: return str(vi.wheel_server_url) return None + @property + def source_resolver(self) -> SourceResolver | None: + """Source resolver for the package variant. + + Returns the variant-level source resolver if configured, + otherwise falls back to the package-level source resolver, + or ``None`` if neither is set. + """ + vi = self._ps.variants.get(self.variant) + if vi is not None and vi.source is not None: + return vi.source + return self._ps.source + @property def override_module_name(self) -> str: """Override module name from package name""" diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index a5107ad98..3fb06f9ba 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -18,9 +18,12 @@ Package, PackageBuildInfo, PackageSettings, + PyPIPrebuiltResolver, + PyPISDistResolver, ResolverDist, Settings, SettingsFile, + SourceResolver, Variant, substitute_template, ) @@ -90,6 +93,7 @@ "use_pypi_org_metadata": True, "min_release_age": None, }, + "source": None, "variants": { "cpu": { "annotations": { @@ -98,6 +102,7 @@ "env": {"EGG": "spam ${EGG}", "EGG_AGAIN": "$EGG"}, "wheel_server_url": "https://wheel.test/simple", "pre_built": False, + "source": None, }, "rocm": { "annotations": { @@ -106,12 +111,14 @@ "env": {"SPAM": ""}, "wheel_server_url": None, "pre_built": True, + "source": None, }, "cuda": { "annotations": None, "env": {}, "wheel_server_url": None, "pre_built": False, + "source": None, }, }, } @@ -152,6 +159,7 @@ "use_pypi_org_metadata": None, "min_release_age": None, }, + "source": None, "variants": {}, } @@ -193,11 +201,13 @@ "use_pypi_org_metadata": None, "min_release_age": None, }, + "source": None, "variants": { "cpu": { "annotations": None, "env": {}, "pre_built": True, + "source": None, "wheel_server_url": None, }, }, @@ -1152,3 +1162,93 @@ def test_filter_env( ) -> None: ec = ExternalCommands(keep_env=keep, delete_env=delete) assert ec.filter_env(env) == expected + + +# -- source field and source_resolver PBI property ---------------------------- + + +def test_source_resolver_importable() -> None: + """``SourceResolver`` is importable from ``fromager.packagesettings``.""" + args = typing.get_args(SourceResolver) + assert len(args) == 2 + union_members = typing.get_args(args[0]) + assert PyPISDistResolver in union_members + assert PyPIPrebuiltResolver in union_members + + +def test_package_settings_with_source() -> None: + """``PackageSettings`` loads YAML with ``source:`` correctly.""" + yaml_str = "source:\n provider: pypi-sdist\n" + ps = PackageSettings.from_string("test-source-pkg", yaml_str) + assert ps.source is not None + assert isinstance(ps.source, PyPISDistResolver) + assert ps.source.provider == "pypi-sdist" + + +def test_variant_info_with_source() -> None: + """``VariantInfo`` with ``source:`` works.""" + yaml_str = "variants:\n cpu:\n source:\n provider: pypi-prebuilt\n" + ps = PackageSettings.from_string("test-variant-source-pkg", yaml_str) + vi = ps.variants[Variant("cpu")] + assert vi.source is not None + assert isinstance(vi.source, PyPIPrebuiltResolver) + + +def test_package_settings_without_source() -> None: + """``PackageSettings`` without ``source:`` has ``source=None``.""" + ps = PackageSettings.from_default("test-pkg") + assert ps.source is None + + +def test_pbi_source_resolver_none(tmp_path: pathlib.Path) -> None: + """``pbi.source_resolver`` returns ``None`` when no source configured.""" + ps = PackageSettings.from_default("test-pkg") + settings = Settings( + settings=SettingsFile(), + package_settings=[ps], + variant="cpu", + patches_dir=tmp_path, + max_jobs=1, + ) + pbi = settings.package_build_info("test-pkg") + assert pbi.source_resolver is None + + +def test_pbi_source_resolver_package_level(tmp_path: pathlib.Path) -> None: + """``pbi.source_resolver`` returns package-level source.""" + yaml_str = "source:\n provider: pypi-sdist\n" + ps = PackageSettings.from_string("test-pkg", yaml_str) + settings = Settings( + settings=SettingsFile(), + package_settings=[ps], + variant="cpu", + patches_dir=tmp_path, + max_jobs=1, + ) + pbi = settings.package_build_info("test-pkg") + assert pbi.source_resolver is not None + assert isinstance(pbi.source_resolver, PyPISDistResolver) + + +def test_pbi_source_resolver_variant_overrides_package( + tmp_path: pathlib.Path, +) -> None: + """``pbi.source_resolver`` returns variant source over package source.""" + yaml_str = ( + "source:\n" + " provider: pypi-sdist\n" + "variants:\n" + " cpu:\n" + " source:\n" + " provider: pypi-prebuilt\n" + ) + ps = PackageSettings.from_string("test-pkg", yaml_str) + settings = Settings( + settings=SettingsFile(), + package_settings=[ps], + variant="cpu", + patches_dir=tmp_path, + max_jobs=1, + ) + pbi = settings.package_build_info("test-pkg") + assert isinstance(pbi.source_resolver, PyPIPrebuiltResolver) From 8fdedf12cf785a945e80603fb90a9601f04b426b Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Sat, 15 Aug 2026 15:16:36 -0400 Subject: [PATCH 02/35] feat(packagesettings): error when `source` and legacy settings are both set Prevent confusing configs where the new `source` field silently overrides legacy fields. Package-level `source` now conflicts with `download_source` and `resolver_dist`; variant-level `source` conflicts with `wheel_server_url` and `pre_built`. Closes: #1207 Co-Authored-By: Claude Signed-off-by: Lalatendu Mohanty --- src/fromager/packagesettings/_models.py | 30 ++++++++ tests/test_packagesettings.py | 97 +++++++++++++++++++++++++ 2 files changed, 127 insertions(+) diff --git a/src/fromager/packagesettings/_models.py b/src/fromager/packagesettings/_models.py index 5e646d898..783cd8d47 100644 --- a/src/fromager/packagesettings/_models.py +++ b/src/fromager/packagesettings/_models.py @@ -483,6 +483,21 @@ class VariantInfo(pydantic.BaseModel): source: SourceResolver | None = None """Source resolver and downloader""" + @pydantic.model_validator(mode="after") + def validate_source_exclusivity(self) -> typing.Self: + """Validate that ``source`` is mutually exclusive with legacy fields.""" + if self.source is None: + return self + legacy_fields = {"wheel_server_url", "pre_built"} + conflicts = legacy_fields & self.model_fields_set + if conflicts: + raise ValueError( + f"'source' is mutually exclusive with legacy settings: " + f"{sorted(conflicts)}. Use 'source' alone or remove it " + f"and use the legacy settings." + ) + return self + class GitOptions(pydantic.BaseModel): """Git repository cloning options @@ -624,6 +639,21 @@ def before_none_dict( v = {} return v + @pydantic.model_validator(mode="after") + def validate_source_exclusivity(self) -> typing.Self: + """Validate that ``source`` is mutually exclusive with legacy fields.""" + if self.source is None: + return self + legacy_fields = {"download_source", "resolver_dist"} + conflicts = legacy_fields & self.model_fields_set + if conflicts: + raise ValueError( + f"'source' is mutually exclusive with legacy settings: " + f"{sorted(conflicts)}. Use 'source' alone or remove it " + f"and use the legacy settings." + ) + return self + @classmethod def from_mapping( cls, diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index 3fb06f9ba..5ecc745ce 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -1252,3 +1252,100 @@ def test_pbi_source_resolver_variant_overrides_package( ) pbi = settings.package_build_info("test-pkg") assert isinstance(pbi.source_resolver, PyPIPrebuiltResolver) + + +# -- source vs legacy mutual exclusivity validation -------------------------- + + +@pytest.mark.parametrize( + "yaml_str", + [ + pytest.param( + "source:\n provider: pypi-sdist\n" + "download_source:\n url: https://egg.test/pkg-${version}.tar.gz\n", + id="source-with-download-source", + ), + pytest.param( + "source:\n provider: pypi-sdist\n" + "resolver_dist:\n sdist_server_url: https://sdist.test/simple\n", + id="source-with-resolver-dist", + ), + pytest.param( + "source:\n provider: pypi-sdist\n" + "download_source:\n url: https://egg.test/pkg.tar.gz\n" + "resolver_dist:\n sdist_server_url: https://sdist.test/simple\n", + id="source-with-both-legacy-fields", + ), + pytest.param( + "source:\n provider: pypi-sdist\ndownload_source:\n", + id="source-with-empty-download-source", + ), + ], +) +def test_package_source_exclusivity_invalid(yaml_str: str) -> None: + """Setting ``source`` alongside legacy package fields raises.""" + with pytest.raises(RuntimeError, match="mutually exclusive"): + PackageSettings.from_string("conflict-pkg", yaml_str) + + +@pytest.mark.parametrize( + "yaml_str", + [ + pytest.param( + "variants:\n cpu:\n source:\n provider: pypi-prebuilt\n" + " wheel_server_url: https://wheel.test/simple\n", + id="source-with-wheel-server-url", + ), + pytest.param( + "variants:\n cpu:\n source:\n provider: pypi-prebuilt\n" + " pre_built: true\n", + id="source-with-pre-built", + ), + ], +) +def test_variant_source_exclusivity_invalid(yaml_str: str) -> None: + """Setting ``source`` alongside legacy variant fields raises.""" + with pytest.raises(RuntimeError, match="mutually exclusive"): + PackageSettings.from_string("conflict-pkg", yaml_str) + + +def test_legacy_only_config_valid() -> None: + """Legacy-only config loads without error.""" + yaml_str = ( + "download_source:\n url: https://egg.test/pkg-${version}.tar.gz\n" + "resolver_dist:\n sdist_server_url: https://sdist.test/simple\n" + ) + ps = PackageSettings.from_string("legacy-pkg", yaml_str) + assert ps.source is None + assert ps.download_source.url is not None + + +def test_source_only_config_valid() -> None: + """Source-only config loads without error.""" + yaml_str = "source:\n provider: pypi-sdist\n" + ps = PackageSettings.from_string("new-pkg", yaml_str) + assert ps.source is not None + assert ps.download_source.url is None + + +def test_variant_legacy_only_valid() -> None: + """Variant with only legacy fields loads without error.""" + yaml_str = "variants:\n cpu:\n pre_built: true\n wheel_server_url: https://wheel.test/simple\n" + ps = PackageSettings.from_string("legacy-variant-pkg", yaml_str) + vi = ps.variants[Variant("cpu")] + assert vi.source is None + assert vi.pre_built is True + + +def test_source_exclusivity_error_lists_fields() -> None: + """Error message includes the specific conflicting field names.""" + yaml_str = ( + "source:\n provider: pypi-sdist\n" + "download_source:\n url: https://egg.test/a.tar.gz\n" + "resolver_dist:\n sdist_server_url: https://sdist.test\n" + ) + with pytest.raises(RuntimeError) as exc_info: + PackageSettings.from_string("conflict-pkg", yaml_str) + error_str = str(exc_info.value) + assert "download_source" in error_str + assert "resolver_dist" in error_str From e697629586c9c3fb80b82234a9c11eb95f5ce21e Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Thu, 10 Sep 2026 09:48:24 -0400 Subject: [PATCH 03/35] feat(resolver): dispatch configured source resolvers Route packages with source resolver configuration through the configured provider while preserving legacy override behavior. Widen `AbstractResolver.resolver_provider` to accept `RequirementType | None` and remove the `typing.cast` that hid the mismatch. Add coverage for source and bootstrap resolution paths and document the dispatch. Refs: #1292 Co-Authored-By: Claude Signed-off-by: Lalatendu Mohanty --- docs/concepts/resolver-architecture.rst | 12 +- src/fromager/packagesettings/_resolver.py | 26 ++-- src/fromager/sources.py | 9 ++ tests/test_bootstrap_requirement_resolver.py | 23 ++-- tests/test_sources.py | 122 ++++++++++++++++++- 5 files changed, 168 insertions(+), 24 deletions(-) diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index 67fefa0e1..1865151ae 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -90,10 +90,16 @@ CLI commands interact with providers through a common Used when versions are already known, e.g. from a prior resolution or a settings-provided URL template. +.. versionchanged:: 0.95.0 + Packages with ``source:`` configuration now select their provider through + the configured source resolver. + Per-package settings in YAML can select which provider to use and -configure its parameters (index URL, tag pattern, etc.). Override -plugins can replace the provider entirely for a specific package via -the ``get_resolver_provider`` hook. +configure its parameters (index URL, tag pattern, etc.). When a package +has a ``source:`` resolver configured, that resolver creates the provider. +Otherwise, override plugins can replace the provider for a specific package +via the ``get_resolver_provider`` hook, after which the legacy resolver +settings are used. Version Filtering Window ------------------------- diff --git a/src/fromager/packagesettings/_resolver.py b/src/fromager/packagesettings/_resolver.py index 213cf0738..0059b1677 100644 --- a/src/fromager/packagesettings/_resolver.py +++ b/src/fromager/packagesettings/_resolver.py @@ -67,7 +67,7 @@ def resolver_provider( self, ctx: context.WorkContext, req: Requirement, - req_type: requirements_file.RequirementType, + req_type: requirements_file.RequirementType | None, ) -> resolver.BaseProvider: """Return a resolver provider for the given requirement.""" raise NotImplementedError @@ -198,7 +198,7 @@ def resolver_provider( self, ctx: context.WorkContext, req: Requirement, - req_type: requirements_file.RequirementType, + req_type: requirements_file.RequirementType | None, ) -> resolver.PyPIProvider: return resolver.PyPIProvider( include_sdists=True, @@ -249,7 +249,7 @@ def resolver_provider( self, ctx: context.WorkContext, req: Requirement, - req_type: requirements_file.RequirementType, + req_type: requirements_file.RequirementType | None, ) -> resolver.PyPIProvider: return resolver.PyPIProvider( include_sdists=False, @@ -322,7 +322,7 @@ def resolver_provider( self, ctx: context.WorkContext, req: Requirement, - req_type: requirements_file.RequirementType, + req_type: requirements_file.RequirementType | None, ) -> resolver.PyPIProvider: return resolver.PyPIProvider( include_sdists=True, @@ -404,7 +404,7 @@ def resolver_provider( self, ctx: context.WorkContext, req: Requirement, - req_type: requirements_file.RequirementType, + req_type: requirements_file.RequirementType | None, ) -> resolver.PyPIProvider: download_url = f"git+{self.clone_url}@refs/tags/{self.tag}" return resolver.PyPIProvider( @@ -510,7 +510,7 @@ def _github_provider( self, *, ctx: context.WorkContext, - req_type: requirements_file.RequirementType, + req_type: requirements_file.RequirementType | None, override_download_url: str | None = None, ) -> resolver.GitHubTagProvider: if self.project_url.host != "github.com": @@ -534,7 +534,7 @@ def _gitlab_provider( self, *, ctx: context.WorkContext, - req_type: requirements_file.RequirementType, + req_type: requirements_file.RequirementType | None, override_download_url: str | None = None, ) -> resolver.GitLabTagProvider: if not self.project_url.path: @@ -571,7 +571,7 @@ def resolver_provider( self, ctx: context.WorkContext, req: Requirement, - req_type: requirements_file.RequirementType, + req_type: requirements_file.RequirementType | None, ) -> resolver.GitHubTagProvider: return self._github_provider( ctx=ctx, @@ -612,7 +612,7 @@ def resolver_provider( self, ctx: context.WorkContext, req: Requirement, - req_type: requirements_file.RequirementType, + req_type: requirements_file.RequirementType | None, ) -> resolver.GitHubTagProvider: return self._github_provider( ctx=ctx, @@ -651,7 +651,7 @@ def resolver_provider( self, ctx: context.WorkContext, req: Requirement, - req_type: requirements_file.RequirementType, + req_type: requirements_file.RequirementType | None, ) -> resolver.GitLabTagProvider: return self._gitlab_provider( ctx=ctx, @@ -692,7 +692,7 @@ def resolver_provider( self, ctx: context.WorkContext, req: Requirement, - req_type: requirements_file.RequirementType, + req_type: requirements_file.RequirementType | None, ) -> resolver.GitLabTagProvider: return self._gitlab_provider( ctx=ctx, @@ -719,7 +719,7 @@ def resolver_provider( self, ctx: context.WorkContext, req: Requirement, - req_type: requirements_file.RequirementType, + req_type: requirements_file.RequirementType | None, ) -> resolver.BaseProvider: raise ValueError(f"package {req.name} is not available") @@ -743,7 +743,7 @@ def resolver_provider( self, ctx: context.WorkContext, req: Requirement, - req_type: requirements_file.RequirementType, + req_type: requirements_file.RequirementType | None, ) -> resolver.BaseProvider: # TODO raise NotImplementedError("Hook resolver needs a hook") diff --git a/src/fromager/sources.py b/src/fromager/sources.py index 534573d41..f29101a32 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -116,8 +116,17 @@ def get_source_provider( Returns a provider configured according to the package's resolver settings (sdist/wheel inclusion, platform matching, server URL override). + + Packages with a ``source:`` resolver use that resolver's provider + configuration. Packages without one retain the legacy override-hook and + ``resolver_dist`` behavior. """ pbi = ctx.package_build_info(req) + + source_resolver = pbi.source_resolver + if source_resolver is not None: + return source_resolver.resolver_provider(ctx, req, req_type) + override_sdist_server_url = pbi.resolver_sdist_server_url(sdist_server_url) provider = typing.cast( diff --git a/tests/test_bootstrap_requirement_resolver.py b/tests/test_bootstrap_requirement_resolver.py index 7222a6fbc..22374e1e3 100644 --- a/tests/test_bootstrap_requirement_resolver.py +++ b/tests/test_bootstrap_requirement_resolver.py @@ -494,7 +494,7 @@ def test_resolve_auto_routes_to_source( mock_pbi.resolver_min_release_age = None with patch.object(tmp_context, "package_build_info", return_value=mock_pbi): - resolver = BootstrapRequirementResolver(tmp_context) + brr = BootstrapRequirementResolver(tmp_context) # Mock source resolution to return expected result (as list) mock_resolve.return_value = [ @@ -502,15 +502,24 @@ def test_resolve_auto_routes_to_source( ] # Call resolve with pre_built=None (should auto-detect) - results = resolver.resolve( - req=req, - req_type=RequirementType.INSTALL, - parent_req=None, - pre_built=None, - ) + with patch( + "fromager.bootstrap_requirement_resolver.sources.get_source_provider" + ) as get_source_provider: + results = brr.resolve( + req=req, + req_type=RequirementType.INSTALL, + parent_req=None, + pre_built=None, + ) # Verify resolution was called mock_resolve.assert_called_once() + get_source_provider.assert_called_once_with( + ctx=tmp_context, + req=req, + sdist_server_url="https://pypi.org/simple", + req_type=RequirementType.INSTALL, + ) assert len(results) == 1 url, version = results[0] assert url == "https://files.pythonhosted.org/mypackage-2.0.tar.gz" diff --git a/tests/test_sources.py b/tests/test_sources.py index 1ed0904b8..e33b31093 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -2,13 +2,133 @@ import sys import tarfile import zipfile -from unittest.mock import Mock, patch +from unittest.mock import Mock, PropertyMock, patch +import pydantic import pytest from packaging.requirements import Requirement from packaging.version import Version from fromager import context, packagesettings, resolver, sources +from fromager.requirements_file import RequirementType + + +def test_get_source_provider_uses_configured_source_resolver( + tmp_context: context.WorkContext, +) -> None: + """Use the configured source resolver instead of legacy hooks.""" + req = Requirement("test-pkg") + provider = Mock(spec=resolver.BaseProvider) + source_resolver = Mock() + source_resolver.resolver_provider.return_value = provider + + with ( + patch.object( + packagesettings.PackageBuildInfo, + "source_resolver", + new_callable=PropertyMock, + return_value=source_resolver, + ), + patch("fromager.sources.overrides.find_and_invoke") as find_and_invoke, + patch("fromager.sources.resolver.resolve_package_cooldown") as cooldown, + ): + result = sources.get_source_provider( + ctx=tmp_context, + req=req, + sdist_server_url=resolver.PYPI_SERVER_URL, + ) + + assert result is provider + source_resolver.resolver_provider.assert_called_once_with(tmp_context, req, None) + find_and_invoke.assert_not_called() + cooldown.assert_not_called() + + +def test_get_source_provider_uses_pypi_sdist_source_resolver( + tmp_context: context.WorkContext, +) -> None: + """Configure a PyPI sdist provider from the source resolver settings.""" + req = Requirement("test-pkg") + source_resolver = packagesettings.PyPISDistResolver( + provider="pypi-sdist", + index_url=pydantic.HttpUrl("https://pypi.test/simple"), + ) + + with patch.object( + packagesettings.PackageBuildInfo, + "source_resolver", + new_callable=PropertyMock, + return_value=source_resolver, + ): + provider = sources.get_source_provider( + ctx=tmp_context, + req=req, + sdist_server_url=resolver.PYPI_SERVER_URL, + ) + + assert isinstance(provider, resolver.PyPIProvider) + assert provider.include_sdists is True + assert provider.include_wheels is False + assert provider.sdist_server_url == "https://pypi.test/simple" + + +def test_get_source_provider_forwards_req_type_to_source_resolver( + tmp_context: context.WorkContext, +) -> None: + """Forward req_type to the source resolver provider.""" + req = Requirement("test-pkg") + source_resolver = packagesettings.PyPISDistResolver( + provider="pypi-sdist", + index_url=pydantic.HttpUrl("https://pypi.test/simple"), + ) + + with patch.object( + packagesettings.PackageBuildInfo, + "source_resolver", + new_callable=PropertyMock, + return_value=source_resolver, + ): + provider = sources.get_source_provider( + ctx=tmp_context, + req=req, + sdist_server_url=resolver.PYPI_SERVER_URL, + req_type=RequirementType.INSTALL, + ) + + assert isinstance(provider, resolver.PyPIProvider) + assert provider.req_type is RequirementType.INSTALL + + +@patch("fromager.resolver.find_all_matching_from_provider") +def test_resolve_source_uses_configured_source_resolver( + find_all_matching_from_provider: Mock, + tmp_context: context.WorkContext, +) -> None: + """Resolve versions through a configured source resolver provider.""" + req = Requirement("test-pkg>=1.0") + provider = Mock(spec=resolver.BaseProvider) + source_resolver = Mock() + source_resolver.resolver_provider.return_value = provider + find_all_matching_from_provider.return_value = [("url", Version("1.0"))] + + with patch.object( + packagesettings.PackageBuildInfo, + "source_resolver", + new_callable=PropertyMock, + return_value=source_resolver, + ): + url, version = sources.resolve_source( + ctx=tmp_context, + req=req, + sdist_server_url=resolver.PYPI_SERVER_URL, + ) + + assert (url, version) == ("url", Version("1.0")) + find_all_matching_from_provider.assert_called_once_with( + provider, + req, + max_age_cutoff=None, + ) @patch("fromager.resolver.find_all_matching_from_provider") From b3de3e343ac69b687bcbd1eb4db953abffbb4e6b Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Thu, 10 Sep 2026 09:50:05 -0400 Subject: [PATCH 04/35] fix(cooldown): apply global cooldown and pin exemption to source resolvers Source resolvers bypassed `resolve_package_cooldown()`, so the global `--min-release-age` flag was silently ignored and top-level `==` pins did not disable the cooldown. Extract `_effective_cooldown()` from `resolve_package_cooldown()` and call it in the source-resolver dispatch path so both legacy and configured resolvers share the same cooldown merge logic. Refs: #1292 Co-Authored-By: Claude Signed-off-by: Lalatendu Mohanty --- src/fromager/resolver.py | 47 ++++++++++++++++++++---------- src/fromager/sources.py | 10 ++++++- tests/test_sources.py | 63 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 101 insertions(+), 19 deletions(-) diff --git a/src/fromager/resolver.py b/src/fromager/resolver.py index 9f2cfa1b7..d9a9a2c65 100644 --- a/src/fromager/resolver.py +++ b/src/fromager/resolver.py @@ -157,6 +157,36 @@ def _has_equality_pin(req: Requirement) -> bool: return len(specs) == 1 and specs[0].operator == "==" and "*" not in specs[0].version +def _effective_cooldown( + global_cooldown: Cooldown | None, + per_package_days: int | None, +) -> Cooldown | None: + """Merge a per-package cooldown override with the global cooldown. + + Args: + global_cooldown: The global cooldown from ``ctx.cooldown``. + per_package_days: Per-package override in days — ``None`` inherits + the global value, ``0`` disables cooldown, a positive integer + overrides it. + + Returns: + The effective cooldown, or ``None`` if disabled. + """ + if per_package_days is None: + return global_cooldown + if per_package_days == 0: + return None + bootstrap_time = ( + global_cooldown.bootstrap_time + if global_cooldown is not None + else datetime.datetime.now(datetime.UTC) + ) + return Cooldown( + min_age=datetime.timedelta(days=per_package_days), + bootstrap_time=bootstrap_time, + ) + + def resolve_package_cooldown( ctx: context.WorkContext, req: Requirement, @@ -178,22 +208,7 @@ def resolve_package_cooldown( return None per_package_days = ctx.package_build_info(req).resolver_min_release_age - global_cooldown = ctx.cooldown - if per_package_days is None: - return global_cooldown - if per_package_days == 0: - return None - # Per-package positive override: inherit bootstrap_time from global so all - # resolutions in a single run share the same fixed cutoff point. - bootstrap_time = ( - global_cooldown.bootstrap_time - if global_cooldown is not None - else datetime.datetime.now(datetime.UTC) - ) - return Cooldown( - min_age=datetime.timedelta(days=per_package_days), - bootstrap_time=bootstrap_time, - ) + return _effective_cooldown(ctx.cooldown, per_package_days) def _compute_max_age_cutoff( diff --git a/src/fromager/sources.py b/src/fromager/sources.py index f29101a32..6cd8633a9 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -125,7 +125,15 @@ def get_source_provider( source_resolver = pbi.source_resolver if source_resolver is not None: - return source_resolver.resolver_provider(ctx, req, req_type) + provider = source_resolver.resolver_provider(ctx, req, req_type) + if req_type == RequirementType.TOP_LEVEL and resolver._has_equality_pin(req): + provider.cooldown = None + else: + per_package_days = getattr(source_resolver, "min_release_age", None) + provider.cooldown = resolver._effective_cooldown( + ctx.cooldown, per_package_days + ) + return provider override_sdist_server_url = pbi.resolver_sdist_server_url(sdist_server_url) diff --git a/tests/test_sources.py b/tests/test_sources.py index e33b31093..74107abc3 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -1,3 +1,4 @@ +import datetime import pathlib import sys import tarfile @@ -10,6 +11,7 @@ from packaging.version import Version from fromager import context, packagesettings, resolver, sources +from fromager.candidate import Cooldown from fromager.requirements_file import RequirementType @@ -21,6 +23,7 @@ def test_get_source_provider_uses_configured_source_resolver( provider = Mock(spec=resolver.BaseProvider) source_resolver = Mock() source_resolver.resolver_provider.return_value = provider + source_resolver.min_release_age = None with ( patch.object( @@ -30,7 +33,6 @@ def test_get_source_provider_uses_configured_source_resolver( return_value=source_resolver, ), patch("fromager.sources.overrides.find_and_invoke") as find_and_invoke, - patch("fromager.sources.resolver.resolve_package_cooldown") as cooldown, ): result = sources.get_source_provider( ctx=tmp_context, @@ -41,7 +43,6 @@ def test_get_source_provider_uses_configured_source_resolver( assert result is provider source_resolver.resolver_provider.assert_called_once_with(tmp_context, req, None) find_and_invoke.assert_not_called() - cooldown.assert_not_called() def test_get_source_provider_uses_pypi_sdist_source_resolver( @@ -99,6 +100,63 @@ def test_get_source_provider_forwards_req_type_to_source_resolver( assert provider.req_type is RequirementType.INSTALL +def test_get_source_provider_source_resolver_inherits_global_cooldown( + tmp_context: context.WorkContext, +) -> None: + """Source resolver with no per-package cooldown inherits the global one.""" + req = Requirement("test-pkg") + global_cooldown = Cooldown(min_age=datetime.timedelta(days=7)) + tmp_context.cooldown = global_cooldown + + source_resolver = packagesettings.PyPISDistResolver( + provider="pypi-sdist", + index_url=pydantic.HttpUrl("https://pypi.test/simple"), + ) + + with patch.object( + packagesettings.PackageBuildInfo, + "source_resolver", + new_callable=PropertyMock, + return_value=source_resolver, + ): + provider = sources.get_source_provider( + ctx=tmp_context, + req=req, + sdist_server_url=resolver.PYPI_SERVER_URL, + ) + + assert provider.cooldown is global_cooldown + + +def test_get_source_provider_source_resolver_toplevel_pin_bypasses_cooldown( + tmp_context: context.WorkContext, +) -> None: + """Top-level == pin disables cooldown even with a source resolver.""" + req = Requirement("test-pkg==1.0") + tmp_context.cooldown = Cooldown(min_age=datetime.timedelta(days=7)) + + source_resolver = packagesettings.PyPISDistResolver( + provider="pypi-sdist", + index_url=pydantic.HttpUrl("https://pypi.test/simple"), + min_release_age=14, + ) + + with patch.object( + packagesettings.PackageBuildInfo, + "source_resolver", + new_callable=PropertyMock, + return_value=source_resolver, + ): + provider = sources.get_source_provider( + ctx=tmp_context, + req=req, + sdist_server_url=resolver.PYPI_SERVER_URL, + req_type=RequirementType.TOP_LEVEL, + ) + + assert provider.cooldown is None + + @patch("fromager.resolver.find_all_matching_from_provider") def test_resolve_source_uses_configured_source_resolver( find_all_matching_from_provider: Mock, @@ -109,6 +167,7 @@ def test_resolve_source_uses_configured_source_resolver( provider = Mock(spec=resolver.BaseProvider) source_resolver = Mock() source_resolver.resolver_provider.return_value = provider + source_resolver.min_release_age = None find_all_matching_from_provider.return_value = [("url", Version("1.0"))] with patch.object( From 68c7d524927aa6da4bc68567c34e849041691771 Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Thu, 10 Sep 2026 16:19:22 -0400 Subject: [PATCH 05/35] fix(resolver): delegate hook source providers Route hook-backed source resolver profiles through `get_resolver_provider`, forwarding resolver context and server URL while preserving legacy hook signatures. Validate returned providers and cover cooldown and error behavior. Refs: #1324 Co-Authored-By: Codex Signed-off-by: Lalatendu Mohanty --- docs/concepts/resolver-architecture.rst | 8 + docs/reference/hooks.rst | 29 +++- src/fromager/packagesettings/_resolver.py | 108 +++++++++++- src/fromager/sources.py | 7 +- tests/test_packagesettings_resolver.py | 191 +++++++++++++++++++++- tests/test_sources.py | 80 ++++++++- 6 files changed, 406 insertions(+), 17 deletions(-) diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index 1865151ae..8359eb9bf 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -94,9 +94,17 @@ CLI commands interact with providers through a common Packages with ``source:`` configuration now select their provider through the configured source resolver. +.. versionchanged:: 0.96.0 + The ``hook-sdist`` and ``hook-prebuilt`` profiles create providers through + the required ``get_resolver_provider`` override hook. They do not fall back + to PyPI, and hook-backed artifact downloading remains a separate feature. + Per-package settings in YAML can select which provider to use and configure its parameters (index URL, tag pattern, etc.). When a package has a ``source:`` resolver configured, that resolver creates the provider. +The ``hook-sdist`` and ``hook-prebuilt`` profiles delegate provider creation +to the package's required ``get_resolver_provider`` hook and validate that it +returns a :class:`~fromager.resolver.BaseProvider`. Otherwise, override plugins can replace the provider for a specific package via the ``get_resolver_provider`` hook, after which the legacy resolver settings are used. diff --git a/docs/reference/hooks.rst b/docs/reference/hooks.rst index fbfe217ad..5e10f0a8b 100644 --- a/docs/reference/hooks.rst +++ b/docs/reference/hooks.rst @@ -138,10 +138,28 @@ Resolver hooks The arguments are the ``WorkContext``, the ``Requirement`` being evaluated, a boolean indicating whether source distributions should be included, a boolean indicating whether built wheels should be - included, and the URL for the sdist server. + included, and the URL for the sdist server. The hook also receives + ``req_type`` and ``ignore_platform`` when those parameters are supported + by its signature. - The return value must be an instance of a class that implements the - ``resolvelib.providers.AbstractProvider`` API. + The ``hook-sdist`` and ``hook-prebuilt`` source profiles require this hook. + They pass the following keyword arguments: + + * ``ctx`` + * ``req`` + * ``include_sdists`` + * ``include_wheels`` + * ``sdist_server_url`` + * ``req_type`` + * ``ignore_platform`` + + Older hooks may omit newer arguments; unsupported keyword arguments are + filtered for compatibility. A missing hook or a return value that is not a + :class:`~fromager.resolver.BaseProvider` is an error. Hook profiles never + fall back to the default PyPI provider. + + The return value must be an instance of + :class:`~fromager.resolver.BaseProvider`. The expectation is that it acts as an engine for any sort of package resolution whether it is for wheels or sources. The provider can @@ -169,7 +187,10 @@ Resolver hooks return VERSIONS.items() - def get_resolver_provider(ctx, req, include_sdists, include_wheels, sdist_server_url): + def get_resolver_provider( + ctx, req, include_sdists, include_wheels, sdist_server_url, + req_type=None, ignore_platform=False, + ): return resolver.GenericProvider(version_source=_version_source, constraints=ctx.constraints) ``GenericProvider``, ``GitHubTagProvider``, and ``GitLabTagProvider`` take diff --git a/src/fromager/packagesettings/_resolver.py b/src/fromager/packagesettings/_resolver.py index 0059b1677..27b980264 100644 --- a/src/fromager/packagesettings/_resolver.py +++ b/src/fromager/packagesettings/_resolver.py @@ -10,7 +10,7 @@ import pydantic -from .. import downloads, resolver +from .. import downloads, overrides, resolver from ..candidate import Cooldown from ._typedefs import MODEL_CONFIG @@ -68,6 +68,8 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, + *, + sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.BaseProvider: """Return a resolver provider for the given requirement.""" raise NotImplementedError @@ -199,6 +201,8 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, + *, + sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.PyPIProvider: return resolver.PyPIProvider( include_sdists=True, @@ -250,6 +254,8 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, + *, + sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.PyPIProvider: return resolver.PyPIProvider( include_sdists=False, @@ -323,6 +329,8 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, + *, + sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.PyPIProvider: return resolver.PyPIProvider( include_sdists=True, @@ -405,6 +413,8 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, + *, + sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.PyPIProvider: download_url = f"git+{self.clone_url}@refs/tags/{self.tag}" return resolver.PyPIProvider( @@ -572,6 +582,8 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, + *, + sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.GitHubTagProvider: return self._github_provider( ctx=ctx, @@ -613,6 +625,8 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, + *, + sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.GitHubTagProvider: return self._github_provider( ctx=ctx, @@ -652,6 +666,8 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, + *, + sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.GitLabTagProvider: return self._gitlab_provider( ctx=ctx, @@ -693,6 +709,8 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, + *, + sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.GitLabTagProvider: return self._gitlab_provider( ctx=ctx, @@ -720,6 +738,8 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, + *, + sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.BaseProvider: raise ValueError(f"package {req.name} is not available") @@ -739,14 +759,58 @@ class AbstractHookResolver(AbstractResolver, CooldownMixin): supports_override_hooks: typing.ClassVar[bool] = True """Hook resolvers support override hooks.""" + def _resolver_provider_from_hook( + self, + *, + ctx: context.WorkContext, + req: Requirement, + req_type: requirements_file.RequirementType | None, + sdist_server_url: str, + include_sdists: bool, + include_wheels: bool, + ignore_platform: bool, + ) -> resolver.BaseProvider: + """Invoke the required package resolver hook and validate its result.""" + hook = overrides.find_override_method(req.name, "get_resolver_provider") + if hook is None: + raise ValueError( + f"{req.name}: source resolver {self.provider!r} requires a " + "get_resolver_provider override hook" + ) + + try: + provider = overrides.invoke( + hook, + ctx=ctx, + req=req, + include_sdists=include_sdists, + include_wheels=include_wheels, + sdist_server_url=sdist_server_url, + req_type=req_type, + ignore_platform=ignore_platform, + ) + except Exception as err: + raise RuntimeError( + f"{req.name}: {self.provider!r} get_resolver_provider hook failed" + ) from err + + if not isinstance(provider, resolver.BaseProvider): + raise TypeError( + f"{req.name}: {self.provider!r} get_resolver_provider hook " + f"returned {type(provider).__name__}, expected " + "fromager.resolver.BaseProvider" + ) + return provider + def resolver_provider( self, ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, + *, + sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.BaseProvider: - # TODO - raise NotImplementedError("Hook resolver needs a hook") + raise NotImplementedError def download( self, @@ -776,6 +840,25 @@ class HookSDistResolver(AbstractHookResolver): {DownloadKind.sdist, DownloadKind.tarball, DownloadKind.git_checkout} ) + def resolver_provider( + self, + ctx: context.WorkContext, + req: Requirement, + req_type: requirements_file.RequirementType | None, + *, + sdist_server_url: str = resolver.PYPI_SERVER_URL, + ) -> resolver.BaseProvider: + """Return a source provider from the package resolver hook.""" + return self._resolver_provider_from_hook( + ctx=ctx, + req=req, + req_type=req_type, + sdist_server_url=sdist_server_url, + include_sdists=True, + include_wheels=False, + ignore_platform=False, + ) + class HookPrebuiltResolver(AbstractHookResolver): """Call resolver_provider and download_source hook, use pre-built wheel @@ -795,6 +878,25 @@ class HookPrebuiltResolver(AbstractHookResolver): ) resolves_prebuilt_wheel: typing.ClassVar[bool] = True + def resolver_provider( + self, + ctx: context.WorkContext, + req: Requirement, + req_type: requirements_file.RequirementType | None, + *, + sdist_server_url: str = resolver.PYPI_SERVER_URL, + ) -> resolver.BaseProvider: + """Return a pre-built wheel provider from the package resolver hook.""" + return self._resolver_provider_from_hook( + ctx=ctx, + req=req, + req_type=req_type, + sdist_server_url=sdist_server_url, + include_sdists=False, + include_wheels=True, + ignore_platform=False, + ) + SourceResolver = typing.Annotated[ PyPISDistResolver diff --git a/src/fromager/sources.py b/src/fromager/sources.py index 6cd8633a9..742741fd8 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -125,7 +125,12 @@ def get_source_provider( source_resolver = pbi.source_resolver if source_resolver is not None: - provider = source_resolver.resolver_provider(ctx, req, req_type) + provider = source_resolver.resolver_provider( + ctx, + req, + req_type, + sdist_server_url=sdist_server_url, + ) if req_type == RequirementType.TOP_LEVEL and resolver._has_equality_pin(req): provider.cooldown = None else: diff --git a/tests/test_packagesettings_resolver.py b/tests/test_packagesettings_resolver.py index 8132b4a49..2dffa5705 100644 --- a/tests/test_packagesettings_resolver.py +++ b/tests/test_packagesettings_resolver.py @@ -679,10 +679,55 @@ def test_parse(self) -> None: {DownloadKind.sdist, DownloadKind.tarball, DownloadKind.git_checkout} ) - def test_resolver_provider(self, tmp_context: WorkContext) -> None: + @mock.patch("fromager.packagesettings._resolver.overrides.find_override_method") + def test_resolver_provider( + self, + find_override_method: mock.Mock, + tmp_context: WorkContext, + ) -> None: r = _parse(self.YAML) - with pytest.raises(NotImplementedError): - r.resolver_provider(tmp_context, _REQ, _REQ_TYPE) + provider = resolver.GenericProvider(version_source=lambda identifier: []) + captured: dict[str, object] = {} + + def hook( + ctx: WorkContext, + req: Requirement, + include_sdists: bool, + include_wheels: bool, + sdist_server_url: str, + req_type: RequirementType | None = None, + ignore_platform: bool = False, + ) -> resolver.BaseProvider: + captured.update( + ctx=ctx, + req=req, + include_sdists=include_sdists, + include_wheels=include_wheels, + sdist_server_url=sdist_server_url, + req_type=req_type, + ignore_platform=ignore_platform, + ) + return provider + + find_override_method.return_value = hook + + result = r.resolver_provider( + tmp_context, + _REQ, + _REQ_TYPE, + sdist_server_url="https://index.test/simple", + ) + + assert result is provider + assert captured == { + "ctx": tmp_context, + "req": _REQ, + "include_sdists": True, + "include_wheels": False, + "sdist_server_url": "https://index.test/simple", + "req_type": _REQ_TYPE, + "ignore_platform": False, + } def test_download(self, tmp_context: WorkContext) -> None: r = _parse(self.YAML) @@ -704,10 +749,40 @@ def test_parse(self) -> None: assert r.resolves_prebuilt_wheel is True assert r.download_kinds == frozenset({DownloadKind.prebuilt_wheel}) - def test_resolver_provider(self, tmp_context: WorkContext) -> None: + @mock.patch("fromager.packagesettings._resolver.overrides.find_override_method") + def test_resolver_provider( + self, + find_override_method: mock.Mock, + tmp_context: WorkContext, + ) -> None: r = _parse(self.YAML) - with pytest.raises(NotImplementedError): - r.resolver_provider(tmp_context, _REQ, _REQ_TYPE) + provider = resolver.GenericProvider(version_source=lambda identifier: []) + + def hook( + ctx: WorkContext, + req: Requirement, + include_sdists: bool, + include_wheels: bool, + sdist_server_url: str, + req_type: RequirementType | None = None, + ignore_platform: bool = False, + ) -> resolver.BaseProvider: + return provider + + find_override_method.return_value = hook + + result = r.resolver_provider( + tmp_context, + _REQ, + _REQ_TYPE, + sdist_server_url="https://index.test/simple", + ) + + assert result is provider + find_override_method.assert_called_once_with( + _REQ.name, + "get_resolver_provider", + ) def test_download(self, tmp_context: WorkContext) -> None: r = _parse(self.YAML) @@ -715,6 +790,110 @@ def test_download(self, tmp_context: WorkContext) -> None: r.download(tmp_context, _REQ, _CANDIDATE_WHEEL) +@mock.patch("fromager.packagesettings._resolver.overrides.find_override_method") +def test_hook_resolver_supports_legacy_hook_signature( + find_override_method: mock.Mock, + tmp_context: WorkContext, +) -> None: + """Legacy five-argument resolver hooks remain compatible.""" + provider = resolver.GenericProvider(version_source=lambda identifier: []) + + def legacy_hook( + ctx: WorkContext, + req: Requirement, + include_sdists: bool, + include_wheels: bool, + sdist_server_url: str, + ) -> resolver.BaseProvider: + return provider + + find_override_method.return_value = legacy_hook + resolver_model = HookSDistResolver(provider="hook-sdist") + + result = resolver_model.resolver_provider(tmp_context, _REQ, _REQ_TYPE) + + assert result is provider + + +@mock.patch("fromager.packagesettings._resolver.resolver.default_resolver_provider") +@mock.patch( + "fromager.packagesettings._resolver.overrides.find_override_method", + return_value=None, +) +def test_hook_resolver_does_not_fall_back_when_hook_is_missing( + find_override_method: mock.Mock, + default_resolver_provider: mock.Mock, + tmp_context: WorkContext, +) -> None: + """A missing hook fails instead of selecting the default PyPI provider.""" + resolver_model = HookSDistResolver(provider="hook-sdist") + + with pytest.raises(ValueError, match="requires a get_resolver_provider"): + resolver_model.resolver_provider(tmp_context, _REQ, _REQ_TYPE) + + find_override_method.assert_called_once_with( + _REQ.name, + "get_resolver_provider", + ) + default_resolver_provider.assert_not_called() + + +@mock.patch("fromager.packagesettings._resolver.overrides.find_override_method") +def test_hook_resolver_rejects_invalid_provider( + find_override_method: mock.Mock, + tmp_context: WorkContext, +) -> None: + """Reject values that are not Fromager resolver providers.""" + + def hook( + ctx: WorkContext, + req: Requirement, + include_sdists: bool, + include_wheels: bool, + sdist_server_url: str, + req_type: RequirementType | None = None, + ignore_platform: bool = False, + ) -> object: + return object() + + find_override_method.return_value = hook + resolver_model = HookPrebuiltResolver(provider="hook-prebuilt") + + with pytest.raises( + TypeError, + match=r"expected fromager\.resolver\.BaseProvider", + ): + resolver_model.resolver_provider(tmp_context, _REQ, _REQ_TYPE) + + +@mock.patch("fromager.packagesettings._resolver.overrides.find_override_method") +def test_hook_resolver_chains_hook_exception( + find_override_method: mock.Mock, + tmp_context: WorkContext, +) -> None: + """Add resolver context while preserving the hook exception as the cause.""" + + def hook( + ctx: WorkContext, + req: Requirement, + include_sdists: bool, + include_wheels: bool, + sdist_server_url: str, + req_type: RequirementType | None = None, + ignore_platform: bool = False, + ) -> resolver.BaseProvider: + raise ValueError("hook failure") + + find_override_method.return_value = hook + resolver_model = HookSDistResolver(provider="hook-sdist") + + with pytest.raises(RuntimeError, match="hook-sdist") as exc_info: + resolver_model.resolver_provider(tmp_context, _REQ, _REQ_TYPE) + + assert isinstance(exc_info.value.__cause__, ValueError) + assert str(exc_info.value.__cause__) == "hook failure" + + # -- Discriminated union validation ------------------------------------------- diff --git a/tests/test_sources.py b/tests/test_sources.py index 74107abc3..e69135738 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -37,11 +37,16 @@ def test_get_source_provider_uses_configured_source_resolver( result = sources.get_source_provider( ctx=tmp_context, req=req, - sdist_server_url=resolver.PYPI_SERVER_URL, + sdist_server_url="https://caller.test/simple", ) assert result is provider - source_resolver.resolver_provider.assert_called_once_with(tmp_context, req, None) + source_resolver.resolver_provider.assert_called_once_with( + tmp_context, + req, + None, + sdist_server_url="https://caller.test/simple", + ) find_and_invoke.assert_not_called() @@ -64,7 +69,7 @@ def test_get_source_provider_uses_pypi_sdist_source_resolver( provider = sources.get_source_provider( ctx=tmp_context, req=req, - sdist_server_url=resolver.PYPI_SERVER_URL, + sdist_server_url="https://caller.test/simple", ) assert isinstance(provider, resolver.PyPIProvider) @@ -73,6 +78,75 @@ def test_get_source_provider_uses_pypi_sdist_source_resolver( assert provider.sdist_server_url == "https://pypi.test/simple" +@patch("fromager.packagesettings._resolver.overrides.find_override_method") +def test_get_source_provider_hook_forwards_url_and_applies_cooldown( + find_override_method: Mock, + tmp_context: context.WorkContext, +) -> None: + """Hook providers receive the runtime URL and centralized cooldown.""" + req = Requirement("test-pkg") + initial_cooldown = Cooldown(min_age=datetime.timedelta(days=1)) + provider = resolver.GenericProvider( + version_source=lambda identifier: [], + cooldown=initial_cooldown, + ) + captured: dict[str, object] = {} + + def hook( + ctx: context.WorkContext, + req: Requirement, + include_sdists: bool, + include_wheels: bool, + sdist_server_url: str, + req_type: RequirementType | None = None, + ignore_platform: bool = False, + ) -> resolver.BaseProvider: + captured.update( + ctx=ctx, + req=req, + include_sdists=include_sdists, + include_wheels=include_wheels, + sdist_server_url=sdist_server_url, + req_type=req_type, + ignore_platform=ignore_platform, + ) + return provider + + find_override_method.return_value = hook + tmp_context.cooldown = Cooldown(min_age=datetime.timedelta(days=7)) + source_resolver = packagesettings.HookSDistResolver( + provider="hook-sdist", + min_release_age=14, + ) + + with patch.object( + packagesettings.PackageBuildInfo, + "source_resolver", + new_callable=PropertyMock, + return_value=source_resolver, + ): + result = sources.get_source_provider( + ctx=tmp_context, + req=req, + sdist_server_url="https://caller.test/simple", + req_type=RequirementType.INSTALL, + ) + + assert result is provider + assert captured == { + "ctx": tmp_context, + "req": req, + "include_sdists": True, + "include_wheels": False, + "sdist_server_url": "https://caller.test/simple", + "req_type": RequirementType.INSTALL, + "ignore_platform": False, + } + assert result.cooldown is not initial_cooldown + assert result.cooldown is not None + assert result.cooldown.min_age == datetime.timedelta(days=14) + + def test_get_source_provider_forwards_req_type_to_source_resolver( tmp_context: context.WorkContext, ) -> None: From 8b946783a367d73200cc02534a9d7f4e0832425d Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Mon, 21 Sep 2026 16:40:57 -0400 Subject: [PATCH 06/35] feat(sources): dispatch configured source downloads New `source:` profiles can download more than sdists. Dispatch configured downloads through their resolver and return `(Path, DownloadKind)` so later phases can handle each artifact. Keep the legacy hook path for packages without `source:`. Closes: #1293 Co-Authored-By: Claude Signed-off-by: Lalatendu Mohanty --- src/fromager/bootstrapper/_bootstrapper.py | 5 +- src/fromager/bootstrapper/_prepare_source.py | 2 +- src/fromager/commands/build.py | 2 +- src/fromager/commands/step.py | 2 +- src/fromager/sources.py | 16 +++- tests/test_bootstrapper.py | 40 +++++++- tests/test_commands.py | 99 +++++++++++++++++++- tests/test_sources.py | 67 ++++++++++++- 8 files changed, 221 insertions(+), 12 deletions(-) diff --git a/src/fromager/bootstrapper/_bootstrapper.py b/src/fromager/bootstrapper/_bootstrapper.py index a12e67532..db599e9aa 100644 --- a/src/fromager/bootstrapper/_bootstrapper.py +++ b/src/fromager/bootstrapper/_bootstrapper.py @@ -465,13 +465,14 @@ def _download_source( source_url: str, ) -> pathlib.Path: """Download source for a package.""" - result: pathlib.Path = sources.download_source( + download_result = sources.download_source( ctx=self.ctx, req=req, version=resolved_version, download_url=source_url, ) - return result + source_path: pathlib.Path = download_result[0] + return source_path def _prepare_source( self, diff --git a/src/fromager/bootstrapper/_prepare_source.py b/src/fromager/bootstrapper/_prepare_source.py index a0ba1da73..6076fdd2b 100644 --- a/src/fromager/bootstrapper/_prepare_source.py +++ b/src/fromager/bootstrapper/_prepare_source.py @@ -43,7 +43,7 @@ def _bg_prepare_source( sdist_root_dir=unpacked / unpacked.stem, cached_wheel_filename=cached_wheel, ) - source_filename = sources.download_source( + source_filename, _ = sources.download_source( ctx=ctx, req=req, version=resolved_version, download_url=source_url ) sdist_root_dir = sources.prepare_source( diff --git a/src/fromager/commands/build.py b/src/fromager/commands/build.py index 2e33f8f0e..831741bfc 100644 --- a/src/fromager/commands/build.py +++ b/src/fromager/commands/build.py @@ -388,7 +388,7 @@ def _build( # If we get here and still don't have a wheel filename, then we need to # build the wheel. if not wheel_filename: - source_filename = sources.download_source( + source_filename, _ = sources.download_source( ctx=wkctx, req=req, version=resolved_version, diff --git a/src/fromager/commands/step.py b/src/fromager/commands/step.py index 5df91ff63..197d06a1f 100644 --- a/src/fromager/commands/step.py +++ b/src/fromager/commands/step.py @@ -50,7 +50,7 @@ def download_source_archive( source_url, version = sources.resolve_source( ctx=wkctx, req=req, sdist_server_url=sdist_server_url ) - filename = sources.download_source( + filename, _ = sources.download_source( ctx=wkctx, req=req, version=version, diff --git a/src/fromager/sources.py b/src/fromager/sources.py index 742741fd8..fa915435e 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -31,6 +31,7 @@ tarballs, vendor_rust, ) +from .candidate import Candidate from .requirements_file import RequirementType, SourceType if typing.TYPE_CHECKING: @@ -84,9 +85,20 @@ def download_source( req: Requirement, version: Version, download_url: str, -) -> pathlib.Path: +) -> tuple[pathlib.Path, packagesettings.DownloadKind]: + """Download a resolved source and return its path and artifact kind. + + Configured source resolvers receive a minimal candidate. Legacy downloads + are adapted to the sdist kind for compatibility. + """ logger.info(f"downloading source for {req}") + pbi = ctx.package_build_info(req) + source_resolver = pbi.source_resolver + if source_resolver is not None: + candidate = Candidate(name=req.name, version=version, url=download_url) + return source_resolver.download(ctx, req, candidate) + source_path = overrides.find_and_invoke( req.name, "download_source", @@ -102,7 +114,7 @@ def download_source( raise ValueError( f"expected a Path back to downloaded source. got {source_path}" ) - return source_path + return source_path, packagesettings.DownloadKind.sdist def get_source_provider( diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index 1f3b25162..03cf7be6b 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -35,6 +35,7 @@ ) from fromager.bootstrapper._work_item import WorkItem from fromager.context import WorkContext +from fromager.packagesettings import DownloadKind from fromager.requirements_file import RequirementType, SourceType @@ -964,6 +965,7 @@ def test_bg_prepare_source_log_prefix_includes_version( logging.setLogRecordFactory(log.FromagerLogRecord) req = Requirement("mypkg==1.2.3") version = Version("1.2.3") + source_filename = pathlib.Path("mypkg-1.2.3.tar.gz") messages: list[str] = [] try: @@ -975,12 +977,15 @@ def test_bg_prepare_source_log_prefix_includes_version( ), patch( "fromager.sources.download_source", - return_value=pathlib.Path("mypkg-1.2.3.tar.gz"), + return_value=( + source_filename, + DownloadKind.sdist, + ), ), patch( "fromager.sources.prepare_source", return_value=pathlib.Path(tmp_context.work_dir / "mypkg-1.2.3"), - ), + ) as prepare_source, log.req_ctxvar_context(req, version), ): _bg_prepare_source( @@ -997,6 +1002,12 @@ def test_bg_prepare_source_log_prefix_includes_version( for r in caplog.records if r.name.startswith("fromager.bootstrapper") ] + prepare_source.assert_called_once_with( + ctx=tmp_context, + req=req, + source_filename=source_filename, + version=version, + ) finally: logging.setLogRecordFactory(old_factory) @@ -1006,6 +1017,31 @@ def test_bg_prepare_source_log_prefix_includes_version( ) +def test_bootstrapper_download_source_returns_downloaded_path( + tmp_context: WorkContext, +) -> None: + """Return only the path from the resolver download result.""" + bt = bootstrapper.Bootstrapper(tmp_context) + req = Requirement("mypkg==1.2.3") + version = Version("1.2.3") + source_url = "https://pkg.test/mypkg-1.2.3.tar.gz" + source_filename = pathlib.Path("mypkg-1.2.3.tar.gz") + + with patch( + "fromager.bootstrapper._bootstrapper.sources.download_source", + return_value=(source_filename, DownloadKind.sdist), + ) as download_source: + result = bt._download_source(req, version, source_url) + + assert result == source_filename + download_source.assert_called_once_with( + ctx=tmp_context, + req=req, + version=version, + download_url=source_url, + ) + + def test_bg_prepare_prebuilt_log_prefix_includes_version( tmp_context: WorkContext, caplog: pytest.LogCaptureFixture, diff --git a/tests/test_commands.py b/tests/test_commands.py index 5678d3d96..1f3ba7cd7 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,8 +1,15 @@ +import pathlib import typing +from unittest.mock import Mock, patch import click +from click.testing import CliRunner +from packaging.requirements import Requirement +from packaging.version import Version -from fromager.commands import bootstrap, build +from fromager import packagesettings +from fromager.commands import bootstrap, build, step +from fromager.context import WorkContext def get_option_names(cmd: click.Command) -> typing.Iterable[str]: @@ -20,3 +27,93 @@ def test_bootstrap_parallel_options() -> None: expected.discard("test_mode") assert set(get_option_names(bootstrap.bootstrap_parallel)) == expected + + +def test_download_source_archive_prints_downloaded_path( + cli_runner: CliRunner, + tmp_context: WorkContext, + tmp_path: pathlib.Path, +) -> None: + """Print only the path when the download helper returns a tuple.""" + expected = tmp_path / "pkg-1.0.tar.gz" + + with ( + patch.object( + step.sources, + "resolve_source", + return_value=("https://pkg.test/pkg-1.0.tar.gz", Version("1.0")), + ), + patch.object( + step.sources, + "download_source", + return_value=(expected, packagesettings.DownloadKind.sdist), + ), + ): + result = cli_runner.invoke( + step.step, + [ + "download-source-archive", + "pkg", + "1.0", + "https://pkg.test/simple", + ], + obj=tmp_context, + ) + + assert result.exit_code == 0 + assert result.output == f"{expected}\n" + + +def test_build_passes_downloaded_path_to_prepare_source( + tmp_context: WorkContext, + tmp_path: pathlib.Path, +) -> None: + """Pass only the downloaded path into source preparation.""" + req = Requirement("pkg==1.0") + version = Version("1.0") + source_url = "https://pkg.test/pkg-1.0.tar.gz" + source_filename = tmp_path / "pkg-1.0.tar.gz" + source_root = tmp_path / "pkg-1.0" + sdist_filename = tmp_path / "rebuilt" / "pkg-1.0.tar.gz" + wheel_filename = tmp_path / "pkg-1.0-py3-none-any.whl" + build_env = Mock() + + with ( + patch.object(build.wheels, "get_wheel_server_urls", return_value=[]), + patch.object( + build.sources, + "download_source", + return_value=(source_filename, packagesettings.DownloadKind.sdist), + ), + patch.object( + build.sources, + "prepare_source", + return_value=source_root, + ) as prepare_source, + patch.object( + build.build_environment, + "prepare_build_environment", + return_value=build_env, + ), + patch.object(build.sources, "build_sdist", return_value=sdist_filename), + patch.object(build.wheels, "build_wheel", return_value=wheel_filename), + patch.object(build.hooks, "run_post_build_hooks"), + patch.object(tmp_context, "clean_build_dirs"), + patch.object(build.server, "update_wheel_mirror"), + ): + result = build._build( + wkctx=tmp_context, + resolved_version=version, + req=req, + source_download_url=source_url, + force=True, + cache_wheel_server_url=None, + ) + + prepare_source.assert_called_once_with( + ctx=tmp_context, + req=req, + source_filename=source_filename, + version=version, + ) + assert result.wheel_filename == tmp_context.wheels_downloads / wheel_filename.name diff --git a/tests/test_sources.py b/tests/test_sources.py index e69135738..b29b2603a 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -11,7 +11,7 @@ from packaging.version import Version from fromager import context, packagesettings, resolver, sources -from fromager.candidate import Cooldown +from fromager.candidate import Candidate, Cooldown from fromager.requirements_file import RequirementType @@ -861,10 +861,73 @@ def test_download_source_regular_package( download_url="https://pkg.test/pkg-1.0.tar.gz", ) - assert result == expected + assert result == (expected, packagesettings.DownloadKind.sdist) mock_invoke.assert_called_once() +def test_download_source_uses_configured_source_resolver( + tmp_context: context.WorkContext, + tmp_path: pathlib.Path, +) -> None: + """Dispatch configured source downloads with a minimal candidate.""" + expected = tmp_path / "pkg-1.0.tar.gz" + source_resolver = packagesettings.PyPISDistResolver(provider="pypi-sdist") + req = Requirement("pkg==1.0") + version = Version("1.0") + download_url = "https://pkg.test/pkg-1.0.tar.gz" + with ( + patch.object( + packagesettings.PyPISDistResolver, + "_download", + return_value=(expected, packagesettings.DownloadKind.sdist), + ) as mock_download, + patch.object( + packagesettings.PackageBuildInfo, + "source_resolver", + new_callable=PropertyMock, + return_value=source_resolver, + ), + patch("fromager.sources.overrides.find_and_invoke") as mock_invoke, + ): + result = sources.download_source( + ctx=tmp_context, req=req, version=version, download_url=download_url + ) + + expected_candidate = Candidate(name=req.name, version=version, url=download_url) + assert result == (expected, packagesettings.DownloadKind.sdist) + mock_download.assert_called_once_with( + tmp_context, + req, + expected_candidate, + packagesettings.DownloadKind.sdist, + ) + mock_invoke.assert_not_called() + + +def test_download_source_returns_configured_result_without_validation( + tmp_context: context.WorkContext, +) -> None: + """Leave configured resolver result validation to the follow-up phase.""" + configured_result = "resolver-owned-result" + source_resolver = Mock() + source_resolver.download.return_value = configured_result + + with patch.object( + packagesettings.PackageBuildInfo, + "source_resolver", + new_callable=PropertyMock, + return_value=source_resolver, + ): + result = sources.download_source( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + version=Version("1.0"), + download_url="https://pkg.test/pkg-1.0.tar.gz", + ) + + assert result == configured_result + + @patch("fromager.overrides.find_and_invoke", return_value="not-a-path") def test_download_source_invalid_return_raises( mock_invoke: Mock, From 5c9ed7109b1b107227e0da36fbeb4f9bad72ae83 Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Tue, 22 Sep 2026 14:16:21 -0400 Subject: [PATCH 07/35] feat(sources): replace download tuple with DownloadedSource dataclass Replace `tuple[Path, DownloadKind]` with a frozen `DownloadedSource` dataclass so the download contract is stable before Phase 4 consumes the artifact kind to select prepare/build strategies. Implement `HookSDistResolver.download()` and `HookPrebuiltResolver.download()` through the `download_source` override hook. For legacy hooks returning a bare `Path`, `HookSDistResolver` treats `.tar.gz` and `.zip` files as `sdist`, while `HookPrebuiltResolver` treats `.whl` files as `prebuilt_wheel`. Bare directories and other path forms require `DownloadedSource` with an explicit supported kind. Reject `any_source`, `not_available`, and kinds incompatible with the selected profile. Closes: #1331 Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Lalatendu Mohanty --- docs/concepts/hooks-and-overrides.rst | 6 + docs/concepts/resolver-architecture.rst | 11 +- docs/reference/config-reference.rst | 10 + docs/reference/hooks.rst | 70 +++- src/fromager/bootstrapper/_bootstrapper.py | 6 +- src/fromager/bootstrapper/_prepare_source.py | 4 +- src/fromager/commands/build.py | 4 +- src/fromager/commands/step.py | 4 +- src/fromager/packagesettings/__init__.py | 2 + src/fromager/packagesettings/_resolver.py | 132 ++++++- src/fromager/sources.py | 6 +- tests/test_bootstrapper.py | 10 +- tests/test_commands.py | 8 +- tests/test_packagesettings_resolver.py | 358 +++++++++++++++++-- tests/test_sources.py | 65 +++- 15 files changed, 618 insertions(+), 78 deletions(-) diff --git a/docs/concepts/hooks-and-overrides.rst b/docs/concepts/hooks-and-overrides.rst index 3fcac8369..9d5274995 100644 --- a/docs/concepts/hooks-and-overrides.rst +++ b/docs/concepts/hooks-and-overrides.rst @@ -56,6 +56,12 @@ a Python module. See :doc:`/reference/hooks` for the complete list of hooks and their arguments. +.. versionchanged:: 0.96.0 + + Configured ``hook-sdist`` and ``hook-prebuilt`` profiles support typed + download results with explicit artifact kinds. See :ref:`download_source_hook` + for the contract, legacy path defaults, and a migration example. + Global Hooks ------------ diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index 4db347fb1..06899f799 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -97,7 +97,8 @@ CLI commands interact with providers through a common .. versionchanged:: 0.96.0 The ``hook-sdist`` and ``hook-prebuilt`` profiles create providers through the required ``get_resolver_provider`` override hook. They do not fall back - to PyPI, and hook-backed artifact downloading remains a separate feature. + to PyPI. Their ``download()`` methods invoke the package's ``download_source`` + hook when present and return a validated ``DownloadedSource``. Per-package settings in YAML can select which provider to use and configure its parameters (index URL, tag pattern, etc.). When a package @@ -109,6 +110,14 @@ Otherwise, override plugins can replace the provider for a specific package via the ``get_resolver_provider`` hook, after which the legacy resolver settings are used. +Hook downloads accept explicit artifact kinds and adapt legacy paths using +profile-specific defaults. If no download hook exists, the profile selects the +standard sdist or wheel downloader and its destination directory. Hook errors +propagate with context instead of triggering a fallback. The typed result makes +the artifact kind available for source preparation; consuming it in preparation +and integrating prebuilt detection into the bootstrapper are separate steps. +See :ref:`download_source_hook` for the return contract and compatibility rules. + Version Filtering Window ------------------------- diff --git a/docs/reference/config-reference.rst b/docs/reference/config-reference.rst index f9c1df4d0..4b0432f1e 100644 --- a/docs/reference/config-reference.rst +++ b/docs/reference/config-reference.rst @@ -77,6 +77,9 @@ Source Resolver .. autoattribute:: pep517 .. autoattribute:: tarball +.. autoclass:: DownloadedSource + :members: path, kind + .. autoclass:: DownloadKind .. autoattribute:: sdist @@ -86,6 +89,13 @@ Source Resolver .. autoattribute:: any_source .. autoattribute:: not_available +.. versionchanged:: 0.96.0 + + Configured hook downloads require a concrete result kind supported by the + selected profile. ``any_source`` and ``not_available`` are not accepted as + result kinds. See :ref:`download_source_hook` for legacy path defaults and + the typed hook contract. + Global Settings --------------- diff --git a/docs/reference/hooks.rst b/docs/reference/hooks.rst index 5e10f0a8b..b8e90ef3b 100644 --- a/docs/reference/hooks.rst +++ b/docs/reference/hooks.rst @@ -219,6 +219,8 @@ Source hooks were removed. Define a :ref:`resolver_provider ` hook to resolve sources. +.. _download_source_hook: + .. autofromagerhook:: default_download_source The ``download_source()`` function is responsible for downloading the @@ -228,7 +230,73 @@ Source hooks evaluated, version of the package being downloaded, the download URL, and the output directory in which the source should be downloaded. - The return value should be a ``pathlib.Path`` file path to the downloaded source. + For packages without ``source:`` configuration, return a ``pathlib.Path`` + to the downloaded source. This legacy contract is unchanged. + + .. versionchanged:: 0.96.0 + + The ``hook-sdist`` and ``hook-prebuilt`` source profiles accept + :class:`~fromager.packagesettings.DownloadedSource` with a local + ``pathlib.Path`` and an explicit + :class:`~fromager.packagesettings.DownloadKind`: + + * ``hook-sdist`` accepts ``sdist``, ``tarball``, or ``git_checkout``. + * ``hook-prebuilt`` accepts only ``prebuilt_wheel``. + + A bare ``Path`` remains supported for legacy hooks: ``hook-sdist`` + treats ``.tar.gz`` and ``.zip`` paths as ``sdist``; ``hook-prebuilt`` + treats ``.whl`` paths as ``prebuilt_wheel``. Bare directories and other + path forms are rejected. The sdist default preserves legacy archive + semantics; it does not infer the artifact kind from archive contents. + Hooks needing tarball or git-checkout semantics must return a typed + result with the explicit kind. ``any_source`` and ``not_available`` + are not valid concrete result kinds. + + Invalid return types, malformed typed fields, and kinds incompatible + with the profile raise errors identifying the package and hook. + Hook exceptions are chained to a ``RuntimeError`` with that context; + neither failures nor invalid results trigger a fallback download. + + If the download hook is absent, ``hook-sdist`` downloads the candidate + URL as an sdist into ``ctx.sdists_downloads``. ``hook-prebuilt`` downloads + it as a wheel into ``ctx.wheels_prebuilt``. These defaults use the + standard archive and wheel validation. Custom tarball URLs and git + downloads require a download hook. The ``get_resolver_provider`` hook + remains required for both profiles. + + To migrate a raw-tarball hook to ``source: {provider: hook-sdist}``, return + its artifact kind explicitly: + + .. code-block:: python + + import pathlib + + from packaging.requirements import Requirement + from packaging.version import Version + + from fromager import downloads + from fromager.context import WorkContext + from fromager.packagesettings import DownloadedSource, DownloadKind + + def download_source( + ctx: WorkContext, + req: Requirement, + version: Version, + download_url: str, + sdists_downloads_dir: pathlib.Path, + ) -> DownloadedSource: + """Download a raw source tarball and identify its kind.""" + path = downloads.download_url( + destination_dir=sdists_downloads_dir, + url=download_url, + destination_filename=f"{req.name}-{version}.tar.gz", + ) + return DownloadedSource(path=path, kind=DownloadKind.tarball) + + A hook that clones sources returns the checkout directory with + ``kind=DownloadKind.git_checkout``. Typed results are supported through + the configured hook profiles; packages using legacy settings still require + a bare ``Path``. .. autofromagerhook:: default_prepare_source diff --git a/src/fromager/bootstrapper/_bootstrapper.py b/src/fromager/bootstrapper/_bootstrapper.py index fa2451012..a22464b56 100644 --- a/src/fromager/bootstrapper/_bootstrapper.py +++ b/src/fromager/bootstrapper/_bootstrapper.py @@ -18,6 +18,7 @@ from .. import ( bootstrap_requirement_resolver, + packagesettings, progress, sources, threading_utils, @@ -483,14 +484,13 @@ def _download_source( source_url: str, ) -> pathlib.Path: """Download source for a package.""" - download_result = sources.download_source( + result: packagesettings.DownloadedSource = sources.download_source( ctx=self.ctx, req=req, version=resolved_version, download_url=source_url, ) - source_path: pathlib.Path = download_result[0] - return source_path + return result.path def _prepare_source( self, diff --git a/src/fromager/bootstrapper/_prepare_source.py b/src/fromager/bootstrapper/_prepare_source.py index 6076fdd2b..67df15469 100644 --- a/src/fromager/bootstrapper/_prepare_source.py +++ b/src/fromager/bootstrapper/_prepare_source.py @@ -43,9 +43,9 @@ def _bg_prepare_source( sdist_root_dir=unpacked / unpacked.stem, cached_wheel_filename=cached_wheel, ) - source_filename, _ = sources.download_source( + source_filename = sources.download_source( ctx=ctx, req=req, version=resolved_version, download_url=source_url - ) + ).path sdist_root_dir = sources.prepare_source( ctx=ctx, req=req, source_filename=source_filename, version=resolved_version ) diff --git a/src/fromager/commands/build.py b/src/fromager/commands/build.py index 42aaaf486..1fe34257c 100644 --- a/src/fromager/commands/build.py +++ b/src/fromager/commands/build.py @@ -384,12 +384,12 @@ def _build( # If we get here and still don't have a wheel filename, then we need to # build the wheel. if not wheel_filename: - source_filename, _ = sources.download_source( + source_filename = sources.download_source( ctx=wkctx, req=req, version=resolved_version, download_url=source_download_url, - ) + ).path logger.debug( "saved sdist of version %s from %s to %s", resolved_version, diff --git a/src/fromager/commands/step.py b/src/fromager/commands/step.py index 197d06a1f..95280934e 100644 --- a/src/fromager/commands/step.py +++ b/src/fromager/commands/step.py @@ -50,12 +50,12 @@ def download_source_archive( source_url, version = sources.resolve_source( ctx=wkctx, req=req, sdist_server_url=sdist_server_url ) - filename, _ = sources.download_source( + filename = sources.download_source( ctx=wkctx, req=req, version=version, download_url=source_url, - ) + ).path requirement_ctxvar.reset(token) print(filename) diff --git a/src/fromager/packagesettings/__init__.py b/src/fromager/packagesettings/__init__.py index 7dafd8f5e..5703c7589 100644 --- a/src/fromager/packagesettings/__init__.py +++ b/src/fromager/packagesettings/__init__.py @@ -17,6 +17,7 @@ from ._pbi import PackageBuildInfo from ._resolver import ( BuildSDist, + DownloadedSource, DownloadKind, GitHubTagCloneResolver, GitHubTagDownloadResolver, @@ -58,6 +59,7 @@ "BuildSDist", "DownloadKind", "DownloadSource", + "DownloadedSource", "EnvKey", "EnvVars", "ExternalCommands", diff --git a/src/fromager/packagesettings/_resolver.py b/src/fromager/packagesettings/_resolver.py index 27b980264..8c0494491 100644 --- a/src/fromager/packagesettings/_resolver.py +++ b/src/fromager/packagesettings/_resolver.py @@ -1,5 +1,6 @@ from __future__ import annotations +import dataclasses import datetime import enum import inspect @@ -47,6 +48,20 @@ def __bool__(self) -> bool: DownloadKindSet = frozenset[DownloadKind] +@dataclasses.dataclass(frozen=True, slots=True) +class DownloadedSource: + """Immutable result of downloading an artifact. + + ``path`` is the local artifact path. ``kind`` identifies how the artifact + should be handled; hook resolvers require a concrete, supported kind. + + .. versionadded:: 0.96.0 + """ + + path: pathlib.Path + kind: DownloadKind + + class AbstractResolver(pydantic.BaseModel): """Abstract base class for resolvers""" @@ -79,7 +94,7 @@ def download( ctx: context.WorkContext, req: Requirement, candidate: Candidate, - ) -> tuple[pathlib.Path, DownloadKind]: + ) -> DownloadedSource: """Download the resolved artifact for *candidate*.""" raise NotImplementedError @@ -89,7 +104,7 @@ def _download( req: Requirement, candidate: Candidate, download_kind: DownloadKind, - ) -> tuple[pathlib.Path, DownloadKind]: + ) -> DownloadedSource: """Download the resolved artifact for *candidate*. Returns the local path and the kind of artifact that was @@ -144,7 +159,7 @@ def _download( ) case _: # includes any_source and not_available typing.assert_never(download_kind) # type: ignore[arg-type] - return path, download_kind + return DownloadedSource(path=path, kind=download_kind) class CooldownMixin(pydantic.BaseModel): @@ -219,7 +234,7 @@ def download( ctx: context.WorkContext, req: Requirement, candidate: Candidate, - ) -> tuple[pathlib.Path, DownloadKind]: + ) -> DownloadedSource: """Download an sdist from PyPI.""" return self._download(ctx, req, candidate, DownloadKind.sdist) @@ -272,7 +287,7 @@ def download( ctx: context.WorkContext, req: Requirement, candidate: Candidate, - ) -> tuple[pathlib.Path, DownloadKind]: + ) -> DownloadedSource: """Download a pre-built wheel from PyPI.""" return self._download(ctx, req, candidate, DownloadKind.prebuilt_wheel) @@ -350,7 +365,7 @@ def download( ctx: context.WorkContext, req: Requirement, candidate: Candidate, - ) -> tuple[pathlib.Path, DownloadKind]: + ) -> DownloadedSource: """Download an sdist or tarball from a custom URL.""" return self._download(ctx, req, candidate, self.download_kind) @@ -433,7 +448,7 @@ def download( ctx: context.WorkContext, req: Requirement, candidate: Candidate, - ) -> tuple[pathlib.Path, DownloadKind]: + ) -> DownloadedSource: """Clone a git repository at a specific tag.""" return self._download(ctx, req, candidate, DownloadKind.git_checkout) @@ -596,7 +611,7 @@ def download( ctx: context.WorkContext, req: Requirement, candidate: Candidate, - ) -> tuple[pathlib.Path, DownloadKind]: + ) -> DownloadedSource: """Download a tarball from a GitHub tag.""" return self._download(ctx, req, candidate, DownloadKind.tarball) @@ -639,7 +654,7 @@ def download( ctx: context.WorkContext, req: Requirement, candidate: Candidate, - ) -> tuple[pathlib.Path, DownloadKind]: + ) -> DownloadedSource: """Clone a git repository from a GitHub tag.""" return self._download(ctx, req, candidate, DownloadKind.git_checkout) @@ -680,7 +695,7 @@ def download( ctx: context.WorkContext, req: Requirement, candidate: Candidate, - ) -> tuple[pathlib.Path, DownloadKind]: + ) -> DownloadedSource: """Download a tarball from a GitLab tag.""" return self._download(ctx, req, candidate, DownloadKind.tarball) @@ -723,7 +738,7 @@ def download( ctx: context.WorkContext, req: Requirement, candidate: Candidate, - ) -> tuple[pathlib.Path, DownloadKind]: + ) -> DownloadedSource: """Clone a git repository from a GitLab tag.""" return self._download(ctx, req, candidate, DownloadKind.git_checkout) @@ -748,7 +763,7 @@ def download( ctx: context.WorkContext, req: Requirement, candidate: Candidate, - ) -> tuple[pathlib.Path, DownloadKind]: + ) -> DownloadedSource: """Raise because package is not available.""" raise ValueError(f"package {req.name} is not available") @@ -812,14 +827,70 @@ def resolver_provider( ) -> resolver.BaseProvider: raise NotImplementedError - def download( + def _download_from_hook( self, ctx: context.WorkContext, req: Requirement, candidate: Candidate, - ) -> tuple[pathlib.Path, DownloadKind]: - """Raise because hook-based download is not yet implemented.""" - raise NotImplementedError("Hook resolver download needs a hook") + legacy_kind: DownloadKind, + ) -> DownloadedSource: + """Download with the package hook and normalize its result.""" + hook = overrides.find_override_method(req.name, "download_source") + if hook is None: + return self._download(ctx, req, candidate, legacy_kind) + + error_context = f"{req.name}: {self.provider!r} download_source hook" + try: + result: object = overrides.invoke( + hook, + ctx=ctx, + req=req, + version=candidate.version, + download_url=candidate.url, + sdists_downloads_dir=ctx.sdists_downloads, + ) + except Exception as err: + raise RuntimeError(f"{error_context} failed") from err + + logger.info("%s returned %s", error_context, result) + if isinstance(result, DownloadedSource): + downloaded = result + elif isinstance(result, pathlib.Path): + suffixes = ( + (".whl",) + if legacy_kind is DownloadKind.prebuilt_wheel + else (".tar.gz", ".zip") + ) + if result.is_dir() or not result.name.endswith(suffixes): + raise ValueError( + f"{error_context} returned legacy Path {result}; expected a " + f"{' or '.join(suffixes)} file. Return DownloadedSource with " + "an explicit supported DownloadKind for other artifact forms" + ) + downloaded = DownloadedSource(path=result, kind=legacy_kind) + else: + raise TypeError( + f"{error_context} returned {type(result).__name__}, " + "expected pathlib.Path or DownloadedSource" + ) + + if not isinstance(downloaded.path, pathlib.Path): + raise TypeError( + f"{error_context}: DownloadedSource.path must be pathlib.Path, " + f"got {type(downloaded.path).__name__}" + ) + if not isinstance(downloaded.kind, DownloadKind): + raise TypeError( + f"{error_context}: DownloadedSource.kind must be DownloadKind, " + f"got {type(downloaded.kind).__name__}" + ) + if downloaded.kind not in self.download_kinds: + supported = ", ".join(sorted(kind.value for kind in self.download_kinds)) + raise ValueError( + f"{error_context} returned unsupported kind {downloaded.kind.value!r}; " + f"expected DownloadedSource with one of: {supported}" + ) + return downloaded class HookSDistResolver(AbstractHookResolver): @@ -827,7 +898,9 @@ class HookSDistResolver(AbstractHookResolver): The ``hook-sdist`` provider delegates resolution and download to plugin hooks. The downloaded artifact can be a source distribution, - a tarball, or a git checkout. + a tarball, or a git checkout. Download hooks can return ``DownloadedSource`` + with a concrete kind, or a legacy archive ``Path`` treated as an sdist. + Without a download hook, the candidate is downloaded as an sdist. Example:: @@ -859,12 +932,24 @@ def resolver_provider( ignore_platform=False, ) + def download( + self, + ctx: context.WorkContext, + req: Requirement, + candidate: Candidate, + ) -> DownloadedSource: + """Download source via the ``download_source`` override hook.""" + return self._download_from_hook(ctx, req, candidate, DownloadKind.sdist) + class HookPrebuiltResolver(AbstractHookResolver): """Call resolver_provider and download_source hook, use pre-built wheel The ``hook-prebuilt`` provider delegates resolution and download to - plugin hooks. The downloaded artifact must be a pre-built wheel. + plugin hooks. The downloaded artifact must be a pre-built wheel. Download + hooks can return ``DownloadedSource`` with kind ``prebuilt_wheel`` or a + legacy wheel ``Path``. Without a download hook, the candidate is downloaded + into ``ctx.wheels_prebuilt``. Example:: @@ -897,6 +982,17 @@ def resolver_provider( ignore_platform=False, ) + def download( + self, + ctx: context.WorkContext, + req: Requirement, + candidate: Candidate, + ) -> DownloadedSource: + """Download a pre-built wheel via the ``download_source`` override hook.""" + return self._download_from_hook( + ctx, req, candidate, DownloadKind.prebuilt_wheel + ) + SourceResolver = typing.Annotated[ PyPISDistResolver diff --git a/src/fromager/sources.py b/src/fromager/sources.py index b5f674c05..12051259d 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -86,7 +86,7 @@ def download_source( req: Requirement, version: Version, download_url: str, -) -> tuple[pathlib.Path, packagesettings.DownloadKind]: +) -> packagesettings.DownloadedSource: """Download a resolved source and return its path and artifact kind. Configured source resolvers receive a minimal candidate. Legacy downloads @@ -115,7 +115,9 @@ def download_source( raise ValueError( f"expected a Path back to downloaded source. got {source_path}" ) - return source_path, packagesettings.DownloadKind.sdist + return packagesettings.DownloadedSource( + path=source_path, kind=packagesettings.DownloadKind.sdist + ) def get_source_provider( diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index 1a9cfd1e6..00b8a0044 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -37,7 +37,7 @@ from fromager.bootstrapper._work_item import WorkItem from fromager.context import WorkContext from fromager.dependency_graph import ROOT -from fromager.packagesettings import DownloadKind, WheelSettings +from fromager.packagesettings import DownloadedSource, DownloadKind, WheelSettings from fromager.requirements_file import RequirementType, SourceType @@ -1170,9 +1170,9 @@ def test_bg_prepare_source_log_prefix_includes_version( ), patch( "fromager.sources.download_source", - return_value=( - source_filename, - DownloadKind.sdist, + return_value=DownloadedSource( + path=source_filename, + kind=DownloadKind.sdist, ), ), patch( @@ -1222,7 +1222,7 @@ def test_bootstrapper_download_source_returns_downloaded_path( with patch( "fromager.bootstrapper._bootstrapper.sources.download_source", - return_value=(source_filename, DownloadKind.sdist), + return_value=DownloadedSource(path=source_filename, kind=DownloadKind.sdist), ) as download_source: result = bt._download_source(req, version, source_url) diff --git a/tests/test_commands.py b/tests/test_commands.py index 1f3ba7cd7..17572993b 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -46,7 +46,9 @@ def test_download_source_archive_prints_downloaded_path( patch.object( step.sources, "download_source", - return_value=(expected, packagesettings.DownloadKind.sdist), + return_value=packagesettings.DownloadedSource( + path=expected, kind=packagesettings.DownloadKind.sdist + ), ), ): result = cli_runner.invoke( @@ -83,7 +85,9 @@ def test_build_passes_downloaded_path_to_prepare_source( patch.object( build.sources, "download_source", - return_value=(source_filename, packagesettings.DownloadKind.sdist), + return_value=packagesettings.DownloadedSource( + path=source_filename, kind=packagesettings.DownloadKind.sdist + ), ), patch.object( build.sources, diff --git a/tests/test_packagesettings_resolver.py b/tests/test_packagesettings_resolver.py index 2dffa5705..54adfa56c 100644 --- a/tests/test_packagesettings_resolver.py +++ b/tests/test_packagesettings_resolver.py @@ -1,6 +1,8 @@ from __future__ import annotations +import dataclasses import datetime +import pathlib import re import textwrap import typing @@ -11,12 +13,14 @@ import yaml from packaging.requirements import Requirement from packaging.version import Version +from wheel.wheelfile import WheelFile from fromager import resolver from fromager.candidate import Candidate, Cooldown from fromager.context import WorkContext from fromager.packagesettings._resolver import ( BuildSDist, + DownloadedSource, DownloadKind, GitHubTagCloneResolver, GitHubTagDownloadResolver, @@ -190,9 +194,10 @@ def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> No expected = tmp_context.sdists_downloads / "test-pkg-1.2.3.tar.gz" mock_dl.return_value = expected r = _parse(self.YAML) - path, kind = r.download(tmp_context, _REQ, _CANDIDATE_SDIST) - assert path == expected - assert kind is DownloadKind.sdist + result = r.download(tmp_context, _REQ, _CANDIDATE_SDIST) + assert isinstance(result, DownloadedSource) + assert result.path == expected + assert result.kind is DownloadKind.sdist mock_dl.assert_called_once_with( destination_dir=tmp_context.sdists_downloads, url=_CANDIDATE_SDIST.url, @@ -231,9 +236,10 @@ def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> No expected = tmp_context.wheels_prebuilt / "test_pkg-1.2.3-py3-none-any.whl" mock_dl.return_value = expected r = _parse(self.YAML) - path, kind = r.download(tmp_context, _REQ, _CANDIDATE_WHEEL) - assert path == expected - assert kind is DownloadKind.prebuilt_wheel + result = r.download(tmp_context, _REQ, _CANDIDATE_WHEEL) + assert isinstance(result, DownloadedSource) + assert result.path == expected + assert result.kind is DownloadKind.prebuilt_wheel mock_dl.assert_called_once_with( destination_dir=tmp_context.wheels_prebuilt, url=_CANDIDATE_WHEEL.url, @@ -304,9 +310,10 @@ def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> No expected = tmp_context.sdists_downloads / "test-pkg-1.2.3.tar.gz" mock_dl.return_value = expected r = _parse(self.YAML) - path, kind = r.download(tmp_context, _REQ, _CANDIDATE_TARBALL) - assert path == expected - assert kind is DownloadKind.tarball + result = r.download(tmp_context, _REQ, _CANDIDATE_TARBALL) + assert isinstance(result, DownloadedSource) + assert result.path == expected + assert result.kind is DownloadKind.tarball mock_dl.assert_called_once_with( destination_dir=tmp_context.sdists_downloads, url=_CANDIDATE_TARBALL.url, @@ -374,9 +381,10 @@ def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> No ) mock_dl.return_value = expected_dest r = _parse(self.YAML) - path, kind = r.download(tmp_context, _REQ, _CANDIDATE_GIT) - assert path == expected_dest - assert kind is DownloadKind.git_checkout + result = r.download(tmp_context, _REQ, _CANDIDATE_GIT) + assert isinstance(result, DownloadedSource) + assert result.path == expected_dest + assert result.kind is DownloadKind.git_checkout mock_dl.assert_called_once_with( destination_dir=expected_dest, vcs_url=_CANDIDATE_GIT.url, @@ -493,9 +501,10 @@ def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> No expected = tmp_context.sdists_downloads / "test-pkg-1.2.3.tar.gz" mock_dl.return_value = expected r = _parse(self.YAML) - path, kind = r.download(tmp_context, _REQ, _CANDIDATE_GITHUB_TARBALL) - assert path == expected - assert kind is DownloadKind.tarball + result = r.download(tmp_context, _REQ, _CANDIDATE_GITHUB_TARBALL) + assert isinstance(result, DownloadedSource) + assert result.path == expected + assert result.kind is DownloadKind.tarball mock_dl.assert_called_once_with( destination_dir=tmp_context.sdists_downloads, url=_CANDIDATE_GITHUB_TARBALL.url, @@ -539,9 +548,10 @@ def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> No ) mock_dl.return_value = expected_dest r = _parse(self.YAML) - path, kind = r.download(tmp_context, _REQ, _CANDIDATE_GITHUB_CLONE) - assert path == expected_dest - assert kind is DownloadKind.git_checkout + result = r.download(tmp_context, _REQ, _CANDIDATE_GITHUB_CLONE) + assert isinstance(result, DownloadedSource) + assert result.path == expected_dest + assert result.kind is DownloadKind.git_checkout mock_dl.assert_called_once_with( destination_dir=expected_dest, vcs_url=_CANDIDATE_GITHUB_CLONE.url, @@ -580,9 +590,10 @@ def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> No expected = tmp_context.sdists_downloads / "test-pkg-1.2.3.tar.gz" mock_dl.return_value = expected r = _parse(self.YAML) - path, kind = r.download(tmp_context, _REQ, _CANDIDATE_GITLAB_TARBALL) - assert path == expected - assert kind is DownloadKind.tarball + result = r.download(tmp_context, _REQ, _CANDIDATE_GITLAB_TARBALL) + assert isinstance(result, DownloadedSource) + assert result.path == expected + assert result.kind is DownloadKind.tarball mock_dl.assert_called_once_with( destination_dir=tmp_context.sdists_downloads, url=_CANDIDATE_GITLAB_TARBALL.url, @@ -626,9 +637,10 @@ def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> No ) mock_dl.return_value = expected_dest r = _parse(self.YAML) - path, kind = r.download(tmp_context, _REQ, _CANDIDATE_GITLAB_CLONE) - assert path == expected_dest - assert kind is DownloadKind.git_checkout + result = r.download(tmp_context, _REQ, _CANDIDATE_GITLAB_CLONE) + assert isinstance(result, DownloadedSource) + assert result.path == expected_dest + assert result.kind is DownloadKind.git_checkout mock_dl.assert_called_once_with( destination_dir=expected_dest, vcs_url=_CANDIDATE_GITLAB_CLONE.url, @@ -729,11 +741,6 @@ def hook( "ignore_platform": False, } - def test_download(self, tmp_context: WorkContext) -> None: - r = _parse(self.YAML) - with pytest.raises(NotImplementedError, match="hook"): - r.download(tmp_context, _REQ, _CANDIDATE_SDIST) - class TestHookPrebuiltResolver: YAML = """\ @@ -784,10 +791,282 @@ def hook( "get_resolver_provider", ) - def test_download(self, tmp_context: WorkContext) -> None: - r = _parse(self.YAML) - with pytest.raises(NotImplementedError, match="hook"): - r.download(tmp_context, _REQ, _CANDIDATE_WHEEL) + +@pytest.mark.parametrize( + ("provider", "kind"), + [ + ("hook-sdist", DownloadKind.sdist), + ("hook-sdist", DownloadKind.tarball), + ("hook-sdist", DownloadKind.git_checkout), + ("hook-prebuilt", DownloadKind.prebuilt_wheel), + ], +) +def test_hook_download_preserves_typed_result_and_arguments( + tmp_context: WorkContext, provider: str, kind: DownloadKind +) -> None: + """Preserve the hook's explicit kind and forward the legacy arguments.""" + r = _parse(f"source:\n provider: {provider}") + candidate = _CANDIDATE_WHEEL if r.resolves_prebuilt_wheel else _CANDIDATE_SDIST + expected = DownloadedSource(path=tmp_context.work_dir / "artifact", kind=kind) + captured: dict[str, object] = {} + + def hook( + ctx: WorkContext, + req: Requirement, + version: Version, + download_url: str, + sdists_downloads_dir: pathlib.Path, + ) -> DownloadedSource: + captured.update( + ctx=ctx, + req=req, + version=version, + download_url=download_url, + sdists_downloads_dir=sdists_downloads_dir, + ) + return expected + + with ( + mock.patch( + "fromager.overrides.find_override_method", return_value=hook + ) as find_hook, + mock.patch.object(type(r), "_download") as fallback, + ): + result = r.download(tmp_context, _REQ, candidate) + + assert result is expected + assert captured == { + "ctx": tmp_context, + "req": _REQ, + "version": candidate.version, + "download_url": candidate.url, + "sdists_downloads_dir": tmp_context.sdists_downloads, + } + find_hook.assert_called_once_with(_REQ.name, "download_source") + fallback.assert_not_called() + + +@pytest.mark.parametrize( + ("provider", "filename", "kind"), + [ + ("hook-sdist", "test-pkg-1.2.3.tar.gz", DownloadKind.sdist), + ("hook-sdist", "test-pkg-1.2.3.zip", DownloadKind.sdist), + ( + "hook-prebuilt", + "test_pkg-1.2.3-py3-none-any.whl", + DownloadKind.prebuilt_wheel, + ), + ], +) +def test_hook_download_adapts_legacy_path_and_narrow_signature( + tmp_context: WorkContext, provider: str, filename: str, kind: DownloadKind +) -> None: + """Adapt bare paths without requiring hooks to accept every argument.""" + r = _parse(f"source:\n provider: {provider}") + expected = tmp_context.work_dir / filename + + def hook(req: Requirement, version: Version) -> pathlib.Path: + assert req == _REQ + assert version == _VERSION + return expected + + with mock.patch("fromager.overrides.find_override_method", return_value=hook): + result = r.download(tmp_context, _REQ, _CANDIDATE_SDIST) + + assert result == DownloadedSource(path=expected, kind=kind) + + +@pytest.mark.parametrize("provider", ["hook-sdist", "hook-prebuilt"]) +@pytest.mark.parametrize( + ("hook_result", "error_type", "message"), + [ + (None, TypeError, "expected pathlib.Path or DownloadedSource"), + ("not-a-path", TypeError, "expected pathlib.Path or DownloadedSource"), + (42, TypeError, "expected pathlib.Path or DownloadedSource"), + ( + (pathlib.Path("source"), DownloadKind.sdist), + TypeError, + "expected pathlib.Path or DownloadedSource", + ), + ( + DownloadedSource( + path=typing.cast(pathlib.Path, "source"), kind=DownloadKind.sdist + ), + TypeError, + "DownloadedSource.path must be pathlib.Path", + ), + ( + DownloadedSource( + path=pathlib.Path("source"), kind=typing.cast(DownloadKind, "sdist") + ), + TypeError, + "DownloadedSource.kind must be DownloadKind", + ), + ( + DownloadedSource(path=pathlib.Path("source"), kind=DownloadKind.any_source), + ValueError, + "unsupported kind 'any_source'", + ), + ( + DownloadedSource( + path=pathlib.Path("source"), kind=DownloadKind.not_available + ), + ValueError, + "unsupported kind 'n/a'", + ), + ], +) +def test_hook_download_rejects_invalid_results_without_fallback( + tmp_context: WorkContext, + provider: str, + hook_result: object, + error_type: type[Exception], + message: str, +) -> None: + """Reject malformed hook results with package and provider context.""" + r = _parse(f"source:\n provider: {provider}") + + def hook() -> object: + return hook_result + + with ( + mock.patch("fromager.overrides.find_override_method", return_value=hook), + mock.patch.object(type(r), "_download") as fallback, + pytest.raises(error_type, match=re.escape(message)) as exc_info, + ): + r.download(tmp_context, _REQ, _CANDIDATE_SDIST) + + assert f"test-pkg: '{provider}' download_source hook" in str(exc_info.value) + fallback.assert_not_called() + + +@pytest.mark.parametrize( + ("provider", "kind"), + [ + ("hook-sdist", DownloadKind.prebuilt_wheel), + ("hook-prebuilt", DownloadKind.sdist), + ("hook-prebuilt", DownloadKind.tarball), + ("hook-prebuilt", DownloadKind.git_checkout), + ], +) +def test_hook_download_rejects_kind_incompatible_with_profile( + tmp_context: WorkContext, provider: str, kind: DownloadKind +) -> None: + """Do not relabel an explicit kind to fit the selected profile.""" + r = _parse(f"source:\n provider: {provider}") + + def hook() -> DownloadedSource: + return DownloadedSource(path=tmp_context.work_dir, kind=kind) + + with ( + mock.patch("fromager.overrides.find_override_method", return_value=hook), + pytest.raises(ValueError, match=f"unsupported kind '{kind.value}'") as exc_info, + ): + r.download(tmp_context, _REQ, _CANDIDATE_SDIST) + + for allowed_kind in r.download_kinds: + assert allowed_kind.value in str(exc_info.value) + + +@pytest.mark.parametrize( + ("provider", "filename", "is_directory"), + [ + ("hook-sdist", "checkout", True), + ("hook-sdist", "source.tar.gz", True), + ("hook-sdist", "source.tar.xz", False), + ("hook-sdist", "test_pkg-1.2.3-py3-none-any.whl", False), + ("hook-prebuilt", "test_pkg-1.2.3-py3-none-any.whl", True), + ("hook-prebuilt", "source.tar.gz", False), + ], +) +def test_hook_download_requires_kind_for_unsupported_legacy_paths( + tmp_context: WorkContext, provider: str, filename: str, is_directory: bool +) -> None: + """Require an explicit kind when a path cannot use the legacy default.""" + r = _parse(f"source:\n provider: {provider}") + path = tmp_context.work_dir / filename + if is_directory: + path.mkdir() + else: + path.touch() + + def hook() -> pathlib.Path: + return path + + with ( + mock.patch("fromager.overrides.find_override_method", return_value=hook), + pytest.raises(ValueError, match="Return DownloadedSource"), + ): + r.download(tmp_context, _REQ, _CANDIDATE_SDIST) + + +@pytest.mark.parametrize("provider", ["hook-sdist", "hook-prebuilt"]) +def test_hook_download_uses_profile_default_when_hook_is_missing( + tmp_context: WorkContext, provider: str +) -> None: + """Select the correct download helper and destination without a hook.""" + r = _parse(f"source:\n provider: {provider}") + prebuilt = r.resolves_prebuilt_wheel + candidate = _CANDIDATE_WHEEL if prebuilt else _CANDIDATE_SDIST + destination = ( + tmp_context.wheels_prebuilt if prebuilt else tmp_context.sdists_downloads + ) + expected = destination / candidate.url.rsplit("/", 1)[1] + kind = DownloadKind.prebuilt_wheel if prebuilt else DownloadKind.sdist + + with ( + mock.patch("fromager.overrides.find_override_method", return_value=None), + mock.patch("fromager.downloads.download_sdist", return_value=expected) as sdist, + mock.patch("fromager.downloads.download_wheel", return_value=expected) as wheel, + ): + result = r.download(tmp_context, _REQ, candidate) + + selected, unused = (wheel, sdist) if prebuilt else (sdist, wheel) + selected.assert_called_once_with(destination_dir=destination, url=candidate.url) + unused.assert_not_called() + assert result == DownloadedSource(path=expected, kind=kind) + + +@pytest.mark.parametrize("provider", ["hook-sdist", "hook-prebuilt"]) +def test_hook_download_chains_failure_without_fallback( + tmp_context: WorkContext, provider: str +) -> None: + """Preserve the hook failure and do not attempt a default download.""" + r = _parse(f"source:\n provider: {provider}") + cause = OSError("download failed") + + def hook() -> pathlib.Path: + raise cause + + with ( + mock.patch("fromager.overrides.find_override_method", return_value=hook), + mock.patch.object(type(r), "_download") as fallback, + pytest.raises( + RuntimeError, match=f"'{provider}' download_source hook failed" + ) as exc_info, + ): + r.download(tmp_context, _REQ, _CANDIDATE_SDIST) + + assert exc_info.value.__cause__ is cause + fallback.assert_not_called() + + +def test_hook_prebuilt_download_fallback_accepts_valid_wheel( + tmp_context: WorkContext, +) -> None: + """Use wheel validation, rather than sdist validation, for a cached wheel.""" + path = tmp_context.wheels_prebuilt / "test_pkg-1.2.3-py3-none-any.whl" + with WheelFile(path, "w") as wheel: + wheel.writestr( + f"{wheel.dist_info_path}/METADATA", + "Metadata-Version: 2.1\nName: test-pkg\nVersion: 1.2.3\n", + ) + r = HookPrebuiltResolver(provider="hook-prebuilt") + + with mock.patch("fromager.overrides.find_override_method", return_value=None): + result = r.download(tmp_context, _REQ, _CANDIDATE_WHEEL) + + assert result == DownloadedSource(path=path, kind=DownloadKind.prebuilt_wheel) @mock.patch("fromager.packagesettings._resolver.overrides.find_override_method") @@ -911,3 +1190,16 @@ def test_missing_provider() -> None: source: index_url: https://pypi.test/simple """) + + +# -- DownloadedSource --------------------------------------------------------- + + +def test_downloaded_source_is_frozen() -> None: + ds = DownloadedSource( + path=pathlib.Path("/tmp/test.tar.gz"), kind=DownloadKind.sdist + ) + assert ds.path == pathlib.Path("/tmp/test.tar.gz") + assert ds.kind is DownloadKind.sdist + with pytest.raises(dataclasses.FrozenInstanceError): + ds.path = pathlib.Path("/other") # type: ignore[misc] diff --git a/tests/test_sources.py b/tests/test_sources.py index 40cdb7af6..94d0ca635 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -861,7 +861,9 @@ def test_download_source_regular_package( download_url="https://pkg.test/pkg-1.0.tar.gz", ) - assert result == (expected, packagesettings.DownloadKind.sdist) + assert result == packagesettings.DownloadedSource( + path=expected, kind=packagesettings.DownloadKind.sdist + ) mock_invoke.assert_called_once() @@ -875,11 +877,14 @@ def test_download_source_uses_configured_source_resolver( req = Requirement("pkg==1.0") version = Version("1.0") download_url = "https://pkg.test/pkg-1.0.tar.gz" + expected_result = packagesettings.DownloadedSource( + path=expected, kind=packagesettings.DownloadKind.sdist + ) with ( patch.object( packagesettings.PyPISDistResolver, "_download", - return_value=(expected, packagesettings.DownloadKind.sdist), + return_value=expected_result, ) as mock_download, patch.object( packagesettings.PackageBuildInfo, @@ -894,7 +899,7 @@ def test_download_source_uses_configured_source_resolver( ) expected_candidate = Candidate(name=req.name, version=version, url=download_url) - assert result == (expected, packagesettings.DownloadKind.sdist) + assert result == expected_result mock_download.assert_called_once_with( tmp_context, req, @@ -904,11 +909,15 @@ def test_download_source_uses_configured_source_resolver( mock_invoke.assert_not_called() -def test_download_source_returns_configured_result_without_validation( +def test_download_source_passes_configured_result_through( tmp_context: context.WorkContext, + tmp_path: pathlib.Path, ) -> None: - """Leave configured resolver result validation to the follow-up phase.""" - configured_result = "resolver-owned-result" + """Pass the configured resolver result through without modification.""" + configured_result = packagesettings.DownloadedSource( + path=tmp_path / "pkg-1.0.tar.gz", + kind=packagesettings.DownloadKind.tarball, + ) source_resolver = Mock() source_resolver.download.return_value = configured_result @@ -925,7 +934,49 @@ def test_download_source_returns_configured_result_without_validation( download_url="https://pkg.test/pkg-1.0.tar.gz", ) - assert result == configured_result + assert result is configured_result + + +def test_download_source_preserves_configured_hook_result( + tmp_context: context.WorkContext, +) -> None: + """Carry the explicit hook artifact kind through source dispatch.""" + req = Requirement("pkg==1.0") + version = Version("1.0") + download_url = "git+https://git.test/pkg.git@v1.0" + expected = packagesettings.DownloadedSource( + path=tmp_context.work_dir, + kind=packagesettings.DownloadKind.git_checkout, + ) + + def hook( + ctx: context.WorkContext, + req: Requirement, + version: Version, + download_url: str, + sdists_downloads_dir: pathlib.Path, + ) -> packagesettings.DownloadedSource: + assert ctx is tmp_context + assert req == Requirement("pkg==1.0") + assert version == Version("1.0") + assert download_url == "git+https://git.test/pkg.git@v1.0" + assert sdists_downloads_dir == tmp_context.sdists_downloads + return expected + + with ( + patch.object( + packagesettings.PackageBuildInfo, + "source_resolver", + new_callable=PropertyMock, + return_value=packagesettings.HookSDistResolver(provider="hook-sdist"), + ), + patch("fromager.overrides.find_override_method", return_value=hook), + ): + result = sources.download_source( + ctx=tmp_context, req=req, version=version, download_url=download_url + ) + + assert result is expected @patch("fromager.overrides.find_and_invoke", return_value="not-a-path") From ac8f01d989bb9b9ac9b389827624b90e8dfc25fa Mon Sep 17 00:00:00 2001 From: Shanmukh Pawan Date: Wed, 14 Oct 2026 10:53:39 -0400 Subject: [PATCH 08/35] fix(packagesettings): reject cross-level source conflicts Reject conflicting source and legacy settings during package settings loading, including explicit pre_built: false. Cover both configuration directions with focused tests and document the rule. Co-Authored-By: Codex Closes: #1368 Signed-off-by: Shanmukh Pawan --- docs/concepts/package-settings.rst | 11 +++++ src/fromager/packagesettings/_models.py | 41 +++++++++++++---- tests/test_packagesettings.py | 61 +++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 9 deletions(-) diff --git a/docs/concepts/package-settings.rst b/docs/concepts/package-settings.rst index 2f0a0d7f3..c516ff93a 100644 --- a/docs/concepts/package-settings.rst +++ b/docs/concepts/package-settings.rst @@ -35,6 +35,17 @@ Settings Loading Flow ``PackageBuildInfo`` objects are created lazily the first time a package is queried and cached for the rest of the run. +``PackageSettings`` validates source settings when it loads each package. +A package-level ``source`` cannot be combined with an explicitly set +variant ``pre_built`` or ``wheel_server_url``. A variant-level ``source`` +cannot be combined with package-level ``download_source`` or +``resolver_dist``. An explicit ``pre_built: false`` also conflicts; +an omitted ``pre_built`` does not. + +.. versionchanged:: 0.102.0 + Source and legacy settings mixed across package and variant levels + now raise a validation error when package settings load. + Directory Structure ------------------- diff --git a/src/fromager/packagesettings/_models.py b/src/fromager/packagesettings/_models.py index 783cd8d47..2bea2b820 100644 --- a/src/fromager/packagesettings/_models.py +++ b/src/fromager/packagesettings/_models.py @@ -32,6 +32,11 @@ logger = logging.getLogger(__name__) +# Legacy setting names that cannot be used with `source:`. +# Share them across the package and variant conflict checks. +_PACKAGE_LEGACY_SOURCE_FIELDS = frozenset({"download_source", "resolver_dist"}) +_VARIANT_LEGACY_SOURCE_FIELDS = frozenset({"wheel_server_url", "pre_built"}) + class SbomSettings(pydantic.BaseModel): """Global SBOM generation settings @@ -488,8 +493,7 @@ def validate_source_exclusivity(self) -> typing.Self: """Validate that ``source`` is mutually exclusive with legacy fields.""" if self.source is None: return self - legacy_fields = {"wheel_server_url", "pre_built"} - conflicts = legacy_fields & self.model_fields_set + conflicts = _VARIANT_LEGACY_SOURCE_FIELDS & self.model_fields_set if conflicts: raise ValueError( f"'source' is mutually exclusive with legacy settings: " @@ -641,17 +645,36 @@ def before_none_dict( @pydantic.model_validator(mode="after") def validate_source_exclusivity(self) -> typing.Self: - """Validate that ``source`` is mutually exclusive with legacy fields.""" - if self.source is None: - return self - legacy_fields = {"download_source", "resolver_dist"} - conflicts = legacy_fields & self.model_fields_set - if conflicts: + """Validate source and legacy fields across package and variant levels.""" + package_legacy_fields = _PACKAGE_LEGACY_SOURCE_FIELDS & self.model_fields_set + if self.source is not None and package_legacy_fields: raise ValueError( f"'source' is mutually exclusive with legacy settings: " - f"{sorted(conflicts)}. Use 'source' alone or remove it " + f"{sorted(package_legacy_fields)}. Use 'source' alone or remove it " f"and use the legacy settings." ) + + cross_level_conflicts: list[str] = [] + for variant_name, variant in self.variants.items(): + if self.source is not None: + variant_legacy_fields = ( + _VARIANT_LEGACY_SOURCE_FIELDS & variant.model_fields_set + ) + cross_level_conflicts.extend( + f"source and variants.{variant_name}.{field}" + for field in sorted(variant_legacy_fields) + ) + if variant.source is not None: + cross_level_conflicts.extend( + f"variants.{variant_name}.source and {field}" + for field in sorted(package_legacy_fields) + ) + + if cross_level_conflicts: + raise ValueError( + "source is mutually exclusive with legacy settings across " + "package and variant levels: " + "; ".join(cross_level_conflicts) + ) return self @classmethod diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index 5ecc745ce..c799edb1d 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -1309,6 +1309,67 @@ def test_variant_source_exclusivity_invalid(yaml_str: str) -> None: PackageSettings.from_string("conflict-pkg", yaml_str) +@pytest.mark.parametrize( + ("yaml_str", "source_path", "legacy_path"), + [ + pytest.param( + "source:\n provider: pypi-sdist\nvariants:\n cpu:\n pre_built: true\n", + "source", + "variants.cpu.pre_built", + id="package-source-variant-pre-built", + ), + pytest.param( + "source:\n provider: pypi-sdist\n" + "variants:\n cpu:\n pre_built: false\n", + "source", + "variants.cpu.pre_built", + id="package-source-variant-explicit-false", + ), + pytest.param( + "source:\n provider: pypi-sdist\n" + "variants:\n cpu:\n wheel_server_url: https://wheel.test/simple\n", + "source", + "variants.cpu.wheel_server_url", + id="package-source-variant-wheel-server", + ), + pytest.param( + "download_source:\n url: https://egg.test/pkg.tar.gz\n" + "variants:\n cpu:\n source:\n provider: pypi-sdist\n", + "variants.cpu.source", + "download_source", + id="variant-source-package-download-source", + ), + pytest.param( + "resolver_dist:\n sdist_server_url: https://sdist.test/simple\n" + "variants:\n cpu:\n source:\n provider: pypi-sdist\n", + "variants.cpu.source", + "resolver_dist", + id="variant-source-package-resolver-dist", + ), + ], +) +def test_cross_level_source_exclusivity_invalid( + yaml_str: str, source_path: str, legacy_path: str +) -> None: + """Reject source and legacy fields mixed across package and variant levels.""" + with pytest.raises(RuntimeError, match="mutually exclusive") as exc_info: + PackageSettings.from_string("conflict-pkg", yaml_str) + error_str = str(exc_info.value) + assert source_path in error_str + assert legacy_path in error_str + + +def test_package_source_with_unrelated_variant_setting_valid() -> None: + """Omitted variant legacy fields do not conflict with package source.""" + yaml_str = ( + "source:\n provider: pypi-sdist\n" + "variants:\n cpu:\n env:\n BUILD_MODE: fast\n" + ) + ps = PackageSettings.from_string("source-pkg", yaml_str) + assert ps.source is not None + assert ps.variants[Variant("cpu")].pre_built is False + + def test_legacy_only_config_valid() -> None: """Legacy-only config loads without error.""" yaml_str = ( From ff5216c5cb054a71baa9dfba27f312d450dae814 Mon Sep 17 00:00:00 2001 From: Justin Larkin Date: Thu, 22 Oct 2026 10:36:09 -0400 Subject: [PATCH 09/35] fix(sources): honor resolver sdist build strategy Honor configured PEP 517 and tarball strategies, warning when an installed build hook is ignored. Require resolvers to declare their strategy and hook-sdist to choose it explicitly. Keep build overrides for legacy settings. Co-Authored-By: Codex Closes: #1206 Related: #1256 Signed-off-by: Justin Larkin --- docs/concepts/resolver-architecture.rst | 8 + docs/reference/hooks.rst | 4 +- src/fromager/packagesettings/_resolver.py | 16 +- src/fromager/sources.py | 65 +++++-- tests/test_packagesettings_resolver.py | 87 ++++++++-- tests/test_sources.py | 202 +++++++++++++++++++++- 6 files changed, 353 insertions(+), 29 deletions(-) diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index 06899f799..0da851d7c 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -118,6 +118,14 @@ the artifact kind available for source preparation; consuming it in preparation and integrating prebuilt detection into the bootstrapper are separate steps. See :ref:`download_source_hook` for the return contract and compatibility rules. +.. versionchanged:: 0.102.0 + Configured source resolvers also select the sdist build method. Git checkout + resolvers use PEP 517 by default; hook sdist resolvers require an explicit + ``build_sdist: pep517`` or ``build_sdist: tarball``. PyPI sdist resolvers + repack a tarball. Both configured strategies take precedence over an installed + ``build_sdist`` override and log a warning when it is ignored. Legacy + settings continue to use the override. + Version Filtering Window ------------------------- diff --git a/docs/reference/hooks.rst b/docs/reference/hooks.rst index b8e90ef3b..c92149684 100644 --- a/docs/reference/hooks.rst +++ b/docs/reference/hooks.rst @@ -264,8 +264,8 @@ Source hooks downloads require a download hook. The ``get_resolver_provider`` hook remains required for both profiles. - To migrate a raw-tarball hook to ``source: {provider: hook-sdist}``, return - its artifact kind explicitly: + To migrate a raw-tarball hook to ``hook-sdist``, specify ``build_sdist`` + in YAML and return its artifact kind explicitly: .. code-block:: python diff --git a/src/fromager/packagesettings/_resolver.py b/src/fromager/packagesettings/_resolver.py index 8c0494491..ac57756ac 100644 --- a/src/fromager/packagesettings/_resolver.py +++ b/src/fromager/packagesettings/_resolver.py @@ -69,6 +69,9 @@ class AbstractResolver(pydantic.BaseModel): provider: str + build_sdist: BuildSDist | None + """Sdist build strategy; None for resolvers that do not build from source.""" + supports_override_hooks: typing.ClassVar[bool] = False """Does resolver support override hooks?""" @@ -208,7 +211,7 @@ class PyPISDistResolver(AbstractPyPIResolver): # It is not safe to use PEP 517 to re-generate a source distribution. # Some PEP 517 backends require VCS to generate correct sdist. - build_sdist: typing.ClassVar[BuildSDist | None] = BuildSDist.tarball + build_sdist: typing.Literal[BuildSDist.tarball] = BuildSDist.tarball download_kinds: typing.ClassVar[DownloadKindSet] = frozenset({DownloadKind.sdist}) def resolver_provider( @@ -258,7 +261,7 @@ class PyPIPrebuiltResolver(AbstractPyPIResolver): provider: typing.Literal["pypi-prebuilt"] - build_sdist: typing.ClassVar[BuildSDist | None] = None + build_sdist: None = None download_kinds: typing.ClassVar[DownloadKindSet] = frozenset( {DownloadKind.prebuilt_wheel} ) @@ -322,7 +325,7 @@ class PyPIDownloadResolver(AbstractPyPIResolver): URL must contain '{version}' template string. """ - build_sdist: typing.ClassVar[BuildSDist | None] = BuildSDist.tarball + build_sdist: typing.Literal[BuildSDist.tarball] = BuildSDist.tarball download_kinds: typing.ClassVar[DownloadKindSet] = frozenset( {DownloadKind.sdist, DownloadKind.tarball} ) @@ -747,6 +750,7 @@ class NotAvailableResolver(AbstractResolver): """Prevent resolve and download""" provider: typing.Literal["not-available"] + build_sdist: None = None def resolver_provider( self, @@ -901,13 +905,18 @@ class HookSDistResolver(AbstractHookResolver): a tarball, or a git checkout. Download hooks can return ``DownloadedSource`` with a concrete kind, or a legacy archive ``Path`` treated as an sdist. Without a download hook, the candidate is downloaded as an sdist. + Specify ``build_sdist`` explicitly: ``tarball`` to repack an existing + sdist, or ``pep517`` to build from a git checkout. Example:: provider: hook-sdist + build_sdist: tarball """ provider: typing.Literal["hook-sdist"] + build_sdist: BuildSDist + """Source distribution build method.""" download_kinds: typing.ClassVar[DownloadKindSet] = frozenset( {DownloadKind.sdist, DownloadKind.tarball, DownloadKind.git_checkout} @@ -957,6 +966,7 @@ class HookPrebuiltResolver(AbstractHookResolver): """ provider: typing.Literal["hook-prebuilt"] + build_sdist: None = None download_kinds: typing.ClassVar[DownloadKindSet] = frozenset( {DownloadKind.prebuilt_wheel} diff --git a/src/fromager/sources.py b/src/fromager/sources.py index 12051259d..efd2f20ea 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -512,8 +512,9 @@ def build_sdist( sdist_root_dir: pathlib.Path, build_env: build_environment.BuildEnvironment, ) -> pathlib.Path: - """Build source distribution""" + """Build a source distribution using the configured strategy or legacy hook.""" pbi = ctx.package_build_info(req) + source_resolver = pbi.source_resolver build_dir = pbi.build_dir(sdist_root_dir) logger.info(f"building {ctx.variant} source distribution for {req} in {build_dir}") @@ -524,18 +525,56 @@ def build_sdist( sdist_root_dir=sdist_root_dir, build_env=build_env, ) - sdist_file: pathlib.Path = overrides.find_and_invoke( - req.name, - "build_sdist", - default_build_sdist, - ctx=ctx, - extra_environ=extra_environ, - req=req, - version=version, - sdist_root_dir=sdist_root_dir, - build_dir=build_dir, - build_env=build_env, - ) + if source_resolver is not None: + if overrides.find_override_method(req.name, "build_sdist") is not None: + logger.warning( + "%s: ignoring build_sdist override because source.build_sdist is %s", + req.name, + source_resolver.build_sdist, + ) + if ( + source_resolver is not None + and source_resolver.build_sdist is packagesettings.BuildSDist.pep517 + ): + sdist_file = pep517_build_sdist( + ctx=ctx, + extra_environ=extra_environ, + req=req, + version=version, + sdist_root_dir=sdist_root_dir, + build_env=build_env, + ) + elif ( + source_resolver is not None + and source_resolver.build_sdist is packagesettings.BuildSDist.tarball + ): + sdist_file = default_build_sdist( + ctx=ctx, + extra_environ=extra_environ, + req=req, + version=version, + sdist_root_dir=sdist_root_dir, + build_dir=build_dir, + build_env=build_env, + ) + elif source_resolver is None: + sdist_file = overrides.find_and_invoke( + req.name, + "build_sdist", + default_build_sdist, + ctx=ctx, + extra_environ=extra_environ, + req=req, + version=version, + sdist_root_dir=sdist_root_dir, + build_dir=build_dir, + build_env=build_env, + ) + else: + raise ValueError( + f"{req.name}: unsupported source.build_sdist strategy " + f"{source_resolver.build_sdist!r}" + ) logger.info(f"built source distribution {sdist_file}") # validate location and file name diff --git a/tests/test_packagesettings_resolver.py b/tests/test_packagesettings_resolver.py index 54adfa56c..8a942d12c 100644 --- a/tests/test_packagesettings_resolver.py +++ b/tests/test_packagesettings_resolver.py @@ -19,6 +19,7 @@ from fromager.candidate import Candidate, Cooldown from fromager.context import WorkContext from fromager.packagesettings._resolver import ( + AbstractResolver, BuildSDist, DownloadedSource, DownloadKind, @@ -128,6 +129,22 @@ def _parse(raw_yaml: str) -> SourceResolver: return wrapper.source +def test_resolver_requires_build_sdist_strategy() -> None: + """Require every resolver to declare an sdist build strategy.""" + with pytest.raises(pydantic.ValidationError, match="build_sdist"): + AbstractResolver.model_validate({"provider": "test"}) + + +@pytest.mark.parametrize( + "provider", + ["pypi-sdist", "pypi-prebuilt", "hook-prebuilt", "not-available"], +) +def test_resolver_rejects_unsupported_build_sdist_strategy(provider: str) -> None: + """Keep fixed resolver strategies from accepting a different build method.""" + with pytest.raises(pydantic.ValidationError, match="build_sdist"): + _parse(f"source:\n provider: {provider}\n build_sdist: pep517\n") + + # -- PyPI resolvers ----------------------------------------------------------- @@ -679,18 +696,34 @@ class TestHookSDistResolver: YAML = """\ source: provider: hook-sdist + build_sdist: tarball """ def test_parse(self) -> None: r = _parse(self.YAML) assert isinstance(r, HookSDistResolver) assert r.provider == "hook-sdist" + assert r.build_sdist is BuildSDist.tarball assert r.supports_override_hooks is True assert r.resolves_prebuilt_wheel is False assert r.download_kinds == frozenset( {DownloadKind.sdist, DownloadKind.tarball, DownloadKind.git_checkout} ) + @pytest.mark.parametrize("build_method", list(BuildSDist)) + def test_build_sdist_method(self, build_method: BuildSDist) -> None: + """Load the hook sdist build method from YAML.""" + r = _parse( + f"source:\n provider: hook-sdist\n build_sdist: {build_method.value}\n" + ) + + assert r.build_sdist is build_method + + def test_build_sdist_method_required(self) -> None: + """Require a build method because hook artifacts need different strategies.""" + with pytest.raises(pydantic.ValidationError, match="build_sdist"): + _parse("source:\n provider: hook-sdist\n") + @mock.patch("fromager.packagesettings._resolver.overrides.find_override_method") def test_resolver_provider( self, @@ -805,7 +838,11 @@ def test_hook_download_preserves_typed_result_and_arguments( tmp_context: WorkContext, provider: str, kind: DownloadKind ) -> None: """Preserve the hook's explicit kind and forward the legacy arguments.""" - r = _parse(f"source:\n provider: {provider}") + r = _parse( + TestHookSDistResolver.YAML + if provider == "hook-sdist" + else TestHookPrebuiltResolver.YAML + ) candidate = _CANDIDATE_WHEEL if r.resolves_prebuilt_wheel else _CANDIDATE_SDIST expected = DownloadedSource(path=tmp_context.work_dir / "artifact", kind=kind) captured: dict[str, object] = {} @@ -862,7 +899,11 @@ def test_hook_download_adapts_legacy_path_and_narrow_signature( tmp_context: WorkContext, provider: str, filename: str, kind: DownloadKind ) -> None: """Adapt bare paths without requiring hooks to accept every argument.""" - r = _parse(f"source:\n provider: {provider}") + r = _parse( + TestHookSDistResolver.YAML + if provider == "hook-sdist" + else TestHookPrebuiltResolver.YAML + ) expected = tmp_context.work_dir / filename def hook(req: Requirement, version: Version) -> pathlib.Path: @@ -924,7 +965,11 @@ def test_hook_download_rejects_invalid_results_without_fallback( message: str, ) -> None: """Reject malformed hook results with package and provider context.""" - r = _parse(f"source:\n provider: {provider}") + r = _parse( + TestHookSDistResolver.YAML + if provider == "hook-sdist" + else TestHookPrebuiltResolver.YAML + ) def hook() -> object: return hook_result @@ -953,7 +998,11 @@ def test_hook_download_rejects_kind_incompatible_with_profile( tmp_context: WorkContext, provider: str, kind: DownloadKind ) -> None: """Do not relabel an explicit kind to fit the selected profile.""" - r = _parse(f"source:\n provider: {provider}") + r = _parse( + TestHookSDistResolver.YAML + if provider == "hook-sdist" + else TestHookPrebuiltResolver.YAML + ) def hook() -> DownloadedSource: return DownloadedSource(path=tmp_context.work_dir, kind=kind) @@ -983,7 +1032,11 @@ def test_hook_download_requires_kind_for_unsupported_legacy_paths( tmp_context: WorkContext, provider: str, filename: str, is_directory: bool ) -> None: """Require an explicit kind when a path cannot use the legacy default.""" - r = _parse(f"source:\n provider: {provider}") + r = _parse( + TestHookSDistResolver.YAML + if provider == "hook-sdist" + else TestHookPrebuiltResolver.YAML + ) path = tmp_context.work_dir / filename if is_directory: path.mkdir() @@ -1005,7 +1058,11 @@ def test_hook_download_uses_profile_default_when_hook_is_missing( tmp_context: WorkContext, provider: str ) -> None: """Select the correct download helper and destination without a hook.""" - r = _parse(f"source:\n provider: {provider}") + r = _parse( + TestHookSDistResolver.YAML + if provider == "hook-sdist" + else TestHookPrebuiltResolver.YAML + ) prebuilt = r.resolves_prebuilt_wheel candidate = _CANDIDATE_WHEEL if prebuilt else _CANDIDATE_SDIST destination = ( @@ -1032,7 +1089,11 @@ def test_hook_download_chains_failure_without_fallback( tmp_context: WorkContext, provider: str ) -> None: """Preserve the hook failure and do not attempt a default download.""" - r = _parse(f"source:\n provider: {provider}") + r = _parse( + TestHookSDistResolver.YAML + if provider == "hook-sdist" + else TestHookPrebuiltResolver.YAML + ) cause = OSError("download failed") def hook() -> pathlib.Path: @@ -1087,7 +1148,9 @@ def legacy_hook( return provider find_override_method.return_value = legacy_hook - resolver_model = HookSDistResolver(provider="hook-sdist") + resolver_model = HookSDistResolver( + provider="hook-sdist", build_sdist=BuildSDist.tarball + ) result = resolver_model.resolver_provider(tmp_context, _REQ, _REQ_TYPE) @@ -1105,7 +1168,9 @@ def test_hook_resolver_does_not_fall_back_when_hook_is_missing( tmp_context: WorkContext, ) -> None: """A missing hook fails instead of selecting the default PyPI provider.""" - resolver_model = HookSDistResolver(provider="hook-sdist") + resolver_model = HookSDistResolver( + provider="hook-sdist", build_sdist=BuildSDist.tarball + ) with pytest.raises(ValueError, match="requires a get_resolver_provider"): resolver_model.resolver_provider(tmp_context, _REQ, _REQ_TYPE) @@ -1164,7 +1229,9 @@ def hook( raise ValueError("hook failure") find_override_method.return_value = hook - resolver_model = HookSDistResolver(provider="hook-sdist") + resolver_model = HookSDistResolver( + provider="hook-sdist", build_sdist=BuildSDist.tarball + ) with pytest.raises(RuntimeError, match="hook-sdist") as exc_info: resolver_model.resolver_provider(tmp_context, _REQ, _REQ_TYPE) diff --git a/tests/test_sources.py b/tests/test_sources.py index 94d0ca635..2816db5b3 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -116,6 +116,7 @@ def hook( tmp_context.cooldown = Cooldown(min_age=datetime.timedelta(days=7)) source_resolver = packagesettings.HookSDistResolver( provider="hook-sdist", + build_sdist=packagesettings.BuildSDist.tarball, min_release_age=14, ) @@ -968,7 +969,9 @@ def hook( packagesettings.PackageBuildInfo, "source_resolver", new_callable=PropertyMock, - return_value=packagesettings.HookSDistResolver(provider="hook-sdist"), + return_value=packagesettings.HookSDistResolver( + provider="hook-sdist", build_sdist=packagesettings.BuildSDist.pep517 + ), ), patch("fromager.overrides.find_override_method", return_value=hook), ): @@ -1061,6 +1064,203 @@ def test_prepare_source_plugin_returns_bad_tuple( ) +@pytest.mark.parametrize( + "source_resolver", + [ + packagesettings.GitHubTagCloneResolver( + provider="github-tag-git", + project_url=pydantic.HttpUrl("https://github.test/org/pkg"), + ), + packagesettings.HookSDistResolver( + provider="hook-sdist", build_sdist=packagesettings.BuildSDist.pep517 + ), + ], + ids=["git-clone", "hook-source"], +) +@pytest.mark.parametrize("has_override", [False, True]) +def test_build_sdist_uses_pep517_for_configured_source( + source_resolver: ( + packagesettings.GitHubTagCloneResolver | packagesettings.HookSDistResolver + ), + has_override: bool, + tmp_context: context.WorkContext, + tmp_path: pathlib.Path, + caplog: pytest.LogCaptureFixture, +) -> None: + """Prefer PEP 517 and warn when it replaces an installed build hook.""" + req = Requirement("pkg==1.0") + version = Version("1.0") + build_env = Mock() + sdist_root_dir = tmp_path / "src" + sdist_file = tmp_context.sdists_builds / "pkg-1.0.tar.gz" + sdist_file.write_text("fake sdist") + override = Mock() if has_override else None + with ( + patch.object( + packagesettings.PackageBuildInfo, + "source_resolver", + new_callable=PropertyMock, + return_value=source_resolver, + ), + patch("fromager.packagesettings.get_extra_environ", return_value={}), + patch("fromager.sources.pep517_build_sdist", return_value=sdist_file) as pep517, + patch("fromager.sources.overrides.find_and_invoke") as find_and_invoke, + patch( + "fromager.sources.overrides.find_override_method", return_value=override + ) as find_override, + ): + result = sources.build_sdist( + ctx=tmp_context, + req=req, + version=version, + sdist_root_dir=sdist_root_dir, + build_env=build_env, + ) + + assert result == sdist_file.resolve() + pep517.assert_called_once_with( + ctx=tmp_context, + extra_environ={}, + req=req, + version=version, + sdist_root_dir=sdist_root_dir, + build_env=build_env, + ) + find_and_invoke.assert_not_called() + find_override.assert_called_once_with(req.name, "build_sdist") + warning = "ignoring build_sdist override because source.build_sdist is pep517" + assert (warning in caplog.text) is has_override + if override is not None: + override.assert_not_called() + + +@pytest.mark.parametrize( + "source_resolver", + [ + None, + packagesettings.HookSDistResolver( + provider="hook-sdist", build_sdist=packagesettings.BuildSDist.tarball + ), + packagesettings.GitHubTagCloneResolver( + provider="github-tag-git", + project_url=pydantic.HttpUrl("https://github.test/org/pkg"), + build_sdist=packagesettings.BuildSDist.tarball, + ), + ], + ids=["legacy", "hook-source", "configured-tarball"], +) +@pytest.mark.parametrize("has_override", [False, True]) +def test_build_sdist_uses_configured_tarball_or_legacy_hook( + source_resolver: ( + packagesettings.HookSDistResolver + | packagesettings.GitHubTagCloneResolver + | None + ), + has_override: bool, + tmp_context: context.WorkContext, + tmp_path: pathlib.Path, + caplog: pytest.LogCaptureFixture, +) -> None: + """Use configured tarball builds directly and keep overrides for legacy settings.""" + req = Requirement("pkg==1.0") + version = Version("1.0") + build_env = Mock() + sdist_root_dir = tmp_path / "src" + sdist_file = tmp_context.sdists_builds / "pkg-1.0.tar.gz" + sdist_file.write_text("fake sdist") + override = Mock() if has_override else None + + with ( + patch.object( + packagesettings.PackageBuildInfo, + "source_resolver", + new_callable=PropertyMock, + return_value=source_resolver, + ), + patch("fromager.packagesettings.get_extra_environ", return_value={}), + patch( + "fromager.sources.overrides.find_and_invoke", return_value=sdist_file + ) as find_and_invoke, + patch("fromager.sources.pep517_build_sdist") as pep517, + patch( + "fromager.sources.default_build_sdist", return_value=sdist_file + ) as tarball, + patch( + "fromager.sources.overrides.find_override_method", return_value=override + ) as find_override, + ): + result = sources.build_sdist( + ctx=tmp_context, + req=req, + version=version, + sdist_root_dir=sdist_root_dir, + build_env=build_env, + ) + + assert result == sdist_file.resolve() + build_kwargs = dict( + ctx=tmp_context, + extra_environ={}, + req=req, + version=version, + sdist_root_dir=sdist_root_dir, + build_dir=sdist_root_dir, + build_env=build_env, + ) + if source_resolver is None: + find_and_invoke.assert_called_once_with( + req.name, "build_sdist", tarball, **build_kwargs + ) + tarball.assert_not_called() + find_override.assert_not_called() + else: + tarball.assert_called_once_with(**build_kwargs) + find_and_invoke.assert_not_called() + find_override.assert_called_once_with(req.name, "build_sdist") + if override is not None: + override.assert_not_called() + warning = "ignoring build_sdist override because source.build_sdist is tarball" + assert (warning in caplog.text) is (source_resolver is not None and has_override) + pep517.assert_not_called() + + +@pytest.mark.parametrize("strategy", [None, "unsupported"]) +def test_build_sdist_rejects_unsupported_configured_strategy( + strategy: str | None, + tmp_context: context.WorkContext, + tmp_path: pathlib.Path, +) -> None: + """Reject unsupported configured strategies without calling a builder or hook.""" + source_resolver = Mock(build_sdist=strategy) + with ( + patch.object( + packagesettings.PackageBuildInfo, + "source_resolver", + new_callable=PropertyMock, + return_value=source_resolver, + ), + patch("fromager.packagesettings.get_extra_environ", return_value={}), + patch("fromager.sources.overrides.find_override_method", return_value=None), + patch("fromager.sources.overrides.find_and_invoke") as find_and_invoke, + patch("fromager.sources.pep517_build_sdist") as pep517, + patch("fromager.sources.default_build_sdist") as tarball, + ): + with pytest.raises( + ValueError, match=r"unsupported source\.build_sdist strategy" + ): + sources.build_sdist( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + version=Version("1.0"), + sdist_root_dir=tmp_path / "src", + build_env=Mock(), + ) + + find_and_invoke.assert_not_called() + pep517.assert_not_called() + tarball.assert_not_called() + + @patch("fromager.overrides.find_and_invoke") @patch("fromager.packagesettings.get_extra_environ", return_value={}) def test_build_sdist_raises_file_not_found( From c5e3a28704ef35e68b750b72a02e6bf642d65973 Mon Sep 17 00:00:00 2001 From: Shanmukh Pawan Date: Fri, 23 Oct 2026 12:25:31 -0400 Subject: [PATCH 10/35] fix(sources): prepare Git checkouts by download kind Carry DownloadedSource.kind through bootstrap and build entry points. Unpack ordinary archives; prepare clean Git checkouts in place so patches, project overrides, and Rust vendoring run. Keep prepare_source override arguments and path-only callers compatible. Document the public Git helper and no-cleanup behavior, and reject dirty nested submodules before preparation. Reject archive input paths that cleanup would remove before extraction, preserving the input and reporting an actionable error. Closes: #1205 Co-Authored-By: Codex Signed-off-by: Shanmukh Pawan --- docs/concepts/bootstrapper-architecture.rst | 7 +- docs/concepts/hooks-and-overrides.rst | 6 + docs/concepts/resolver-architecture.rst | 31 +- docs/reference/hooks.rst | 35 +- docs/using.md | 16 +- src/fromager/bootstrapper/_bootstrapper.py | 9 +- src/fromager/bootstrapper/_prepare_source.py | 12 +- src/fromager/commands/build.py | 11 +- src/fromager/commands/step.py | 67 ++- src/fromager/packagesettings/_resolver.py | 9 + src/fromager/sources.py | 133 ++++- tests/test_bootstrapper.py | 67 ++- tests/test_commands.py | 179 +++++- tests/test_packagesettings_resolver.py | 128 +++- tests/test_sources.py | 597 +++++++++++++++++++ 15 files changed, 1251 insertions(+), 56 deletions(-) diff --git a/docs/concepts/bootstrapper-architecture.rst b/docs/concepts/bootstrapper-architecture.rst index 80a3532b2..37d7d5493 100644 --- a/docs/concepts/bootstrapper-architecture.rst +++ b/docs/concepts/bootstrapper-architecture.rst @@ -57,7 +57,8 @@ the contract every phase must satisfy: * - ``Start`` - Add to dependency graph; deduplicate already-seen packages * - ``PrepareSource`` - - Download source or prebuilt wheel; read build-system deps + - Download source or prebuilt wheel; prepare archives or checkouts; + read build-system deps * - ``PrepareBuild`` - Install build-system deps; discover backend and sdist deps * - ``Build`` @@ -67,6 +68,10 @@ the contract every phase must satisfy: * - ``Complete`` - Clean up build directories (terminal phase) +.. versionchanged:: 0.102.0 + ``PrepareSource`` passes the downloaded artifact kind through preparation, + so Git checkouts can be prepared without unpacking them. + Each phase wraps a ``WorkItem`` dataclass that accumulates per-package state (resolved version, build environment, build result, etc.) as it moves through the pipeline. diff --git a/docs/concepts/hooks-and-overrides.rst b/docs/concepts/hooks-and-overrides.rst index 9d5274995..6b048bbf2 100644 --- a/docs/concepts/hooks-and-overrides.rst +++ b/docs/concepts/hooks-and-overrides.rst @@ -62,6 +62,12 @@ arguments. download results with explicit artifact kinds. See :ref:`download_source_hook` for the contract, legacy path defaults, and a migration example. +.. versionchanged:: 0.102.0 + + ``prepare_source`` selects an archive or Git default using the downloaded + artifact kind. Package overrides remain active. See :ref:`prepare_source_hook` + for their arguments and the public Git checkout helper. + Global Hooks ------------ diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index 0da851d7c..4702b957b 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -113,15 +113,34 @@ settings are used. Hook downloads accept explicit artifact kinds and adapt legacy paths using profile-specific defaults. If no download hook exists, the profile selects the standard sdist or wheel downloader and its destination directory. Hook errors -propagate with context instead of triggering a fallback. The typed result makes -the artifact kind available for source preparation; consuming it in preparation -and integrating prebuilt detection into the bootstrapper are separate steps. +propagate with context instead of triggering a fallback. In-process callers +pass the downloaded path and kind to preparation: ``sdist`` and ``tarball`` +are unpacked, while ``git_checkout`` uses the checkout directory. The checkout +uses the same double-nested work directory pattern as an unpacked archive; +the directory names can differ for normalized package names. +The default Git preparer rejects a checkout with local changes before applying +patches, project overrides, and Rust vendoring. Git status does not prove that the +checkout is the original clone or preserve it after preparation. Build metadata +is written after preparation but is not used to choose the preparation path. +With ``--no-cleanup``, downloading to an existing checkout fails and preserves +its files for inspection. Enable cleanup or use a new work directory to +download and prepare a fresh checkout. Git status cannot detect changes to +patch or settings files outside the checkout. If preparation left the checkout +clean, a manual second ``prepare-source`` call runs the preparation steps again. +The separate ``step prepare-source`` invocation accepts ``--source-kind`` and +``--source-path`` because it cannot retrieve the earlier Python download result. +``step download-source-archive --output-format json`` exposes that result's +actual path and kind without changing its default path-only output. +Without ``--source-kind``, it keeps the legacy archive behavior. +Integrating prebuilt detection into the bootstrapper remains a separate step. See :ref:`download_source_hook` for the return contract and compatibility rules. .. versionchanged:: 0.102.0 - Configured source resolvers also select the sdist build method. Git checkout - resolvers use PEP 517 by default; hook sdist resolvers require an explicit - ``build_sdist: pep517`` or ``build_sdist: tarball``. PyPI sdist resolvers + Explicit ``DownloadKind.git_checkout`` results can be prepared without + unpacking an archive. Configured source resolvers also select the sdist + build method. Git checkout resolvers use PEP 517 by default; hook sdist + resolvers require an explicit ``build_sdist: pep517`` or + ``build_sdist: tarball``. PyPI sdist resolvers repack a tarball. Both configured strategies take precedence over an installed ``build_sdist`` override and log a warning when it is ignored. Legacy settings continue to use the override. diff --git a/docs/reference/hooks.rst b/docs/reference/hooks.rst index c92149684..5479a9937 100644 --- a/docs/reference/hooks.rst +++ b/docs/reference/hooks.rst @@ -298,17 +298,32 @@ Source hooks the configured hook profiles; packages using legacy settings still require a bare ``Path``. -.. autofromagerhook:: default_prepare_source - - The ``prepare_source()`` function is responsible for setting up a tree - of source files in a format that is ready to be built. The default - implementation unpacks the source archive and applies patches. +.. _prepare_source_hook: - The arguments are the ``WorkContext``, the ``Requirement`` being - evaluated, the ``Path`` to the source archive, and the version. +.. autofromagerhook:: default_prepare_source - The return value should be the ``Path`` to the root of the source tree, - ideally inside the ``ctx.work_dir`` directory. + The ``prepare_source()`` function sets up a source tree for building. It + selects ``default_prepare_source()`` for archives or + ``default_prepare_git_checkout()`` for a Git checkout according to the + downloaded ``DownloadKind``. Both defaults apply patches and other project + preparation. + + In-process callers pass the ``path`` and ``kind`` from ``DownloadedSource`` + to ``prepare_source()``. The Python function accepts a source ``Path`` and + optional ``download_kind``; path-only callers default to ``sdist``. + A configured ``source:`` resolver does not disable the package + ``prepare_source`` override. The override receives the ``WorkContext``, + ``Requirement``, source ``Path``, and version, but not ``download_kind``. + An archive-specific override can call ``sources.default_prepare_source()``; + a Git-specific override can call + ``sources.default_prepare_git_checkout()`` before its own changes. + + The override returns the source root ``Path`` or a ``(Path, bool)`` pair. + Git checkouts must use the double-nested package/version layout under + ``ctx.work_dir`` so the later bootstrap and step commands can find them. + + .. versionchanged:: 0.102.0 + Passing ``DownloadKind.git_checkout`` prepares a clean Git checkout. .. autofromagerhook:: default_build_sdist @@ -328,6 +343,8 @@ Source helper functions The following helper functions are available in the ``fromager.sources`` module for use in custom source hooks: +.. autofunction:: fromager.sources.default_prepare_git_checkout + .. autofunction:: fromager.sources.ensure_pkg_info .. autofunction:: fromager.sources.pep517_build_sdist diff --git a/docs/using.md b/docs/using.md index 7439ed529..7ffab10e8 100644 --- a/docs/using.md +++ b/docs/using.md @@ -256,9 +256,19 @@ package index and downloads it. It will be common to run this step with `pypi.org`, but for truly isolated and reproducible builds a private index server is more robust. -The `step prepare-source` command unpacks the source archive -downloaded from the previous step and applies any patches (refer to -[customization](customization.md) for details about patching). +The `step prepare-source` command unpacks a source archive and applies any +patches (refer to [customization](customization.md) for details about patching). +It defaults to the archive kind for existing scripts. For a Git checkout, pass +`--source-kind git_checkout`. Pass `--source-path` when the downloaded source +cannot be found at its standard location. Git checkouts must use the standard +double-nested work directory layout so later build steps can find them. The +download command's optional +`--output-format json` prints both `path` and `kind` when a hook can return +different artifact kinds. Its default output remains the path alone. Git +preparation applies patches to the selected checkout itself, so use a +disposable clone for `--source-path`. Git downloads do not reuse an existing +checkout with `--no-cleanup`; the files remain available for inspection. Enable +cleanup or use a new work directory to download it again. The `step prepare-build` command creates a virtualenv with the build dependencies for building the wheel. It expects a `--wheel-server-url` diff --git a/src/fromager/bootstrapper/_bootstrapper.py b/src/fromager/bootstrapper/_bootstrapper.py index a22464b56..c524324c7 100644 --- a/src/fromager/bootstrapper/_bootstrapper.py +++ b/src/fromager/bootstrapper/_bootstrapper.py @@ -482,7 +482,7 @@ def _download_source( req: Requirement, resolved_version: Version, source_url: str, - ) -> pathlib.Path: + ) -> packagesettings.DownloadedSource: """Download source for a package.""" result: packagesettings.DownloadedSource = sources.download_source( ctx=self.ctx, @@ -490,20 +490,21 @@ def _download_source( version=resolved_version, download_url=source_url, ) - return result.path + return result def _prepare_source( self, req: Requirement, resolved_version: Version, - source_filename: pathlib.Path, + downloaded_source: packagesettings.DownloadedSource, ) -> pathlib.Path: """Prepare (unpack/patch) source for building.""" result: pathlib.Path = sources.prepare_source( ctx=self.ctx, req=req, - source_filename=source_filename, + source_filename=downloaded_source.path, version=resolved_version, + download_kind=downloaded_source.kind, ) return result diff --git a/src/fromager/bootstrapper/_prepare_source.py b/src/fromager/bootstrapper/_prepare_source.py index 67df15469..76f5212de 100644 --- a/src/fromager/bootstrapper/_prepare_source.py +++ b/src/fromager/bootstrapper/_prepare_source.py @@ -29,7 +29,7 @@ def _bg_prepare_source( resolved_version: Version, source_url: str, ) -> PreparedSourceData: - """Background-safe source download+unpack: no Bootstrapper state accessed.""" + """Background-safe source download and preparation without Bootstrapper state.""" # Thread-safe: _seen_requirements in the main thread prevents the same # package from being submitted to the thread pool more than once. # Paths from download_source() and prepare_source() include {name}-{version}, @@ -43,11 +43,15 @@ def _bg_prepare_source( sdist_root_dir=unpacked / unpacked.stem, cached_wheel_filename=cached_wheel, ) - source_filename = sources.download_source( + downloaded_source = sources.download_source( ctx=ctx, req=req, version=resolved_version, download_url=source_url - ).path + ) sdist_root_dir = sources.prepare_source( - ctx=ctx, req=req, source_filename=source_filename, version=resolved_version + ctx=ctx, + req=req, + source_filename=downloaded_source.path, + version=resolved_version, + download_kind=downloaded_source.kind, ) return PreparedSourceData(sdist_root_dir=sdist_root_dir) diff --git a/src/fromager/commands/build.py b/src/fromager/commands/build.py index 1fe34257c..70d549484 100644 --- a/src/fromager/commands/build.py +++ b/src/fromager/commands/build.py @@ -384,25 +384,26 @@ def _build( # If we get here and still don't have a wheel filename, then we need to # build the wheel. if not wheel_filename: - source_filename = sources.download_source( + downloaded_source = sources.download_source( ctx=wkctx, req=req, version=resolved_version, download_url=source_download_url, - ).path + ) logger.debug( - "saved sdist of version %s from %s to %s", + "saved source of version %s from %s to %s", resolved_version, source_download_url, - source_filename, + downloaded_source.path, ) # Prepare source source_root_dir = sources.prepare_source( ctx=wkctx, req=req, - source_filename=source_filename, + source_filename=downloaded_source.path, version=resolved_version, + download_kind=downloaded_source.kind, ) # Build environment diff --git a/src/fromager/commands/step.py b/src/fromager/commands/step.py index 95280934e..f9ea3ef04 100644 --- a/src/fromager/commands/step.py +++ b/src/fromager/commands/step.py @@ -1,3 +1,4 @@ +import json import logging import pathlib @@ -10,6 +11,7 @@ clickext, context, finders, + packagesettings, server, sources, wheels, @@ -26,17 +28,25 @@ def step() -> None: @step.command() +@click.option( + "--output-format", + type=click.Choice(["path", "json"]), + default="path", + show_default=True, + help="Print the path or a JSON object with path and artifact kind.", +) @click.argument("dist_name") @click.argument("dist_version", type=clickext.PackageVersion()) @click.argument("sdist_server_url") @click.pass_obj def download_source_archive( wkctx: context.WorkContext, + output_format: str, dist_name: str, dist_version: Version, sdist_server_url: str, ) -> None: - """download the source code archive for one version of one package + """download source for one version of one package DIST_NAME is the name of a distribution @@ -50,22 +60,52 @@ def download_source_archive( source_url, version = sources.resolve_source( ctx=wkctx, req=req, sdist_server_url=sdist_server_url ) - filename = sources.download_source( + downloaded_source = sources.download_source( ctx=wkctx, req=req, version=version, download_url=source_url, - ).path + ) requirement_ctxvar.reset(token) - print(filename) + if output_format == "json": + print( + json.dumps( + { + "path": str(downloaded_source.path), + "kind": downloaded_source.kind.value, + } + ) + ) + else: + print(downloaded_source.path) @step.command() +@click.option( + "--source-kind", + type=click.Choice( + [ + packagesettings.DownloadKind.sdist.value, + packagesettings.DownloadKind.tarball.value, + packagesettings.DownloadKind.git_checkout.value, + ] + ), + default=packagesettings.DownloadKind.sdist.value, + show_default=True, + help="Kind returned by the source download step.", +) +@click.option( + "--source-path", + type=click.Path(exists=True, path_type=pathlib.Path), + help="Downloaded source path; Git checkouts must use the standard work directory layout.", +) @click.argument("dist_name") @click.argument("dist_version", type=clickext.PackageVersion()) @click.pass_obj def prepare_source( wkctx: context.WorkContext, + source_kind: str, + source_path: pathlib.Path | None, dist_name: str, dist_version: Version, ) -> None: @@ -78,22 +118,35 @@ def prepare_source( """ req = Requirement(f"{dist_name}=={dist_version}") token = requirement_ctxvar.set(req) + download_kind = packagesettings.DownloadKind(source_kind) sdists_downloads = pathlib.Path(wkctx.sdists_repo) / "downloads" - source_filename = finders.find_sdist( - wkctx, wkctx.sdists_downloads, req, str(dist_version) + checkout = ( + wkctx.work_dir / f"{req.name}-{dist_version}" / f"{req.name}-{dist_version}" ) + source_filename: pathlib.Path | None + if source_path is not None: + source_filename = source_path + elif download_kind is packagesettings.DownloadKind.git_checkout: + source_filename = checkout if checkout.is_dir() else None + else: + source_filename = finders.find_sdist( + wkctx, wkctx.sdists_downloads, req, str(dist_version) + ) if source_filename is None: + if download_kind is packagesettings.DownloadKind.git_checkout: + raise RuntimeError(f"Cannot find Git checkout for {req} at {checkout}") dir_contents: list[str] = [] for ext in ["*.tar.gz", "*.zip"]: dir_contents.extend(str(e) for e in wkctx.sdists_downloads.glob(ext)) raise RuntimeError( - f"Cannot find sdist for {req.name} version {dist_version} in {sdists_downloads} among {dir_contents}" + f"Cannot find source for {req.name} version {dist_version} in {sdists_downloads} among {dir_contents}" ) source_root_dir = sources.prepare_source( ctx=wkctx, req=req, source_filename=source_filename, version=dist_version, + download_kind=download_kind, ) requirement_ctxvar.reset(token) print(source_root_dir) diff --git a/src/fromager/packagesettings/_resolver.py b/src/fromager/packagesettings/_resolver.py index ac57756ac..363f536f8 100644 --- a/src/fromager/packagesettings/_resolver.py +++ b/src/fromager/packagesettings/_resolver.py @@ -7,6 +7,7 @@ import logging import pathlib import re +import shutil import typing import pydantic @@ -156,6 +157,14 @@ def _download( / f"{req.name}-{candidate.version}" / f"{req.name}-{candidate.version}" ) + if destination_dir.parent.exists(): + if ctx.cleanup: + shutil.rmtree(destination_dir.parent) + elif destination_dir.exists(): + raise FileExistsError( + f"Git checkout {destination_dir} already exists; " + "enable cleanup or use a new work directory to download it again" + ) path = downloads.download_git_source( destination_dir=destination_dir, vcs_url=candidate.url, diff --git a/src/fromager/sources.py b/src/fromager/sources.py index efd2f20ea..ec36a4782 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -5,6 +5,7 @@ import logging import pathlib import shutil +import subprocess import tarfile import typing import warnings @@ -293,6 +294,9 @@ def unpack_source( source_filename: pathlib.Path, ) -> tuple[pathlib.Path, bool]: """Extracts a downloaded source archive (.tar.gz or .zip file) into a standardized directory.""" + if source_filename.is_dir(): + raise ValueError(f"{source_filename} is a directory, not a source archive") + # sdist names are less standardized and the names of the directories they # contain are also not very standard. Force the names into a predictable # form based on the override module name for the requirement. @@ -304,6 +308,17 @@ def unpack_source( unpack_dir = ctx.work_dir / expected_name if unpack_dir.exists(): if ctx.cleanup: + resolved_unpack_dir = unpack_dir.resolve() + # Deleting a parent can also break a path that uses ".." or a symlink. + source_paths = (source_filename, *source_filename.parents) + if any( + path.resolve().is_relative_to(resolved_unpack_dir) + for path in source_paths + ): + raise ValueError( + f"source archive path {source_filename} overlaps {unpack_dir}; " + "use an archive path outside the directory being cleaned" + ) logger.debug("cleaning up %s", unpack_dir) shutil.rmtree(unpack_dir) else: @@ -412,18 +427,43 @@ def prepare_source( req: Requirement, source_filename: pathlib.Path, version: Version, + download_kind: packagesettings.DownloadKind = packagesettings.DownloadKind.sdist, ) -> pathlib.Path: - """Unpack and prepare source for building. - - Calls ``find_and_invoke`` which may call a package-specific override. - The plugin may return a ``Path`` or a ``(Path, bool)`` tuple; both - forms are handled. + """Prepare an archive or checkout according to its downloaded kind. + + Path-only callers default to an sdist. Package-specific ``prepare_source`` + overrides may replace the selected default, including when a source + resolver is configured. Overrides receive the original path-based + arguments and may return a ``Path`` or a ``(Path, bool)`` pair. + Git-specific overrides can call ``default_prepare_git_checkout()``. + + Args: + ctx: Work context for the build. + req: Package requirement being prepared. + source_filename: Downloaded archive or checkout path. + version: Resolved package version. + download_kind: Artifact kind; defaults to an sdist. + + Returns: + Path to the prepared source root. + + .. versionchanged:: 0.102.0 + Pass ``download_kind`` to prepare a Git checkout. """ logger.info(f"preparing source for {req} from {source_filename}") + match download_kind: + case packagesettings.DownloadKind.sdist | packagesettings.DownloadKind.tarball: + default_fn = default_prepare_source + case packagesettings.DownloadKind.git_checkout: + default_fn = default_prepare_git_checkout + case _: + raise ValueError( + f"cannot prepare source with download kind {download_kind}" + ) prepare_source_details = overrides.find_and_invoke( req.name, "prepare_source", - default_prepare_source, + default_fn, ctx=ctx, req=req, source_filename=source_filename, @@ -450,7 +490,7 @@ def default_prepare_source( source_filename: pathlib.Path, version: Version, ) -> tuple[pathlib.Path, bool]: - """Unpack, modify, and check sdist sources + """Unpack an archive, then prepare newly extracted source. Calls :func:`~fromager.sources.prepare_new_source` by default. """ @@ -460,7 +500,79 @@ def default_prepare_source( version=version, source_filename=source_filename, ) - if is_new: + return _prepare_source_root( + ctx, req, source_root_dir, version, should_prepare=is_new + ) + + +def default_prepare_git_checkout( + ctx: context.WorkContext, + req: Requirement, + source_filename: pathlib.Path, + version: Version, +) -> tuple[pathlib.Path, bool]: + """Prepare a clean Git checkout in place. + + Reject tracked, untracked, or ignored local changes in the checkout and its + initialized submodules before applying patches, project overrides, and Rust + vendoring. A clean status does not prove where the checkout came from or + that it will remain unchanged after preparation. + + Args: + ctx: Work context for the build. + req: Package requirement being prepared. + source_filename: Path to the Git checkout. + version: Resolved package version. + + Returns: + The checkout path and ``True`` because preparation ran. + + .. versionadded:: 0.102.0 + """ + if not source_filename.is_dir() or not (source_filename / ".git").exists(): + raise ValueError(f"{source_filename} is not a Git checkout") + status_command = ( + "git", + "status", + "--porcelain=v1", + "--untracked-files=all", + "--ignored=matching", + "--ignore-submodules=none", + ) + try: + changes = external_commands.run(status_command, cwd=str(source_filename)) + if not changes: + # A parent repository can hide changes inside ignored submodules. + changes = external_commands.run( + [ + "git", + "submodule", + "--quiet", + "foreach", + "--recursive", + " ".join(status_command), + ], + cwd=str(source_filename), + ) + except subprocess.CalledProcessError as err: + raise ValueError(f"cannot inspect Git checkout {source_filename}") from err + if changes: + raise ValueError( + f"Git checkout {source_filename} has local changes; " + "download a fresh checkout before preparing" + ) + return _prepare_source_root(ctx, req, source_filename, version, should_prepare=True) + + +def _prepare_source_root( + ctx: context.WorkContext, + req: Requirement, + source_root_dir: pathlib.Path, + version: Version, + should_prepare: bool, +) -> tuple[pathlib.Path, bool]: + """Run requested preparation and scan the resulting source tree.""" + if should_prepare: prepare_new_source( ctx=ctx, req=req, @@ -476,7 +588,7 @@ def default_prepare_source( ", ".join(str(f) for f in sorted(potential_issues)), ) - return source_root_dir, is_new + return source_root_dir, should_prepare def prepare_new_source( @@ -491,7 +603,8 @@ def prepare_new_source( - apply project overrides from settings - vendor Rust dependencies - :func:`~default_prepare_source` runs this function when the sources are new. + :func:`~default_prepare_source` and + :func:`~default_prepare_git_checkout` run this function for new sources. """ patch_source(ctx, source_root_dir, req, version) pyproject.apply_project_override( diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index 00b8a0044..0d775339c 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -1,6 +1,7 @@ import json import logging import pathlib +import subprocess import typing from unittest.mock import Mock, patch @@ -1200,6 +1201,7 @@ def test_bg_prepare_source_log_prefix_includes_version( req=req, source_filename=source_filename, version=version, + download_kind=DownloadKind.sdist, ) finally: logging.setLogRecordFactory(old_factory) @@ -1210,10 +1212,42 @@ def test_bg_prepare_source_log_prefix_includes_version( ) -def test_bootstrapper_download_source_returns_downloaded_path( +def test_bg_prepare_source_accepts_git_checkout(tmp_context: WorkContext) -> None: + req = Requirement("mypkg==1.2.3") + version = Version("1.2.3") + checkout = tmp_context.work_dir / "mypkg-1.2.3" / "mypkg-1.2.3" + checkout.mkdir(parents=True) + subprocess.run(["git", "init", str(checkout)], check=True, capture_output=True) + + with ( + patch( + "fromager.bootstrapper._cache.find_cached_wheel", return_value=(None, None) + ), + patch( + "fromager.sources.download_source", + return_value=DownloadedSource(checkout, DownloadKind.git_checkout), + ), + patch("fromager.sources.prepare_new_source") as prepare_new_source, + ): + result = _bg_prepare_source( + ctx=tmp_context, + cache_wheel_server_url=None, + req=req, + resolved_version=version, + source_url="git+https://code.test/org/mypkg.git@v1.2.3", + ) + + assert result.sdist_root_dir == checkout + prepare_new_source.assert_called_once_with( + ctx=tmp_context, req=req, source_root_dir=checkout, version=version + ) + assert (checkout.parent / "build-meta.json").is_file() + + +def test_bootstrapper_download_source_preserves_download_kind( tmp_context: WorkContext, ) -> None: - """Return only the path from the resolver download result.""" + """Keep the artifact kind for the legacy Bootstrapper helper.""" bt = bootstrapper.Bootstrapper(tmp_context) req = Requirement("mypkg==1.2.3") version = Version("1.2.3") @@ -1226,7 +1260,7 @@ def test_bootstrapper_download_source_returns_downloaded_path( ) as download_source: result = bt._download_source(req, version, source_url) - assert result == source_filename + assert result == DownloadedSource(source_filename, DownloadKind.sdist) download_source.assert_called_once_with( ctx=tmp_context, req=req, @@ -1235,6 +1269,33 @@ def test_bootstrapper_download_source_returns_downloaded_path( ) +def test_bootstrapper_prepare_source_passes_download_kind( + tmp_context: WorkContext, +) -> None: + bt = bootstrapper.Bootstrapper(tmp_context) + req = Requirement("mypkg==1.2.3") + version = Version("1.2.3") + downloaded = DownloadedSource( + tmp_context.work_dir / "mypkg-1.2.3" / "mypkg-1.2.3", + DownloadKind.git_checkout, + ) + + with patch( + "fromager.bootstrapper._bootstrapper.sources.prepare_source", + return_value=downloaded.path, + ) as prepare_source: + result = bt._prepare_source(req, version, downloaded) + + assert result == downloaded.path + prepare_source.assert_called_once_with( + ctx=tmp_context, + req=req, + source_filename=downloaded.path, + version=version, + download_kind=DownloadKind.git_checkout, + ) + + def test_bg_prepare_prebuilt_log_prefix_includes_version( tmp_context: WorkContext, caplog: pytest.LogCaptureFixture, diff --git a/tests/test_commands.py b/tests/test_commands.py index 17572993b..6b4ae406c 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,4 +1,6 @@ +import json import pathlib +import subprocess import typing from unittest.mock import Mock, patch @@ -34,7 +36,7 @@ def test_download_source_archive_prints_downloaded_path( tmp_context: WorkContext, tmp_path: pathlib.Path, ) -> None: - """Print only the path when the download helper returns a tuple.""" + """Keep the path-only output for existing scripts.""" expected = tmp_path / "pkg-1.0.tar.gz" with ( @@ -66,11 +68,181 @@ def test_download_source_archive_prints_downloaded_path( assert result.output == f"{expected}\n" -def test_build_passes_downloaded_path_to_prepare_source( +def test_download_source_archive_json_includes_kind( + cli_runner: CliRunner, + tmp_context: WorkContext, + tmp_path: pathlib.Path, +) -> None: + checkout = tmp_path / "pkg-1.0" + + with ( + patch.object( + step.sources, + "resolve_source", + return_value=("git+https://code.test/pkg.git@v1.0", Version("1.0")), + ), + patch.object( + step.sources, + "download_source", + return_value=packagesettings.DownloadedSource( + checkout, packagesettings.DownloadKind.git_checkout + ), + ), + ): + result = cli_runner.invoke( + step.step, + [ + "download-source-archive", + "--output-format", + "json", + "pkg", + "1.0", + "https://pkg.test/simple", + ], + obj=tmp_context, + ) + + assert result.exit_code == 0, result.output + assert json.loads(result.output) == { + "path": str(checkout), + "kind": "git_checkout", + } + + +def test_step_prepare_source_prepares_clean_git_checkout( + cli_runner: CliRunner, tmp_context: WorkContext +) -> None: + checkout = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" + checkout.mkdir(parents=True) + subprocess.run(["git", "init", str(checkout)], check=True, capture_output=True) + + with patch.object(step.sources, "prepare_new_source") as prepare_new_source: + result = cli_runner.invoke( + step.step, + ["prepare-source", "--source-kind", "git_checkout", "pkg", "1.0"], + obj=tmp_context, + ) + + assert result.exit_code == 0, result.output + assert result.output == f"{checkout}\n" + prepare_new_source.assert_called_once_with( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + source_root_dir=checkout, + version=Version("1.0"), + ) + assert (checkout.parent / "build-meta.json").is_file() + + +def test_step_prepare_source_kind_selects_checkout_over_archive( + cli_runner: CliRunner, tmp_context: WorkContext +) -> None: + archive = tmp_context.sdists_downloads / "pkg-1.0.tar.gz" + archive.write_bytes(b"old archive") + checkout = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" + checkout.mkdir(parents=True) + with patch.object( + step.sources, "prepare_source", return_value=checkout + ) as prepare_source: + result = cli_runner.invoke( + step.step, + ["prepare-source", "--source-kind", "git_checkout", "pkg", "1.0"], + obj=tmp_context, + ) + + assert result.exit_code == 0 + assert result.output == f"{checkout}\n" + prepare_source.assert_called_once_with( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + source_filename=checkout, + version=Version("1.0"), + download_kind=packagesettings.DownloadKind.git_checkout, + ) + + +def test_step_prepare_source_defaults_to_archive_kind( + cli_runner: CliRunner, tmp_context: WorkContext +) -> None: + archive = tmp_context.sdists_downloads / "pkg-1.0.tar.gz" + archive.write_bytes(b"archive") + checkout = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" + checkout.mkdir(parents=True) + + with patch.object( + step.sources, "prepare_source", return_value=checkout + ) as prepare_source: + result = cli_runner.invoke( + step.step, ["prepare-source", "pkg", "1.0"], obj=tmp_context + ) + + assert result.exit_code == 0, result.output + prepare_source.assert_called_once_with( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + source_filename=archive, + version=Version("1.0"), + download_kind=packagesettings.DownloadKind.sdist, + ) + + +def test_step_prepare_source_accepts_explicit_hook_path( + cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path +) -> None: + checkout = tmp_path / "hook-checkout" + checkout.mkdir() + + with patch.object( + step.sources, "prepare_source", return_value=checkout + ) as prepare_source: + result = cli_runner.invoke( + step.step, + [ + "prepare-source", + "--source-kind", + "git_checkout", + "--source-path", + str(checkout), + "pkg", + "1.0", + ], + obj=tmp_context, + ) + + assert result.exit_code == 0, result.output + prepare_source.assert_called_once_with( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + source_filename=checkout, + version=Version("1.0"), + download_kind=packagesettings.DownloadKind.git_checkout, + ) + + +def test_step_prepare_source_keeps_archive_inside_cleanup_dir( + cli_runner: CliRunner, tmp_context: WorkContext +) -> None: + """Reject an archive that cleanup would delete before opening it.""" + archive = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0.tar.gz" + archive.parent.mkdir(parents=True) + archive.write_bytes(b"original archive") + + result = cli_runner.invoke( + step.step, + ["prepare-source", "--source-path", str(archive), "pkg", "1.0"], + obj=tmp_context, + ) + + assert isinstance(result.exception, ValueError) + assert "overlaps" in str(result.exception) + assert archive.read_bytes() == b"original archive" + + +def test_build_passes_downloaded_kind_to_prepare_source( tmp_context: WorkContext, tmp_path: pathlib.Path, ) -> None: - """Pass only the downloaded path into source preparation.""" + """Pass the downloaded path and kind into source preparation.""" req = Requirement("pkg==1.0") version = Version("1.0") source_url = "https://pkg.test/pkg-1.0.tar.gz" @@ -119,5 +291,6 @@ def test_build_passes_downloaded_path_to_prepare_source( req=req, source_filename=source_filename, version=version, + download_kind=packagesettings.DownloadKind.sdist, ) assert result.wheel_filename == tmp_context.wheels_downloads / wheel_filename.name diff --git a/tests/test_packagesettings_resolver.py b/tests/test_packagesettings_resolver.py index 8a942d12c..749285231 100644 --- a/tests/test_packagesettings_resolver.py +++ b/tests/test_packagesettings_resolver.py @@ -4,6 +4,7 @@ import datetime import pathlib import re +import subprocess import textwrap import typing from unittest import mock @@ -15,7 +16,7 @@ from packaging.version import Version from wheel.wheelfile import WheelFile -from fromager import resolver +from fromager import resolver, sources from fromager.candidate import Candidate, Cooldown from fromager.context import WorkContext from fromager.packagesettings._resolver import ( @@ -407,6 +408,131 @@ def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> No vcs_url=_CANDIDATE_GIT.url, ) + def test_download_rejects_existing_checkout_without_cleanup( + self, tmp_context: WorkContext + ) -> None: + tmp_context.cleanup = False + checkout = ( + tmp_context.work_dir / f"{_REQ.name}-{_VERSION}" / f"{_REQ.name}-{_VERSION}" + ) + checkout.mkdir(parents=True) + (checkout / ".git").mkdir() + (checkout / "keep.txt").write_text("prepared") + + with pytest.raises(FileExistsError, match="enable cleanup"): + _parse(self.YAML).download(tmp_context, _REQ, _CANDIDATE_GIT) + + assert (checkout / "keep.txt").read_text() == "prepared" + + def test_download_cleans_checkout_before_reclone( + self, tmp_context: WorkContext + ) -> None: + checkout = ( + tmp_context.work_dir / f"{_REQ.name}-{_VERSION}" / f"{_REQ.name}-{_VERSION}" + ) + checkout.mkdir(parents=True) + (checkout / "old.txt").write_text("old") + (checkout.parent / "build-meta.json").write_text("old") + + def clone(*, destination_dir: pathlib.Path, vcs_url: str) -> pathlib.Path: + assert vcs_url == _CANDIDATE_GIT.url + destination_dir.mkdir(parents=True) + (destination_dir / "new.txt").write_text("new") + return destination_dir + + with mock.patch("fromager.downloads.download_git_source", side_effect=clone): + result = _parse(self.YAML).download(tmp_context, _REQ, _CANDIDATE_GIT) + + assert result == DownloadedSource(checkout, DownloadKind.git_checkout) + assert (checkout / "new.txt").read_text() == "new" + assert not (checkout / "old.txt").exists() + assert not (checkout.parent / "build-meta.json").exists() + + def test_download_preserves_non_checkout_on_collision( + self, tmp_context: WorkContext + ) -> None: + tmp_context.cleanup = False + checkout = ( + tmp_context.work_dir / f"{_REQ.name}-{_VERSION}" / f"{_REQ.name}-{_VERSION}" + ) + checkout.mkdir(parents=True) + (checkout / "keep.txt").write_text("existing source") + meta_file = checkout.parent / "build-meta.json" + meta_file.write_text("existing metadata") + + with pytest.raises(FileExistsError, match="already exists"): + _parse(self.YAML).download(tmp_context, _REQ, _CANDIDATE_GIT) + + assert (checkout / "keep.txt").read_text() == "existing source" + assert meta_file.read_text() == "existing metadata" + + def test_local_git_checkout_prepares_and_reclones( + self, tmp_context: WorkContext, tmp_path: pathlib.Path + ) -> None: + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", str(repo)], check=True, capture_output=True) + (repo / "pyproject.toml").write_text("[build-system]\nrequires = []\n") + subprocess.run(["git", "-C", str(repo), "add", "."], check=True) + subprocess.run( + [ + "git", + "-C", + str(repo), + "-c", + "user.name=Test", + "-c", + "user.email=test@pkg.test", + "commit", + "-m", + "Initial source", + ], + check=True, + capture_output=True, + ) + subprocess.run(["git", "-C", str(repo), "tag", "v1.2.3"], check=True) + candidate = Candidate( + name=_REQ.name, + version=_VERSION, + url=f"git+file://{repo}@refs/tags/v1.2.3", + ) + source_resolver = _parse(self.YAML) + tmp_context.cleanup = False + package_dir = tmp_context.work_dir / f"{_REQ.name}-{_VERSION}" + (package_dir / "other-dir").mkdir(parents=True) + (package_dir / "build-meta.json").write_text("stale") + + downloaded = source_resolver.download(tmp_context, _REQ, candidate) + assert (package_dir / "other-dir").is_dir() + assert (package_dir / "build-meta.json").read_text() == "stale" + prepared = sources.prepare_source( + ctx=tmp_context, + req=_REQ, + source_filename=downloaded.path, + version=_VERSION, + download_kind=downloaded.kind, + ) + (prepared / "keep.txt").write_text("prepared") + with pytest.raises(FileExistsError, match="already exists"): + source_resolver.download(tmp_context, _REQ, candidate) + + assert downloaded.kind is DownloadKind.git_checkout + assert prepared == downloaded.path + assert (prepared / ".git").is_dir() + assert (prepared / "keep.txt").read_text() == "prepared" + assert sources.read_build_meta(prepared.parent)["source-filename"] == str( + prepared + ) + + tmp_context.cleanup = True + recloned = source_resolver.download(tmp_context, _REQ, candidate) + + assert recloned.path == prepared + assert (recloned.path / ".git").is_dir() + assert not (recloned.path / "keep.txt").exists() + assert not (package_dir / "other-dir").exists() + assert not (recloned.path.parent / "build-meta.json").exists() + # -- Git source resolvers ---------------------------------------------------- diff --git a/tests/test_sources.py b/tests/test_sources.py index 2816db5b3..1059c8156 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -1,5 +1,6 @@ import datetime import pathlib +import subprocess import sys import tarfile import zipfile @@ -788,6 +789,99 @@ def test_unpack_source_unknown_extension( ) +@pytest.mark.parametrize("cleanup", [True, False]) +def test_unpack_source_rejects_directory_before_cleanup( + tmp_context: context.WorkContext, + cleanup: bool, +) -> None: + req = Requirement("pkg==1.0") + version = Version("1.0") + tmp_context.cleanup = cleanup + checkout = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" + checkout.mkdir(parents=True) + (checkout / "keep.txt").write_text("original") + + with pytest.raises(ValueError, match="is a directory, not a source archive"): + sources.unpack_source( + ctx=tmp_context, req=req, version=version, source_filename=checkout + ) + + assert (checkout / "keep.txt").read_text() == "original" + + +def test_unpack_source_rejects_archive_path_through_cleanup_dir( + tmp_context: context.WorkContext, +) -> None: + """A path using the soon-to-be-deleted directory stops working after cleanup.""" + unpack_dir = tmp_context.work_dir / "pkg-1.0" + unpack_dir.mkdir() + archive = tmp_context.work_dir / "pkg-1.0.tar.gz" + archive.write_bytes(b"original archive") + source_path = unpack_dir / ".." / archive.name + + with pytest.raises(ValueError, match="overlaps"): + sources.unpack_source( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + version=Version("1.0"), + source_filename=source_path, + ) + + assert unpack_dir.is_dir() + assert archive.read_bytes() == b"original archive" + + +def test_unpack_source_rejects_symlink_into_cleanup_dir( + tmp_context: context.WorkContext, +) -> None: + """Reject an external symlink whose archive target would be deleted.""" + unpack_dir = tmp_context.work_dir / "pkg-1.0" + unpack_dir.mkdir() + archive = unpack_dir / "pkg-1.0.tar.gz" + archive.write_bytes(b"original archive") + source_path = tmp_context.work_dir / "pkg-1.0.tar.gz" + source_path.symlink_to(archive) + + with pytest.raises(ValueError, match="overlaps"): + sources.unpack_source( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + version=Version("1.0"), + source_filename=source_path, + ) + + assert archive.read_bytes() == b"original archive" + + +def test_unpack_source_replaces_existing_tree_from_external_archive( + tmp_context: context.WorkContext, + tmp_path: pathlib.Path, +) -> None: + """Keep the normal cleanup path for an archive outside the work tree.""" + unpack_dir = tmp_context.work_dir / "pkg-1.0" + unpack_dir.mkdir() + (unpack_dir / "stale.txt").write_text("old output") + content_dir = tmp_path / "content" / "pkg-1.0" + content_dir.mkdir(parents=True) + (content_dir / "setup.py").write_text("# setup") + archive = tmp_path / "pkg-1.0.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + tar.add(content_dir, arcname=content_dir.name) + + source_root, is_new = sources.unpack_source( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + version=Version("1.0"), + source_filename=archive, + ) + + assert is_new is True + assert source_root == unpack_dir / "pkg-1.0" + assert (source_root / "setup.py").read_text() == "# setup" + assert not (unpack_dir / "stale.txt").exists() + assert archive.is_file() + + def test_unpack_source_renames_mismatched_dir( tmp_context: context.WorkContext, tmp_path: pathlib.Path, @@ -1021,6 +1115,509 @@ def test_prepare_source_plugin_returns_path( assert result == source_dir +def test_prepare_source_keeps_override_with_configured_resolver( + tmp_context: context.WorkContext, +) -> None: + req = Requirement("pkg==1.0") + version = Version("1.0") + checkout = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" + checkout.mkdir(parents=True) + downloaded = packagesettings.DownloadedSource( + path=checkout, kind=packagesettings.DownloadKind.git_checkout + ) + source_resolver = Mock() + source_resolver.download.return_value = downloaded + + def prepare_override( + ctx: context.WorkContext, + req: Requirement, + source_filename: pathlib.Path, + version: Version, + ) -> pathlib.Path: + assert ctx is tmp_context + assert req == Requirement("pkg==1.0") + assert source_filename == checkout + assert version == Version("1.0") + return checkout + + with ( + patch.object( + packagesettings.PackageBuildInfo, + "source_resolver", + new_callable=PropertyMock, + return_value=source_resolver, + ), + patch( + "fromager.overrides.find_override_method", return_value=prepare_override + ) as find_override_method, + ): + result = sources.download_source( + ctx=tmp_context, + req=req, + version=version, + download_url="git+https://code.test/org/pkg.git@v1.0", + ) + prepared = sources.prepare_source( + ctx=tmp_context, + req=req, + source_filename=result.path, + version=version, + download_kind=result.kind, + ) + + assert result is downloaded + assert prepared == checkout + source_resolver.download.assert_called_once_with( + tmp_context, + req, + Candidate( + name=req.name, + version=version, + url="git+https://code.test/org/pkg.git@v1.0", + ), + ) + find_override_method.assert_called_once_with(req.name, "prepare_source") + assert (checkout.parent / "build-meta.json").is_file() + + +def test_prepare_source_git_checkout_rejects_dirty_retry( + tmp_context: context.WorkContext, +) -> None: + req = Requirement("pkg==1.0") + version = Version("1.0") + checkout = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" + checkout.mkdir(parents=True) + subprocess.run(["git", "init", str(checkout)], check=True, capture_output=True) + + def add_prepared_file(**_: object) -> None: + (checkout / "prepared.txt").write_text("patched") + + with patch( + "fromager.sources.prepare_new_source", side_effect=add_prepared_file + ) as prepare_new_source: + result = sources.prepare_source( + ctx=tmp_context, + req=req, + source_filename=checkout, + version=version, + download_kind=packagesettings.DownloadKind.git_checkout, + ) + with pytest.raises(ValueError, match="has local changes"): + sources.prepare_source( + ctx=tmp_context, + req=req, + source_filename=checkout, + version=version, + download_kind=packagesettings.DownloadKind.git_checkout, + ) + + assert result == checkout + assert (checkout / "prepared.txt").read_text() == "patched" + prepare_new_source.assert_called_once_with( + ctx=tmp_context, req=req, source_root_dir=checkout, version=version + ) + assert sources.read_build_meta(checkout.parent) == { + "req": "pkg==1.0", + "source-filename": str(checkout), + "version": "1.0", + } + + +def test_prepare_source_git_checkout_rejects_ignored_files( + tmp_context: context.WorkContext, +) -> None: + checkout = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" + checkout.mkdir(parents=True) + subprocess.run(["git", "init", str(checkout)], check=True, capture_output=True) + (checkout / ".gitignore").write_text("build/\n") + subprocess.run(["git", "-C", str(checkout), "add", ".gitignore"], check=True) + subprocess.run( + [ + "git", + "-C", + str(checkout), + "-c", + "user.name=Test", + "-c", + "user.email=test@pkg.test", + "commit", + "-m", + "Ignore build outputs", + ], + check=True, + capture_output=True, + ) + (checkout / "build").mkdir() + (checkout / "build" / "generated.txt").write_text("generated") + + with patch("fromager.sources.prepare_new_source") as prepare_new_source: + with pytest.raises(ValueError, match="has local changes"): + sources.prepare_source( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + source_filename=checkout, + version=Version("1.0"), + download_kind=packagesettings.DownloadKind.git_checkout, + ) + + prepare_new_source.assert_not_called() + + +@pytest.mark.parametrize("change_kind", ["tracked", "ignored"]) +def test_prepare_source_git_checkout_rejects_nested_submodule_changes( + tmp_context: context.WorkContext, + tmp_path: pathlib.Path, + change_kind: str, +) -> None: + def run_git(repo: pathlib.Path, *args: str) -> str: + result = subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + ) + return result.stdout + + leaf = tmp_path / "leaf" + leaf.mkdir() + run_git(leaf, "init") + (leaf / "tracked.txt").write_text("original") + (leaf / ".gitignore").write_text("build/\n") + run_git(leaf, "add", ".") + run_git( + leaf, + "-c", + "user.name=Test", + "-c", + "user.email=test@pkg.test", + "commit", + "-m", + "Initial leaf", + ) + + middle = tmp_path / "middle" + middle.mkdir() + run_git(middle, "init") + run_git( + middle, + "-c", + "protocol.file.allow=always", + "submodule", + "add", + str(leaf), + "leaf", + ) + run_git(middle, "config", "-f", ".gitmodules", "submodule.leaf.ignore", "all") + run_git(middle, "add", ".") + run_git( + middle, + "-c", + "user.name=Test", + "-c", + "user.email=test@pkg.test", + "commit", + "-m", + "Initial middle", + ) + + checkout = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" + checkout.mkdir(parents=True) + run_git(checkout, "init") + run_git( + checkout, + "-c", + "protocol.file.allow=always", + "submodule", + "add", + str(middle), + "middle", + ) + run_git(checkout, "config", "-f", ".gitmodules", "submodule.middle.ignore", "all") + run_git(checkout, "add", ".") + run_git( + checkout, + "-c", + "user.name=Test", + "-c", + "user.email=test@pkg.test", + "commit", + "-m", + "Initial checkout", + ) + run_git( + checkout, + "-c", + "protocol.file.allow=always", + "submodule", + "update", + "--init", + "--recursive", + ) + + nested_checkout = checkout / "middle" / "leaf" + if change_kind == "tracked": + (nested_checkout / "tracked.txt").write_text("changed") + else: + (nested_checkout / "build").mkdir() + (nested_checkout / "build" / "generated.txt").write_text("generated") + + assert ( + run_git( + checkout, + "status", + "--porcelain=v1", + "--untracked-files=all", + "--ignored=matching", + ) + == "" + ) + with patch("fromager.sources.prepare_new_source") as prepare_new_source: + with pytest.raises(ValueError, match="has local changes"): + sources.prepare_source( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + source_filename=checkout, + version=Version("1.0"), + download_kind=packagesettings.DownloadKind.git_checkout, + ) + + prepare_new_source.assert_not_called() + + +def test_prepare_source_checkout_ignores_stale_build_metadata( + tmp_context: context.WorkContext, +) -> None: + req = Requirement("pkg==1.0") + version = Version("1.0") + checkout = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" + checkout.mkdir(parents=True) + subprocess.run(["git", "init", str(checkout)], check=True, capture_output=True) + (checkout.parent / "build-meta.json").write_text("stale") + + with patch("fromager.sources.prepare_new_source") as prepare_new_source: + result = sources.prepare_source( + ctx=tmp_context, + req=req, + source_filename=checkout, + version=version, + download_kind=packagesettings.DownloadKind.git_checkout, + ) + + assert result == checkout + prepare_new_source.assert_called_once() + assert sources.read_build_meta(checkout.parent)["source-filename"] == str(checkout) + + +def test_prepare_source_git_checkout_runs_preparation_steps( + tmp_context: context.WorkContext, +) -> None: + req = Requirement("pkg==1.0") + version = Version("1.0") + checkout = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" + checkout.mkdir(parents=True) + subprocess.run(["git", "init", str(checkout)], check=True, capture_output=True) + + with ( + patch("fromager.sources.patch_source") as patch_source, + patch( + "fromager.sources.pyproject.apply_project_override" + ) as apply_project_override, + patch("fromager.sources.vendor_rust.vendor_rust") as vendor_rust, + ): + result = sources.prepare_source( + ctx=tmp_context, + req=req, + source_filename=checkout, + version=version, + download_kind=packagesettings.DownloadKind.git_checkout, + ) + + assert result == checkout + patch_source.assert_called_once_with(tmp_context, checkout, req, version) + apply_project_override.assert_called_once_with( + ctx=tmp_context, req=req, sdist_root_dir=checkout + ) + vendor_rust.assert_called_once_with( + req, tmp_context.package_build_info(req).build_dir(checkout) + ) + + +def test_prepare_source_rejects_non_git_directory( + tmp_context: context.WorkContext, +) -> None: + checkout = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" + checkout.mkdir(parents=True) + + with pytest.raises(ValueError, match="is not a Git checkout"): + sources.prepare_source( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + source_filename=checkout, + version=Version("1.0"), + download_kind=packagesettings.DownloadKind.git_checkout, + ) + + +def test_prepare_source_git_kind_rejects_archive_file( + tmp_context: context.WorkContext, +) -> None: + archive = tmp_context.sdists_downloads / "pkg-1.0.tar.gz" + archive.write_bytes(b"archive") + + with pytest.raises(ValueError, match="is not a Git checkout"): + sources.prepare_source( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + source_filename=archive, + version=Version("1.0"), + download_kind=packagesettings.DownloadKind.git_checkout, + ) + + assert archive.read_bytes() == b"archive" + + +def test_prepare_source_path_only_rejects_checkout_without_removing_it( + tmp_context: context.WorkContext, +) -> None: + checkout = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" + checkout.mkdir(parents=True) + (checkout / "keep.txt").write_text("original") + + with pytest.raises(ValueError, match="is a directory, not a source archive"): + sources.prepare_source( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + source_filename=checkout, + version=Version("1.0"), + ) + + assert (checkout / "keep.txt").read_text() == "original" + + +@pytest.mark.parametrize( + "download_kind", + [packagesettings.DownloadKind.sdist, packagesettings.DownloadKind.tarball], +) +def test_prepare_source_archive_still_unpacks_and_prepares( + tmp_context: context.WorkContext, + tmp_path: pathlib.Path, + download_kind: packagesettings.DownloadKind, +) -> None: + req = Requirement("pkg==1.0") + version = Version("1.0") + content = tmp_path / "content" / "pkg-1.0" + content.mkdir(parents=True) + (content / "pyproject.toml").write_text("[build-system]\n") + archive = tmp_path / "pkg-1.0.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + tar.add(content, arcname="pkg-1.0") + + with patch("fromager.sources.prepare_new_source") as prepare_new_source: + result = sources.prepare_source( + ctx=tmp_context, + req=req, + source_filename=archive, + version=version, + download_kind=download_kind, + ) + + expected = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" + assert result == expected + assert (result / "pyproject.toml").is_file() + prepare_new_source.assert_called_once_with( + ctx=tmp_context, req=req, source_root_dir=expected, version=version + ) + assert sources.read_build_meta(result.parent)["source-filename"] == str(archive) + + +def test_prepare_source_checkout_keeps_override_contract( + tmp_context: context.WorkContext, +) -> None: + req = Requirement("pkg==1.0") + version = Version("1.0") + checkout = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" + checkout.mkdir(parents=True) + + with patch( + "fromager.sources.overrides.find_and_invoke", return_value=(checkout, True) + ) as invoke: + result = sources.prepare_source( + ctx=tmp_context, + req=req, + source_filename=checkout, + version=version, + download_kind=packagesettings.DownloadKind.git_checkout, + ) + + assert result == checkout + invoke.assert_called_once_with( + req.name, + "prepare_source", + sources.default_prepare_git_checkout, + ctx=tmp_context, + req=req, + source_filename=checkout, + version=version, + ) + assert (checkout.parent / "build-meta.json").is_file() + + +def test_prepare_source_git_override_calls_public_default( + tmp_context: context.WorkContext, +) -> None: + req = Requirement("pkg==1.0") + version = Version("1.0") + checkout = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" + checkout.mkdir(parents=True) + subprocess.run(["git", "init", str(checkout)], check=True, capture_output=True) + + def prepare_override( + ctx: context.WorkContext, + req: Requirement, + source_filename: pathlib.Path, + version: Version, + ) -> tuple[pathlib.Path, bool]: + assert ctx is tmp_context + assert req == Requirement("pkg==1.0") + assert source_filename == checkout + assert version == Version("1.0") + return sources.default_prepare_git_checkout(ctx, req, source_filename, version) + + with ( + patch("fromager.overrides.find_override_method", return_value=prepare_override), + patch("fromager.sources.prepare_new_source") as prepare_new_source, + ): + result = sources.prepare_source( + ctx=tmp_context, + req=req, + source_filename=checkout, + version=version, + download_kind=packagesettings.DownloadKind.git_checkout, + ) + + assert result == checkout + prepare_new_source.assert_called_once_with( + ctx=tmp_context, req=req, source_root_dir=checkout, version=version + ) + assert (checkout.parent / "build-meta.json").is_file() + + +def test_prepare_source_rejects_prebuilt_wheel_kind( + tmp_context: context.WorkContext, +) -> None: + with patch("fromager.sources.overrides.find_and_invoke") as invoke: + with pytest.raises(ValueError, match="cannot prepare source"): + sources.prepare_source( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + source_filename=tmp_context.wheels_prebuilt / "pkg-1.0.whl", + version=Version("1.0"), + download_kind=packagesettings.DownloadKind.prebuilt_wheel, + ) + + invoke.assert_not_called() + + @patch("fromager.sources.write_build_meta") @patch("fromager.overrides.find_and_invoke") def test_prepare_source_plugin_returns_tuple( From c9cc5ddb3414536b5cc6fc40a1cad13a87fdfd78 Mon Sep 17 00:00:00 2001 From: Shanmukh Pawan Date: Fri, 23 Oct 2026 17:23:06 -0400 Subject: [PATCH 11/35] fix(resolver): honor configured prebuilt wheel resolvers Resolve and download fresh prebuilt wheels through the selected source resolver. Preserve local wheel cache reuse, legacy prebuilt downloads, prebuilt hooks, and the test-mode fallback. Add focused coverage for both configured profiles, build-order replay, and legacy behavior. Closes #1208 Co-Authored-By: Codex Signed-off-by: Shanmukh Pawan --- docs/concepts/bootstrapper-architecture.rst | 6 + docs/concepts/package-settings.rst | 10 + docs/concepts/resolver-architecture.rst | 16 +- .../bootstrap_requirement_resolver.py | 63 ++-- src/fromager/bootstrapper/_bootstrapper.py | 8 +- src/fromager/bootstrapper/_cache.py | 26 +- src/fromager/commands/build.py | 47 ++- src/fromager/commands/download_sequence.py | 88 ++--- src/fromager/packagesettings/_pbi.py | 5 +- src/fromager/server.py | 26 +- src/fromager/wheels.py | 59 ++++ tests/test_bootstrap_requirement_resolver.py | 182 +++++++++- ...bootstrap_requirement_resolver_multiple.py | 1 + tests/test_bootstrapper.py | 218 +++++++++++- tests/test_commands.py | 327 +++++++++++++++++- tests/test_packagesettings.py | 25 ++ tests/test_server.py | 38 ++ tests/test_wheels.py | 223 +++++++++++- 18 files changed, 1276 insertions(+), 92 deletions(-) diff --git a/docs/concepts/bootstrapper-architecture.rst b/docs/concepts/bootstrapper-architecture.rst index 37d7d5493..86a2cc91b 100644 --- a/docs/concepts/bootstrapper-architecture.rst +++ b/docs/concepts/bootstrapper-architecture.rst @@ -16,6 +16,12 @@ Bootstrap Phase Flow Every package passes through a sequence of phases. Source packages traverse the full pipeline; prebuilt wheels skip the build phases. +``pypi-prebuilt`` and ``hook-prebuilt`` use the selected source resolver to +download a fresh wheel in ``PrepareSource``. The legacy ``pre_built`` setting +keeps its existing wheel download path. Both routes update the local mirror +and record the package as prebuilt in the graph and build order. +When a local wheel is already available, bootstrap reuses it before asking +the selected provider to resolve a fresh wheel. .. code-block:: text diff --git a/docs/concepts/package-settings.rst b/docs/concepts/package-settings.rst index c516ff93a..e588b7f96 100644 --- a/docs/concepts/package-settings.rst +++ b/docs/concepts/package-settings.rst @@ -42,6 +42,16 @@ cannot be combined with package-level ``download_source`` or ``resolver_dist``. An explicit ``pre_built: false`` also conflicts; an omitted ``pre_built`` does not. +``PackageBuildInfo.pre_built`` follows the selected ``source`` resolver's +``resolves_prebuilt_wheel`` capability. Without ``source``, it reads the +legacy variant ``pre_built`` setting. Configured prebuilt resolvers download +fresh wheels through their ``download()`` contract; legacy prebuilt packages +keep the wheel downloader. + +.. versionchanged:: 0.102.0 + Configured prebuilt source resolvers now select the prebuilt pipeline + during bootstrap and later builds. + .. versionchanged:: 0.102.0 Source and legacy settings mixed across package and variant levels now raise a validation error when package settings load. diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index 4702b957b..18d4cf590 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -43,6 +43,8 @@ the cache already contains a matching version. The configured wheel cache server is not a version-resolution fallback. After a version has been selected, the bootstrapper checks the cache server separately for a previously built wheel. +For prebuilt packages, the local wheel server can supply a cached wheel +before a fresh resolution through the selected source provider. Provider Hierarchy ------------------ @@ -132,7 +134,15 @@ The separate ``step prepare-source`` invocation accepts ``--source-kind`` and ``step download-source-archive --output-format json`` exposes that result's actual path and kind without changing its default path-only output. Without ``--source-kind``, it keeps the legacy archive behavior. -Integrating prebuilt detection into the bootstrapper remains a separate step. +Configured prebuilt resolvers select the prebuilt bootstrap path. Fresh +external wheels use their ``download()`` contract and are staged in the +prebuilt wheel directory. Bootstrap, build, and download-sequence use the +direct wheel downloader for local mirror URLs, so they do not rerun the +configured download hook. Test-mode fallback also downloads directly. +When a fresh configured wheel shares a filename with an older cached wheel, +the local mirror and later build replay use the fresh staged wheel. +Legacy build downloads still pass through the build wheel directory before +the mirror moves them to downloads. See :ref:`download_source_hook` for the return contract and compatibility rules. .. versionchanged:: 0.102.0 @@ -143,7 +153,9 @@ See :ref:`download_source_hook` for the return contract and compatibility rules. ``build_sdist: tarball``. PyPI sdist resolvers repack a tarball. Both configured strategies take precedence over an installed ``build_sdist`` override and log a warning when it is ignored. Legacy - settings continue to use the override. + settings continue to use the override. Configured prebuilt profiles use + their source provider after a local cache check, and local mirror URLs + bypass configured download hooks during replay. Version Filtering Window ------------------------- diff --git a/src/fromager/bootstrap_requirement_resolver.py b/src/fromager/bootstrap_requirement_resolver.py index 8ee267511..6e8407627 100644 --- a/src/fromager/bootstrap_requirement_resolver.py +++ b/src/fromager/bootstrap_requirement_resolver.py @@ -10,9 +10,11 @@ import threading import typing +import requests.exceptions from packaging.requirements import Requirement from packaging.utils import NormalizedName, canonicalize_name from packaging.version import Version +from resolvelib.resolvers import ResolverException from . import resolver, sources, wheels from .dependency_graph import DependencyGraph @@ -171,13 +173,17 @@ def _resolve_and_extend( parent_req=parent_req, ) + source_resolver = self.ctx.package_build_info(req).source_resolver + configured_prebuilt = ( + source_resolver is not None and source_resolver.resolves_prebuilt_wheel + ) if cached_resolution: logger.debug( f"resolved from previous bootstrap: {len(cached_resolution)} version(s)" ) results = cached_resolution - elif pre_built: - # Resolve prebuilt wheel + elif pre_built and not configured_prebuilt: + # Legacy prebuilt resolution also serves test-mode source-build fallback. wheel_server_urls = wheels.get_wheel_server_urls( self.ctx, req, cache_wheel_server_url=resolver.PYPI_SERVER_URL ) @@ -188,30 +194,39 @@ def _resolve_and_extend( req_type=req_type, ) else: - # Resolve source (sdist) - provider = sources.get_source_provider( - ctx=self.ctx, - req=req, - sdist_server_url=resolver.PYPI_SERVER_URL, - req_type=req_type, - ) - max_age_cutoff = resolver._compute_max_age_cutoff(self.ctx) - age_fallback = ( - resolver.AgeFallback.NEWEST - if self.multiple_versions - else resolver.AgeFallback.ALL - ) - results = resolver.find_all_matching_from_provider( - provider, - req, - max_age_cutoff=max_age_cutoff, - age_fallback=age_fallback, - ) + results = [] + if configured_prebuilt and self.ctx.wheel_server_url: + try: + results = wheels.resolve_all_cached_wheels( + ctx=self.ctx, + req=req, + cache_server_url=self.ctx.wheel_server_url, + ) + except ResolverException: + logger.debug("no matching wheel in local cache") + except requests.exceptions.RequestException as err: + logger.warning("could not check local wheel cache: %s", err) if not results: - logger.warning( - "resolver returned no results (%s version mode)", - "multiple" if self.multiple_versions else "single", + # The configured source provider resolves either source archives + # or prebuilt wheels, according to its selected capability. + provider = sources.get_source_provider( + ctx=self.ctx, + req=req, + sdist_server_url=resolver.PYPI_SERVER_URL, + req_type=req_type, + ) + max_age_cutoff = resolver._compute_max_age_cutoff(self.ctx) + age_fallback = ( + resolver.AgeFallback.NEWEST + if self.multiple_versions + else resolver.AgeFallback.ALL + ) + results = resolver.find_all_matching_from_provider( + provider, + req, + max_age_cutoff=max_age_cutoff, + age_fallback=age_fallback, ) if results: diff --git a/src/fromager/bootstrapper/_bootstrapper.py b/src/fromager/bootstrapper/_bootstrapper.py index c524324c7..0cdeb5cdf 100644 --- a/src/fromager/bootstrapper/_bootstrapper.py +++ b/src/fromager/bootstrapper/_bootstrapper.py @@ -470,8 +470,12 @@ def _download_prebuilt( resolved_version: Version, wheel_url: str, ) -> tuple[pathlib.Path, pathlib.Path]: - result = _cache.bg_prepare_prebuilt( - self.ctx, req, req_type, resolved_version, wheel_url + result = _cache.bg_prepare_test_mode_prebuilt( + self.ctx, + req, + req_type, + resolved_version, + wheel_url, ) assert result.wheel_filename is not None assert result.unpack_dir is not None diff --git a/src/fromager/bootstrapper/_cache.py b/src/fromager/bootstrapper/_cache.py index 089895f5b..e241a7d63 100644 --- a/src/fromager/bootstrapper/_cache.py +++ b/src/fromager/bootstrapper/_cache.py @@ -246,10 +246,34 @@ def bg_prepare_prebuilt( resolved_version: Version, wheel_url: str, ) -> PreparedSourceData: - """Background-safe prebuilt download: no Bootstrapper state accessed.""" + """Download a prebuilt wheel and make it available to the local mirror.""" # Thread-safe: paths include {req.name}-{resolved_version} (unique per package), # mkdir uses exist_ok=True (atomic), and update_wheel_mirror() is already locked. logger.info(f"using pre-built wheel for {req_type} requirement") + wheel_filename = wheels.download_prebuilt_wheel( + ctx=ctx, + req=req, + version=resolved_version, + download_url=wheel_url, + direct_download_directory=ctx.wheels_prebuilt, + ) + unpack_dir = _create_unpack_dir(ctx.work_dir, req, resolved_version) + if ctx.package_build_info(req).source_resolver is not None: + server.update_wheel_mirror(ctx, preferred_prebuilt_wheel=wheel_filename) + else: + server.update_wheel_mirror(ctx) + return PreparedSourceData(wheel_filename=wheel_filename, unpack_dir=unpack_dir) + + +def bg_prepare_test_mode_prebuilt( + ctx: context.WorkContext, + req: Requirement, + req_type: RequirementType, + resolved_version: Version, + wheel_url: str, +) -> PreparedSourceData: + """Download a test-mode fallback wheel directly and update the mirror.""" + logger.info(f"using pre-built wheel for {req_type} requirement") wheel_filename = wheels.download_wheel(req, wheel_url, ctx.wheels_prebuilt) unpack_dir = _create_unpack_dir(ctx.work_dir, req, resolved_version) server.update_wheel_mirror(ctx) diff --git a/src/fromager/commands/build.py b/src/fromager/commands/build.py index 70d549484..24ceab603 100644 --- a/src/fromager/commands/build.py +++ b/src/fromager/commands/build.py @@ -362,17 +362,18 @@ def _build( if prebuilt: if not wheel_filename: logger.info("downloading prebuilt wheel") - wheel_filename = wheels.download_wheel( + wheel_filename = wheels.download_prebuilt_wheel( + ctx=wkctx, req=req, - wheel_url=source_download_url, - output_directory=wkctx.wheels_build, + version=resolved_version, + download_url=source_download_url, + direct_download_directory=wkctx.wheels_build, ) else: # already downloaded prebuilt wheel use_exiting_wheel = True - # Run hooks for prebuilt wheels. At this point wheel_filename should - # be set either from _is_wheel_built() or download_wheel(). + # Run hooks for both cached and freshly downloaded prebuilt wheels. hooks.run_prebuilt_wheel_hooks( ctx=wkctx, req=req, @@ -446,11 +447,19 @@ def _build( root_logger.removeHandler(file_handler) file_handler.close() - server.update_wheel_mirror(wkctx) + if ( + prebuilt + and pbi.source_resolver is not None + and wheel_filename.parent == wkctx.wheels_prebuilt + ): + server.update_wheel_mirror(wkctx, preferred_prebuilt_wheel=wheel_filename) + else: + server.update_wheel_mirror(wkctx) - # After we update the wheel mirror, the built file has - # moved to a new directory. - wheel_filename = wkctx.wheels_downloads / wheel_filename.name + # The mirror moves wheels from the build directory to downloads; + # configured prebuilt wheels stay in the prebuilt directory. + if wheel_filename.parent != wkctx.wheels_prebuilt: + wheel_filename = wkctx.wheels_downloads / wheel_filename.name return BuildSequenceEntry( name=canonicalize_name(req.name), @@ -529,8 +538,8 @@ def _is_wheel_built( logger.info("found candidate wheel %s", url) try: wheel_basename = downloads.extract_filename_from_url(url) - _, _, build_tag_from_name, wheel_tags = parse_wheel_filename( - wheel_basename + normalized_name, _, build_tag_from_name, wheel_tags = ( + parse_wheel_filename(wheel_basename) ) except Exception: logger.debug("could not parse candidate wheel %s", url, exc_info=True) @@ -567,9 +576,23 @@ def _is_wheel_built( continue wheel_filename: pathlib.Path | None = None - if wkctx.wheel_server_url and url.startswith(wkctx.wheel_server_url): + if wheels.is_local_wheel_url(url, wkctx.wheel_server_url): logging.debug("found wheel on local server") wheel_filename = wkctx.wheels_downloads / wheel_basename + mirror_wheel = wkctx.wheel_server_dir / normalized_name / wheel_basename + if mirror_wheel.is_symlink(): + mirror_target = mirror_wheel.resolve() + if ( + mirror_target.parent + in ( + wkctx.wheels_downloads, + wkctx.wheels_prebuilt, + ) + and mirror_target.is_file() + ): + # The mirror may point to a fresh prebuilt wheel while + # an older copy remains in downloads. + wheel_filename = mirror_target if not wheel_filename.exists(): logger.info("wheel not found in local cache, preparing to download") wheel_filename = None diff --git a/src/fromager/commands/download_sequence.py b/src/fromager/commands/download_sequence.py index 177440fd5..43876263b 100644 --- a/src/fromager/commands/download_sequence.py +++ b/src/fromager/commands/download_sequence.py @@ -8,7 +8,7 @@ from packaging.version import Version from .. import context, progress, read, sources, threading_utils, wheels -from ..log import requirement_ctxvar +from ..log import req_ctxvar_context logger = logging.getLogger(__name__) @@ -65,54 +65,60 @@ def download_sequence( def download_one(entry: dict[str, typing.Any]) -> None: req = Requirement(f"{entry['dist']}=={entry['version']}") - token = requirement_ctxvar.set(req) - - if entry["prebuilt"]: - if include_wheels: - wheels.download_wheel(req, entry["source_url"], wkctx.wheels_prebuilt) - else: - logger.info(f"{entry['dist']}: uses a pre-built wheel, skipping") - return - - if entry["source_url_type"] == "sdist": - try: - sources.download_source( - ctx=wkctx, - req=req, - version=Version(entry["version"]), - download_url=entry["source_url"], - ) - except Exception as err: - logger.error(f"failed to download sdist for {req}: {err}") - if not ignore_missing_sdists: - raise RuntimeError(f"Failed to download sdist for {req}") from err - else: - logger.info( - f"{entry['dist']}: uses a {entry['source_url_type']} downloader, skipping" - ) - - if include_wheels: - try: - if use_cache_resolution: - resolved_url, _ = wheels.resolve_cached_wheel( + with req_ctxvar_context(req): + if entry["prebuilt"]: + if include_wheels: + wheels.download_prebuilt_wheel( ctx=wkctx, req=req, - cache_server_url=wheel_server_url, + version=Version(entry["version"]), + download_url=entry["source_url"], + direct_download_directory=wkctx.wheels_prebuilt, ) else: - resolved_url, _ = wheels.resolve_prebuilt_wheel( + logger.info(f"{entry['dist']}: uses a pre-built wheel, skipping") + return + + if entry["source_url_type"] == "sdist": + try: + sources.download_source( ctx=wkctx, req=req, - wheel_server_urls=[wheel_server_url], + version=Version(entry["version"]), + download_url=entry["source_url"], ) - wheels.download_wheel( - req=req, - wheel_url=resolved_url, - output_directory=wkctx.wheels_downloads, + except Exception as err: + logger.error(f"failed to download sdist for {req}: {err}") + if not ignore_missing_sdists: + raise RuntimeError( + f"Failed to download sdist for {req}" + ) from err + else: + logger.info( + f"{entry['dist']}: uses a {entry['source_url_type']} downloader, skipping" ) - except Exception as err: - logger.error(f"failed to download wheel for {req}: {err}") - requirement_ctxvar.reset(token) + + if include_wheels: + try: + if use_cache_resolution: + resolved_url, _ = wheels.resolve_cached_wheel( + ctx=wkctx, + req=req, + cache_server_url=wheel_server_url, + ) + else: + resolved_url, _ = wheels.resolve_prebuilt_wheel( + ctx=wkctx, + req=req, + wheel_server_urls=[wheel_server_url], + ) + wheels.download_wheel( + req=req, + wheel_url=resolved_url, + output_directory=wkctx.wheels_downloads, + ) + except Exception as err: + logger.error(f"failed to download wheel for {req}: {err}") num_items = len(build_order) logger.debug( diff --git a/src/fromager/packagesettings/_pbi.py b/src/fromager/packagesettings/_pbi.py index ba62f9bf9..74d55a45b 100644 --- a/src/fromager/packagesettings/_pbi.py +++ b/src/fromager/packagesettings/_pbi.py @@ -163,7 +163,10 @@ def has_customizations(self) -> bool: @property def pre_built(self) -> bool: - """Does the variant use pre-build wheels?""" + """Whether the selected source or legacy variant uses prebuilt wheels.""" + source_resolver = self.source_resolver + if source_resolver is not None: + return source_resolver.resolves_prebuilt_wheel vi = self._ps.variants.get(self.variant) if vi is not None: return vi.pre_built diff --git a/src/fromager/server.py b/src/fromager/server.py index 7d282f871..17e03ae8f 100644 --- a/src/fromager/server.py +++ b/src/fromager/server.py @@ -10,6 +10,7 @@ import textwrap import threading import typing +import uuid from urllib.parse import quote import uvicorn @@ -59,7 +60,10 @@ def run_wheel_server( @with_thread_lock() -def update_wheel_mirror(ctx: context.WorkContext) -> None: +def update_wheel_mirror( + ctx: context.WorkContext, *, preferred_prebuilt_wheel: pathlib.Path | None = None +) -> None: + """Refresh the local index, preferring a newly staged wheel when provided.""" for wheel in ctx.wheels_build.glob("*.whl"): logger.info("adding %s to local wheel server", wheel.name) downloads_dest_filename = ctx.wheels_downloads / wheel.name @@ -88,6 +92,26 @@ def update_wheel_mirror(ctx: context.WorkContext) -> None: simple_dest_filename.parent.mkdir(parents=True, exist_ok=True) simple_dest_filename.symlink_to(relpath) + # A fresh configured wheel can share a filename with an older download. + # Switch the index link atomically so readers always see a complete wheel. + if preferred_prebuilt_wheel is not None: + (normalized_name, _, _, _) = parse_wheel_filename(preferred_prebuilt_wheel.name) + simple_dest_filename = ( + ctx.wheel_server_dir / normalized_name / preferred_prebuilt_wheel.name + ) + if simple_dest_filename.resolve() != preferred_prebuilt_wheel.resolve(): + relpath = os.path.relpath( + preferred_prebuilt_wheel, simple_dest_filename.parent + ) + temporary_link = simple_dest_filename.with_name( + f".{simple_dest_filename.name}.{uuid.uuid4().hex}" + ) + try: + temporary_link.symlink_to(relpath) + os.replace(temporary_link, simple_dest_filename) + finally: + temporary_link.unlink(missing_ok=True) + class SimpleHTMLIndex: """Simple HTML Repository API (1.0) diff --git a/src/fromager/wheels.py b/src/fromager/wheels.py index 943edea8d..b1a36cb29 100644 --- a/src/fromager/wheels.py +++ b/src/fromager/wheels.py @@ -32,6 +32,7 @@ requirements_file, resolver, sbom, + sources, ) from .pkgmetadata.pep376 import verbatim_dist_name @@ -520,6 +521,64 @@ def download_wheel( return wheel_filename +def is_local_wheel_url(wheel_url: str, wheel_server_url: str | None) -> bool: + """Return whether a wheel URL belongs to the local wheel mirror. + + .. versionadded:: 0.102.0 + """ + if not wheel_server_url: + return False + return wheel_url.startswith(wheel_server_url.rstrip("/") + "/") + + +def download_prebuilt_wheel( + *, + ctx: context.WorkContext, + req: Requirement, + version: Version, + download_url: str, + direct_download_directory: pathlib.Path, +) -> pathlib.Path: + """Download a prebuilt wheel via its resolver or the direct wheel path. + + Configured external wheels use the resolver download contract. Legacy + wheels and local-mirror URLs use the direct wheel downloader. + + .. versionadded:: 0.102.0 + """ + source_resolver = ctx.package_build_info(req).source_resolver + if source_resolver is not None and not source_resolver.resolves_prebuilt_wheel: + raise ValueError( + f"{req}: source resolver {source_resolver.provider!r} " + "does not resolve prebuilt wheels" + ) + if source_resolver is None or is_local_wheel_url( + download_url, ctx.wheel_server_url + ): + return download_wheel(req, download_url, direct_download_directory) + + downloaded: packagesettings.DownloadedSource = sources.download_source( + ctx=ctx, req=req, version=version, download_url=download_url + ) + wheel_path = downloaded.path + if wheel_path.parent != ctx.wheels_prebuilt: + ctx.wheels_prebuilt.mkdir(parents=True, exist_ok=True) + staged_path = ctx.wheels_prebuilt / wheel_path.name + with tempfile.NamedTemporaryFile( + dir=ctx.wheels_prebuilt, + prefix=f".tmp.{wheel_path.name}.", + delete=False, + ) as temporary_file: + temporary_path = pathlib.Path(temporary_file.name) + try: + shutil.copy2(wheel_path, temporary_path) + os.replace(temporary_path, staged_path) + finally: + temporary_path.unlink(missing_ok=True) + return staged_path + return wheel_path + + def get_wheel_server_urls( ctx: context.WorkContext, req: Requirement, *, cache_wheel_server_url: str | None ) -> list[str]: diff --git a/tests/test_bootstrap_requirement_resolver.py b/tests/test_bootstrap_requirement_resolver.py index af359313c..f6fa5b570 100644 --- a/tests/test_bootstrap_requirement_resolver.py +++ b/tests/test_bootstrap_requirement_resolver.py @@ -2,11 +2,14 @@ from unittest.mock import MagicMock, patch +import pytest +import requests.exceptions from packaging.requirements import Requirement from packaging.utils import canonicalize_name from packaging.version import Version +from resolvelib.resolvers import ResolverException -from fromager import resolver +from fromager import packagesettings, resolver from fromager.bootstrap_requirement_resolver import BootstrapRequirementResolver from fromager.context import WorkContext from fromager.dependency_graph import DependencyGraph @@ -433,6 +436,182 @@ def test_resolve_from_graph_name_fallback_returns_none_for_missing_package( assert result is None +def test_configured_pypi_prebuilt_resolves_from_selected_index( + tmp_context: WorkContext, +) -> None: + """Fresh prebuilt resolution uses the configured index provider.""" + req = Requirement("pkg==1.0") + tmp_context.settings._package_settings[canonicalize_name("pkg")] = ( + packagesettings.PackageSettings.from_string( + "pkg", + "source:\n provider: pypi-prebuilt\n" + " index_url: https://custom.test/simple/\n", + ) + ) + expected = [("https://custom.test/pkg-1.0-py3-none-any.whl", Version("1.0"))] + with ( + patch( + "fromager.bootstrap_requirement_resolver.resolver.find_all_matching_from_provider", + return_value=expected, + ) as find_matches, + patch( + "fromager.bootstrap_requirement_resolver.wheels.resolve_all_prebuilt_wheels" + ) as legacy, + ): + result = BootstrapRequirementResolver(tmp_context).resolve( + req, RequirementType.TOP_LEVEL + ) + + assert result == expected + provider = find_matches.call_args.args[0] + assert isinstance(provider, resolver.PyPIProvider) + assert provider.sdist_server_url == "https://custom.test/simple/" + legacy.assert_not_called() + + +def test_configured_hook_prebuilt_uses_hook_provider( + tmp_context: WorkContext, +) -> None: + """Fresh prebuilt resolution calls the selected hook provider.""" + req = Requirement("pkg==1.0") + tmp_context.settings._package_settings[canonicalize_name("pkg")] = ( + packagesettings.PackageSettings.from_string( + "pkg", "source:\n provider: hook-prebuilt\n" + ) + ) + provider = resolver.PyPIProvider(sdist_server_url="https://hook.test/simple/") + expected = [("https://hook.test/pkg-1.0-py3-none-any.whl", Version("1.0"))] + with ( + patch.object( + packagesettings.HookPrebuiltResolver, + "resolver_provider", + return_value=provider, + ) as configured, + patch( + "fromager.bootstrap_requirement_resolver.resolver.find_all_matching_from_provider", + return_value=expected, + ) as find_matches, + patch( + "fromager.bootstrap_requirement_resolver.wheels.resolve_all_prebuilt_wheels" + ) as legacy, + ): + result = BootstrapRequirementResolver(tmp_context).resolve( + req, RequirementType.TOP_LEVEL + ) + + assert result == expected + configured.assert_called_once() + assert find_matches.call_args.args[0] is provider + legacy.assert_not_called() + + +def test_configured_prebuilt_reuses_local_wheel_cache( + tmp_context: WorkContext, +) -> None: + """A local wheel is resolved before the configured upstream provider.""" + req = Requirement("pkg==1.0") + tmp_context.settings._package_settings[canonicalize_name("pkg")] = ( + packagesettings.PackageSettings.from_string( + "pkg", "source:\n provider: pypi-prebuilt\n" + ) + ) + tmp_context.wheel_server_url = "https://local.test/simple/" + expected = [("https://local.test/pkg-1.0-py3-none-any.whl", Version("1.0"))] + with ( + patch( + "fromager.bootstrap_requirement_resolver.wheels.resolve_all_cached_wheels", + return_value=expected, + ) as cache, + patch( + "fromager.bootstrap_requirement_resolver.sources.get_source_provider" + ) as configured, + patch( + "fromager.bootstrap_requirement_resolver.wheels.resolve_all_prebuilt_wheels", + return_value=[], + ) as legacy, + ): + result = BootstrapRequirementResolver(tmp_context).resolve( + req, RequirementType.TOP_LEVEL + ) + + assert result == expected + cache.assert_called_once_with( + ctx=tmp_context, req=req, cache_server_url=tmp_context.wheel_server_url + ) + configured.assert_not_called() + legacy.assert_not_called() + + +@pytest.mark.parametrize( + "cache_error", + [ + ResolverException("cache miss"), + requests.exceptions.ConnectionError("cache down"), + ], +) +def test_configured_prebuilt_cache_miss_uses_selected_provider( + tmp_context: WorkContext, cache_error: Exception +) -> None: + """A local cache miss or network error falls through to the source provider.""" + req = Requirement("pkg==1.0") + tmp_context.settings._package_settings[canonicalize_name("pkg")] = ( + packagesettings.PackageSettings.from_string( + "pkg", + "source:\n provider: pypi-prebuilt\n" + " index_url: https://custom.test/simple/\n", + ) + ) + tmp_context.wheel_server_url = "https://local.test/simple/" + expected = [("https://custom.test/pkg-1.0-py3-none-any.whl", Version("1.0"))] + with ( + patch( + "fromager.bootstrap_requirement_resolver.wheels.resolve_all_cached_wheels", + side_effect=cache_error, + ) as cache, + patch( + "fromager.bootstrap_requirement_resolver.resolver.find_all_matching_from_provider", + return_value=expected, + ) as find_matches, + ): + result = BootstrapRequirementResolver(tmp_context).resolve( + req, RequirementType.TOP_LEVEL + ) + + assert result == expected + cache.assert_called_once() + assert find_matches.call_args.args[0].sdist_server_url == ( + "https://custom.test/simple/" + ) + + +def test_configured_prebuilt_unexpected_cache_error_propagates( + tmp_context: WorkContext, +) -> None: + """An unexpected cache failure is not mistaken for a cache miss.""" + req = Requirement("pkg==1.0") + tmp_context.settings._package_settings[canonicalize_name("pkg")] = ( + packagesettings.PackageSettings.from_string( + "pkg", "source:\n provider: pypi-prebuilt\n" + ) + ) + tmp_context.wheel_server_url = "https://local.test/simple/" + with ( + patch( + "fromager.bootstrap_requirement_resolver.wheels.resolve_all_cached_wheels", + side_effect=ValueError("invalid cache response"), + ), + patch( + "fromager.bootstrap_requirement_resolver.sources.get_source_provider" + ) as configured, + pytest.raises(ValueError, match="invalid cache response"), + ): + BootstrapRequirementResolver(tmp_context).resolve( + req, RequirementType.TOP_LEVEL + ) + + configured.assert_not_called() + + @patch("fromager.resolver.find_all_matching_from_provider") def test_resolve_auto_routes_to_prebuilt( mock_resolve: MagicMock, @@ -444,6 +623,7 @@ def test_resolve_auto_routes_to_prebuilt( # Mock package build info to return pre_built=True mock_pbi = MagicMock() mock_pbi.pre_built = True + mock_pbi.source_resolver = None mock_pbi.wheel_server_url = None mock_pbi.resolver_min_release_age = None diff --git a/tests/test_bootstrap_requirement_resolver_multiple.py b/tests/test_bootstrap_requirement_resolver_multiple.py index ce9482def..5cf208cff 100644 --- a/tests/test_bootstrap_requirement_resolver_multiple.py +++ b/tests/test_bootstrap_requirement_resolver_multiple.py @@ -25,6 +25,7 @@ def tmp_context(tmp_path: Path) -> WorkContext: ctx.package_build_info = MagicMock() pbi = MagicMock() pbi.pre_built = False + pbi.source_resolver = None pbi.resolver_include_sdists = True pbi.resolver_include_wheels = False pbi.resolver_ignore_platform = False diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index 0d775339c..fc2cda9d3 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -1,3 +1,4 @@ +import concurrent.futures import json import logging import pathlib @@ -12,7 +13,7 @@ from packaging.version import Version from resolvelib.resolvers import ResolverException -from fromager import bootstrapper, log +from fromager import bootstrapper, log, sources, wheels from fromager.bootstrapper._build import Build from fromager.bootstrapper._cache import ( _download_wheel_from_cache, @@ -38,10 +39,21 @@ from fromager.bootstrapper._work_item import WorkItem from fromager.context import WorkContext from fromager.dependency_graph import ROOT -from fromager.packagesettings import DownloadedSource, DownloadKind, WheelSettings +from fromager.packagesettings import ( + DownloadedSource, + DownloadKind, + PackageSettings, + WheelSettings, +) from fromager.requirements_file import RequirementType, SourceType +def _configure_package(ctx: WorkContext, yaml_str: str) -> None: + """Apply one package setting to the test context.""" + ps = PackageSettings.from_string("pkg", yaml_str) + ctx.settings._package_settings[ps.name] = ps + + def test_phase_subclass_without_phase_attribute_raises() -> None: """Concrete Phase subclass missing 'phase' raises TypeError at class definition.""" with pytest.raises(TypeError, match="must define the 'phase' class attribute"): @@ -63,6 +75,206 @@ def helper(self) -> None: ... assert _AbstractMiddle.__abstractmethods__ +@pytest.mark.parametrize("provider", ["pypi-prebuilt", "hook-prebuilt"]) +def test_bootstrap_prebuilt_uses_configured_download( + tmp_context: WorkContext, provider: str +) -> None: + """Bootstrap consumes the configured wheel path and updates its mirror.""" + _configure_package(tmp_context, f"source:\n provider: {provider}\n") + req = Requirement("pkg==1.0") + version = Version("1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + wheel = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(wheels, "download_prebuilt_wheel", return_value=wheel) as download, + patch("fromager.bootstrapper._cache.server.update_wheel_mirror") as mirror, + ): + result = bg_prepare_prebuilt( + tmp_context, req, RequirementType.TOP_LEVEL, version, url + ) + + assert result.wheel_filename == wheel + download.assert_called_once_with( + ctx=tmp_context, + req=req, + version=version, + download_url=url, + direct_download_directory=tmp_context.wheels_prebuilt, + ) + mirror.assert_called_once_with(tmp_context, preferred_prebuilt_wheel=wheel) + + +def test_bootstrap_configured_prebuilt_reuses_local_wheel( + tmp_context: WorkContext, +) -> None: + """A local mirror URL bypasses the configured download hook.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + req = Requirement("pkg==1.0") + version = Version("1.0") + tmp_context.wheel_server_url = "https://local.test/simple/" + url = "https://local.test/simple/pkg/pkg-1.0-py3-none-any.whl" + wheel = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(wheels, "download_wheel", return_value=wheel) as cached, + patch.object(sources, "download_source") as configured, + patch("fromager.bootstrapper._cache.server.update_wheel_mirror") as mirror, + ): + result = bg_prepare_prebuilt( + tmp_context, req, RequirementType.TOP_LEVEL, version, url + ) + + assert result.wheel_filename == wheel + cached.assert_called_once_with(req, url, tmp_context.wheels_prebuilt) + configured.assert_not_called() + mirror.assert_called_once_with(tmp_context, preferred_prebuilt_wheel=wheel) + + +@pytest.mark.parametrize("local_cache", [False, True]) +def test_bootstrap_prebuilt_rejects_source_only_resolver( + tmp_context: WorkContext, local_cache: bool +) -> None: + """Do not treat a configured sdist resolver as a legacy wheel source.""" + _configure_package(tmp_context, "source:\n provider: pypi-sdist\n") + req = Requirement("pkg==1.0") + if local_cache: + tmp_context.wheel_server_url = "https://local.test/simple/" + url = ( + "https://local.test/simple/pkg/pkg-1.0-py3-none-any.whl" + if local_cache + else "https://pkg.test/pkg-1.0-py3-none-any.whl" + ) + with ( + patch.object(wheels, "download_wheel") as direct, + patch.object(sources, "download_source") as configured, + pytest.raises(ValueError, match="does not resolve prebuilt wheels"), + ): + bg_prepare_prebuilt( + tmp_context, + req, + RequirementType.TOP_LEVEL, + Version("1.0"), + url, + ) + + direct.assert_not_called() + configured.assert_not_called() + + +def test_legacy_bootstrap_keeps_direct_download(tmp_context: WorkContext) -> None: + """Legacy bootstrap still uses the wheel downloader.""" + _configure_package(tmp_context, "variants:\n cpu:\n pre_built: true\n") + req = Requirement("pkg==1.0") + version = Version("1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + wheel = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(wheels, "download_wheel", return_value=wheel) as direct, + patch.object(sources, "download_source") as configured, + patch("fromager.bootstrapper._cache.server.update_wheel_mirror") as mirror, + ): + result = bg_prepare_prebuilt( + tmp_context, req, RequirementType.TOP_LEVEL, version, url + ) + + assert result.wheel_filename == wheel + direct.assert_called_once_with(req, url, tmp_context.wheels_prebuilt) + configured.assert_not_called() + mirror.assert_called_once_with(tmp_context) + + +@pytest.mark.parametrize("provider", ["pypi-prebuilt", "hook-prebuilt"]) +def test_bootstrap_records_configured_prebuilt_in_graph_and_order( + tmp_context: WorkContext, provider: str +) -> None: + """The selected source makes bootstrap skip build phases and persist its flag.""" + _configure_package(tmp_context, f"source:\n provider: {provider}\n") + bt = bootstrapper.Bootstrapper(tmp_context) + req = Requirement("pkg==1.0") + version = Version("1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + with patch.object(bt, "resolve_versions", return_value=[(url, version)]): + bt._resolve_and_add_top_level(req) + + wi = WorkItem( + req=req, + req_type=RequirementType.TOP_LEVEL, + why_snapshot=[], + source_url=url, + resolved_version=version, + ) + next_phases = Start(wi).run(bt) + assert wi.pbi_pre_built is True + assert len(next_phases) == 1 + assert isinstance(next_phases[0], PrepareSource) + wheel = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(wheels, "download_prebuilt_wheel", return_value=wheel), + patch.object(sources, "prepare_source") as prepare_source, + patch("fromager.bootstrapper._cache.server.update_wheel_mirror"), + ): + background_work = next_phases[0].background_work(bt) + assert background_work is not None + prepared = background_work() + prepare_source.assert_not_called() + future: concurrent.futures.Future[typing.Any] = concurrent.futures.Future() + future.set_result(prepared) + next_phases[0].bg_future = future + install_phases = next_phases[0].run(bt) + assert len(install_phases) == 1 + assert isinstance(install_phases[0], ProcessInstallDeps) + assert wi.build_result is not None + assert wi.build_result.source_type is SourceType.PREBUILT + with ( + patch( + "fromager.bootstrapper._process_install_deps.hooks.run_post_bootstrap_hooks" + ), + patch( + "fromager.bootstrapper._process_install_deps._get_install_dependencies", + return_value=[], + ), + patch.object(bt, "create_unresolved_work_items", return_value=[]), + ): + install_phases[0].run(bt) + bt.finalize() + + assert tmp_context.dependency_graph.nodes["pkg==1.0"].pre_built is True + build_order = json.loads(bt._build_order_filename.read_text()) + assert build_order[0]["prebuilt"] is True + assert build_order[0]["source_url"] == url + + +def test_test_mode_source_resolver_uses_legacy_prebuilt_fallback( + tmp_context: WorkContext, +) -> None: + """A failed source build can use the forced legacy wheel fallback.""" + req = Requirement("pkg==1.0") + version = Version("1.0") + url = "https://fallback.test/pkg-1.0-py3-none-any.whl" + wheel = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" + tmp_context.settings._package_settings[canonicalize_name("pkg")] = ( + PackageSettings.from_string("pkg", "source:\n provider: pypi-sdist\n") + ) + bt = bootstrapper.Bootstrapper(tmp_context, test_mode=True) + with ( + patch.object(bt._resolver, "resolve", return_value=[(url, version)]), + patch( + "fromager.bootstrapper._cache.wheels.download_wheel", return_value=wheel + ) as legacy, + patch("fromager.bootstrapper._cache.server.update_wheel_mirror"), + ): + result = bt._handle_test_mode_failure( + req=req, + resolved_version=version, + req_type=RequirementType.TOP_LEVEL, + build_error=RuntimeError("source build failed"), + ) + + assert result is not None + assert result.source_type is SourceType.PREBUILT + assert result.wheel_filename == wheel + legacy.assert_called_once_with(req, url, tmp_context.wheels_prebuilt) + + def test_seen(tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) req = Requirement("testdist") @@ -949,8 +1161,6 @@ def test_check_write_error_logs_on_failure( bt = bootstrapper.Bootstrapper(tmp_context) # Arrange: create a future that has already failed with an OSError - import concurrent.futures - fut: concurrent.futures.Future[int] = concurrent.futures.Future() fut.set_exception(OSError("disk full")) diff --git a/tests/test_commands.py b/tests/test_commands.py index 6b4ae406c..61b752e60 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,3 +1,4 @@ +import concurrent.futures import json import pathlib import subprocess @@ -5,13 +6,335 @@ from unittest.mock import Mock, patch import click +import pytest from click.testing import CliRunner from packaging.requirements import Requirement from packaging.version import Version -from fromager import packagesettings -from fromager.commands import bootstrap, build, step +from fromager import packagesettings, server, wheels +from fromager.commands import bootstrap, build, download_sequence, step from fromager.context import WorkContext +from fromager.log import requirement_ctxvar + + +def _configure_package(ctx: WorkContext, yaml_str: str) -> None: + """Apply one package setting to the test context.""" + ps = packagesettings.PackageSettings.from_string("pkg", yaml_str) + ctx.settings._package_settings[ps.name] = ps + + +@pytest.mark.parametrize("provider", ["pypi-prebuilt", "hook-prebuilt"]) +def test_build_order_replay_uses_configured_download( + tmp_context: WorkContext, provider: str +) -> None: + """A later build run selects the configured wheel download contract.""" + _configure_package(tmp_context, f"source:\n provider: {provider}\n") + req = Requirement("pkg==1.0") + version = Version("1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + wheel = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(build, "_is_wheel_built", return_value=None), + patch.object(wheels, "download_prebuilt_wheel", return_value=wheel) as download, + patch.object(build.wheels, "download_wheel") as legacy, + patch.object(build.hooks, "run_prebuilt_wheel_hooks") as hooks, + patch.object(build.server, "update_wheel_mirror") as mirror, + ): + result = build._build(tmp_context, version, req, url, False, None) + + assert result.prebuilt is True + assert result.wheel_filename == wheel + download.assert_called_once_with( + ctx=tmp_context, + req=req, + version=version, + download_url=url, + direct_download_directory=tmp_context.wheels_build, + ) + legacy.assert_not_called() + hooks.assert_called_once() + mirror.assert_called_once_with(tmp_context, preferred_prebuilt_wheel=wheel) + + +def test_build_replay_local_mirror_url_skips_download_hook( + tmp_context: WorkContext, +) -> None: + """Build replay downloads a local mirror URL directly and runs wheel hooks.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + tmp_context.wheel_server_url = "https://local.test/simple/" + req = Requirement("pkg==1.0") + url = "https://local.test/simple/pkg/pkg-1.0-py3-none-any.whl" + wheel = tmp_context.wheels_build / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(build, "_is_wheel_built", return_value=None), + patch.object(wheels, "download_wheel", return_value=wheel) as direct, + patch("fromager.wheels.sources.download_source") as configured, + patch.object(build.hooks, "run_prebuilt_wheel_hooks") as hooks, + patch.object(build.server, "update_wheel_mirror"), + ): + result = build._build(tmp_context, Version("1.0"), req, url, False, None) + + assert result.wheel_filename == tmp_context.wheels_downloads / wheel.name + direct.assert_called_once_with(req, url, tmp_context.wheels_build) + configured.assert_not_called() + hooks.assert_called_once() + + +def test_fresh_configured_prebuilt_replaces_old_mirror_entry( + tmp_context: WorkContext, +) -> None: + """A forced hook download makes its new wheel visible in the mirror.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + req = Requirement("pkg==1.0") + wheel_name = "pkg-1.0-py3-none-any.whl" + old_wheel = tmp_context.wheels_downloads / wheel_name + old_wheel.write_bytes(b"old wheel") + server.update_wheel_mirror(tmp_context) + + fresh_wheel = tmp_context.wheels_prebuilt / wheel_name + fresh_wheel.write_bytes(b"fresh wheel") + with ( + patch.object(wheels, "download_prebuilt_wheel", return_value=fresh_wheel), + patch.object(build.hooks, "run_prebuilt_wheel_hooks"), + ): + result = build._build( + tmp_context, + Version("1.0"), + req, + f"https://pkg.test/{wheel_name}", + True, + None, + ) + + mirror_wheel = tmp_context.wheel_server_dir / "pkg" / wheel_name + assert result.wheel_filename == fresh_wheel + assert mirror_wheel.resolve() == fresh_wheel + assert mirror_wheel.read_bytes() == b"fresh wheel" + assert old_wheel.read_bytes() == b"old wheel" + + tmp_context.wheel_server_url = "https://local.test/simple/" + mirror_url = f"https://local.test/simple/pkg/{wheel_name}" + with ( + patch.object( + wheels, + "resolve_all_cached_wheels", + return_value=[(mirror_url, Version("1.0"))], + ), + patch.object(wheels, "download_wheel") as direct, + ): + cached_wheel = build._is_wheel_built(tmp_context, "pkg", Version("1.0")) + + assert cached_wheel == fresh_wheel + direct.assert_not_called() + + +def test_configured_prebuilt_reuses_cached_wheel(tmp_context: WorkContext) -> None: + """A cache hit avoids a fresh resolver download while retaining hooks.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + req = Requirement("pkg==1.0") + version = Version("1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + cached = tmp_context.wheels_downloads / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(build, "_is_wheel_built", return_value=cached), + patch.object(wheels, "download_prebuilt_wheel") as download, + patch.object(build.hooks, "run_prebuilt_wheel_hooks") as hooks, + patch.object(build.server, "update_wheel_mirror"), + ): + result = build._build(tmp_context, version, req, url, False, None) + + assert result.wheel_filename == cached + assert result.skipped is True + download.assert_not_called() + hooks.assert_called_once() + + +def test_download_sequence_replays_configured_prebuilt( + cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path +) -> None: + """Download sequence uses the saved URL with the selected source resolver.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [{"dist": "pkg", "version": "1.0", "prebuilt": True, "source_url": url}] + ) + ) + with ( + patch.object(wheels, "download_prebuilt_wheel") as download, + patch.object(download_sequence.wheels, "download_wheel") as legacy, + ): + result = cli_runner.invoke( + download_sequence.download_sequence, + [ + str(build_order), + "https://index.test/simple", + "--include-wheels", + "--num-threads", + "1", + ], + obj=tmp_context, + ) + + assert result.exit_code == 0, result.output + download.assert_called_once_with( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + version=Version("1.0"), + download_url=url, + direct_download_directory=tmp_context.wheels_prebuilt, + ) + legacy.assert_not_called() + + +def test_download_sequence_local_mirror_url_skips_download_hook( + cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path +) -> None: + """Download replay fetches a local mirror wheel without invoking its hook.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + tmp_context.wheel_server_url = "https://local.test/simple/" + url = "https://local.test/simple/pkg/pkg-1.0-py3-none-any.whl" + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [{"dist": "pkg", "version": "1.0", "prebuilt": True, "source_url": url}] + ) + ) + with ( + patch.object(wheels, "download_wheel") as direct, + patch("fromager.wheels.sources.download_source") as configured, + ): + result = cli_runner.invoke( + download_sequence.download_sequence, + [ + str(build_order), + "https://index.test/simple", + "--include-wheels", + "--num-threads", + "1", + ], + obj=tmp_context, + ) + + assert result.exit_code == 0, result.output + direct.assert_called_once_with( + Requirement("pkg==1.0"), url, tmp_context.wheels_prebuilt + ) + configured.assert_not_called() + + +@pytest.mark.parametrize("download_fails", [False, True]) +def test_download_sequence_clears_requirement_context( + cli_runner: CliRunner, + tmp_context: WorkContext, + tmp_path: pathlib.Path, + download_fails: bool, +) -> None: + """A prebuilt return or error leaves no package context on the worker.""" + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [ + { + "dist": "pkg", + "version": "1.0", + "prebuilt": True, + "source_url": "https://pkg.test/pkg-1.0-py3-none-any.whl", + } + ] + ) + ) + failure = ValueError("download failed") if download_fails else None + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + with ( + patch.object( + download_sequence.futures, + "ThreadPoolExecutor", + return_value=executor, + ), + patch.object(wheels, "download_prebuilt_wheel", side_effect=failure), + ): + result = cli_runner.invoke( + download_sequence.download_sequence, + [ + str(build_order), + "https://index.test/simple", + "--include-wheels", + "--num-threads", + "1", + ], + obj=tmp_context, + ) + assert executor.submit(requirement_ctxvar.get, None).result() is None + + if download_fails: + assert isinstance(result.exception, ValueError) + else: + assert result.exit_code == 0, result.output + + +def test_download_sequence_rejects_stale_prebuilt_flag( + cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path +) -> None: + """A source-only resolver cannot replay a prebuilt build-order entry.""" + _configure_package(tmp_context, "source:\n provider: pypi-sdist\n") + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [ + { + "dist": "pkg", + "version": "1.0", + "prebuilt": True, + "source_url": "https://pkg.test/pkg-1.0-py3-none-any.whl", + } + ] + ) + ) + with ( + patch.object(wheels, "download_wheel") as direct, + patch("fromager.wheels.sources.download_source") as configured, + ): + result = cli_runner.invoke( + download_sequence.download_sequence, + [ + str(build_order), + "https://index.test/simple", + "--include-wheels", + "--num-threads", + "1", + ], + obj=tmp_context, + ) + + assert isinstance(result.exception, ValueError) + assert "does not resolve prebuilt wheels" in str(result.exception) + configured.assert_not_called() + direct.assert_not_called() + + +def test_legacy_prebuilt_keeps_direct_download(tmp_context: WorkContext) -> None: + """Legacy pre_built still downloads into the build wheel directory.""" + _configure_package(tmp_context, "variants:\n cpu:\n pre_built: true\n") + req = Requirement("pkg==1.0") + version = Version("1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + wheel = tmp_context.wheels_build / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(build, "_is_wheel_built", return_value=None), + patch.object(build.wheels, "download_wheel", return_value=wheel) as legacy, + patch("fromager.wheels.sources.download_source") as configured, + patch.object(build.hooks, "run_prebuilt_wheel_hooks"), + patch.object(build.server, "update_wheel_mirror") as mirror, + ): + result = build._build(tmp_context, version, req, url, False, None) + + assert result.prebuilt is True + assert result.wheel_filename == tmp_context.wheels_downloads / wheel.name + legacy.assert_called_once_with(req, url, tmp_context.wheels_build) + configured.assert_not_called() + mirror.assert_called_once_with(tmp_context) def get_option_names(cmd: click.Command) -> typing.Iterable[str]: diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index 610f2097f..ce18db756 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -1255,6 +1255,31 @@ def test_pbi_source_resolver_variant_overrides_package( assert isinstance(pbi.source_resolver, PyPIPrebuiltResolver) +@pytest.mark.parametrize( + ("yaml_str", "expected"), + [ + ("source:\n provider: pypi-prebuilt\n", True), + ("source:\n provider: hook-prebuilt\n", True), + ("source:\n provider: pypi-sdist\n", False), + ("variants:\n cpu:\n pre_built: true\n", True), + ], +) +def test_pbi_prebuilt_uses_selected_source_or_legacy_variant( + tmp_path: pathlib.Path, yaml_str: str, expected: bool +) -> None: + """Read configured source capability before the legacy variant flag.""" + ps = PackageSettings.from_string("test-pkg", yaml_str) + settings = Settings( + settings=SettingsFile(), + package_settings=[ps], + variant="cpu", + patches_dir=tmp_path, + max_jobs=1, + ) + + assert settings.package_build_info("test-pkg").pre_built is expected + + # -- source vs legacy mutual exclusivity validation -------------------------- diff --git a/tests/test_server.py b/tests/test_server.py index d507682ec..c5cf7cec0 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -86,6 +86,44 @@ def test_update_wheel_mirror_prebuilt_wheels( assert symlink.is_file() +def test_update_wheel_mirror_prefers_fresh_prebuilt_wheel( + tmp_context: context.WorkContext, +) -> None: + """An explicit fresh wheel replaces an old mirror link with the same name.""" + wheel_name = "foo-1.0-py3-none-any.whl" + old_wheel = _create_fake_wheel(tmp_context.wheels_downloads, wheel_name) + server.update_wheel_mirror(tmp_context) + mirror_wheel = tmp_context.wheel_server_dir / "foo" / wheel_name + + fresh_wheel = _create_fake_wheel(tmp_context.wheels_prebuilt, wheel_name) + server.update_wheel_mirror(tmp_context) + assert mirror_wheel.resolve() == old_wheel + + server.update_wheel_mirror(tmp_context, preferred_prebuilt_wheel=fresh_wheel) + assert mirror_wheel.resolve() == fresh_wheel + assert old_wheel.is_file() + + +def test_update_wheel_mirror_failed_relink_keeps_old_wheel( + tmp_context: context.WorkContext, +) -> None: + """A failed atomic switch leaves the existing mirror link intact.""" + wheel_name = "foo-1.0-py3-none-any.whl" + old_wheel = _create_fake_wheel(tmp_context.wheels_downloads, wheel_name) + server.update_wheel_mirror(tmp_context) + fresh_wheel = _create_fake_wheel(tmp_context.wheels_prebuilt, wheel_name) + mirror_wheel = tmp_context.wheel_server_dir / "foo" / wheel_name + + with ( + patch("fromager.server.os.replace", side_effect=OSError("replace failed")), + pytest.raises(OSError, match="replace failed"), + ): + server.update_wheel_mirror(tmp_context, preferred_prebuilt_wheel=fresh_wheel) + + assert mirror_wheel.resolve() == old_wheel + assert not list(mirror_wheel.parent.glob(f".{wheel_name}.*")) + + def test_update_wheel_mirror_cleans_dangling_symlinks( tmp_context: context.WorkContext, ) -> None: diff --git a/tests/test_wheels.py b/tests/test_wheels.py index 91bb698e2..a4d3f2260 100644 --- a/tests/test_wheels.py +++ b/tests/test_wheels.py @@ -9,10 +9,231 @@ from packaging.tags import Tag from packaging.version import Version -from fromager import build_environment, context, downloads, wheels +from fromager import build_environment, context, downloads, packagesettings, wheels from fromager.packagesettings import SbomSettings, Settings, SettingsFile, WheelSettings +def _configure_package(ctx: context.WorkContext, yaml_str: str) -> None: + """Apply one package setting to the test context.""" + ps = packagesettings.PackageSettings.from_string("pkg", yaml_str) + ctx.settings._package_settings[ps.name] = ps + + +@pytest.mark.parametrize("provider", ["pypi-prebuilt", "hook-prebuilt"]) +def test_configured_prebuilt_download_uses_resolver_default( + tmp_context: context.WorkContext, provider: str +) -> None: + """Both profiles use their own default when no download hook exists.""" + _configure_package(tmp_context, f"source:\n provider: {provider}\n") + req = Requirement("pkg==1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + expected = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" + with ( + patch("fromager.overrides.find_override_method", return_value=None), + patch("fromager.downloads.download_wheel", return_value=expected) as default, + ): + result = wheels.download_prebuilt_wheel( + ctx=tmp_context, + req=req, + version=Version("1.0"), + download_url=url, + direct_download_directory=tmp_context.wheels_build, + ) + + assert result == expected + default.assert_called_once_with( + destination_dir=tmp_context.wheels_prebuilt, url=url + ) + + +def test_hook_prebuilt_download_stages_hook_result( + tmp_context: context.WorkContext, +) -> None: + """Use the hook's path and make its wheel visible to the local mirror.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + req = Requirement("pkg==1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + hook_path = tmp_context.work_dir / "pkg-1.0-py3-none-any.whl" + hook_path.write_bytes(b"wheel") + calls: list[str] = [] + + def hook(download_url: str) -> packagesettings.DownloadedSource: + calls.append(download_url) + return packagesettings.DownloadedSource( + path=hook_path, kind=packagesettings.DownloadKind.prebuilt_wheel + ) + + with ( + patch("fromager.overrides.find_override_method", return_value=hook), + patch("fromager.downloads.download_wheel") as default, + ): + result = wheels.download_prebuilt_wheel( + ctx=tmp_context, + req=req, + version=Version("1.0"), + download_url=url, + direct_download_directory=tmp_context.wheels_build, + ) + + assert result == tmp_context.wheels_prebuilt / hook_path.name + assert result.read_bytes() == b"wheel" + assert calls == [url] + default.assert_not_called() + + +def test_hook_prebuilt_local_mirror_url_uses_direct_download( + tmp_context: context.WorkContext, +) -> None: + """A local mirror URL does not invoke the configured download hook.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + tmp_context.wheel_server_url = "https://local.test/simple/" + req = Requirement("pkg==1.0") + url = "https://local.test/simple/pkg/pkg-1.0-py3-none-any.whl" + expected = tmp_context.wheels_build / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(wheels, "download_wheel", return_value=expected) as direct, + patch("fromager.wheels.sources.download_source") as configured, + ): + result = wheels.download_prebuilt_wheel( + ctx=tmp_context, + req=req, + version=Version("1.0"), + download_url=url, + direct_download_directory=tmp_context.wheels_build, + ) + + assert result == expected + direct.assert_called_once_with(req, url, tmp_context.wheels_build) + configured.assert_not_called() + + +def test_hook_prebuilt_new_result_replaces_staged_wheel( + tmp_context: context.WorkContext, +) -> None: + """A fresh hook result replaces an older staged wheel with the same name.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + req = Requirement("pkg==1.0") + wheel_name = "pkg-1.0-py3-none-any.whl" + hook_path = tmp_context.work_dir / wheel_name + hook_path.write_bytes(b"new wheel") + staged_path = tmp_context.wheels_prebuilt / wheel_name + staged_path.write_bytes(b"old wheel") + + with patch( + "fromager.overrides.find_override_method", return_value=lambda: hook_path + ): + result = wheels.download_prebuilt_wheel( + ctx=tmp_context, + req=req, + version=Version("1.0"), + download_url=f"https://pkg.test/{wheel_name}", + direct_download_directory=tmp_context.wheels_build, + ) + + assert result == staged_path + assert staged_path.read_bytes() == b"new wheel" + assert hook_path.read_bytes() == b"new wheel" + assert not list(tmp_context.wheels_prebuilt.glob(f".tmp.{wheel_name}.*")) + + +def test_hook_prebuilt_staging_failure_preserves_existing_wheel( + tmp_context: context.WorkContext, +) -> None: + """A failed copy leaves the old staged wheel intact and removes the temp file.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + wheel_name = "pkg-1.0-py3-none-any.whl" + hook_path = tmp_context.work_dir / wheel_name + hook_path.write_bytes(b"new wheel") + staged_path = tmp_context.wheels_prebuilt / wheel_name + staged_path.write_bytes(b"old wheel") + + with ( + patch( + "fromager.overrides.find_override_method", return_value=lambda: hook_path + ), + patch("fromager.wheels.shutil.copy2", side_effect=OSError("copy failed")), + pytest.raises(OSError, match="copy failed"), + ): + wheels.download_prebuilt_wheel( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + version=Version("1.0"), + download_url=f"https://pkg.test/{wheel_name}", + direct_download_directory=tmp_context.wheels_build, + ) + + assert staged_path.read_bytes() == b"old wheel" + assert not list(tmp_context.wheels_prebuilt.glob(f".tmp.{wheel_name}.*")) + + +def test_local_wheel_url_requires_path_boundary() -> None: + """A different index sharing a URL prefix is not the local mirror.""" + assert wheels.is_local_wheel_url( + "https://local.test/simple/pkg/pkg-1.0-py3-none-any.whl", + "https://local.test/simple/", + ) + assert not wheels.is_local_wheel_url( + "https://local.test/simple-other/pkg/pkg-1.0-py3-none-any.whl", + "https://local.test/simple/", + ) + + +def test_configured_prebuilt_rejects_other_kind( + tmp_context: context.WorkContext, +) -> None: + """An invalid hook result fails without trying a different downloader.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + req = Requirement("pkg==1.0") + + def hook() -> packagesettings.DownloadedSource: + return packagesettings.DownloadedSource( + path=tmp_context.work_dir / "pkg-1.0.tar.gz", + kind=packagesettings.DownloadKind.sdist, + ) + + with ( + patch("fromager.overrides.find_override_method", return_value=hook), + patch("fromager.downloads.download_wheel") as default, + pytest.raises(ValueError, match="unsupported kind 'sdist'"), + ): + wheels.download_prebuilt_wheel( + ctx=tmp_context, + req=req, + version=Version("1.0"), + download_url="https://pkg.test/pkg-1.0-py3-none-any.whl", + direct_download_directory=tmp_context.wheels_build, + ) + default.assert_not_called() + + +def test_hook_prebuilt_failure_does_not_fall_back( + tmp_context: context.WorkContext, +) -> None: + """Keep the hook failure visible and avoid the default wheel fetch.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + req = Requirement("pkg==1.0") + failure = OSError("hook download failed") + + def hook() -> pathlib.Path: + raise failure + + with ( + patch("fromager.overrides.find_override_method", return_value=hook), + patch("fromager.downloads.download_wheel") as default, + pytest.raises(RuntimeError, match="download_source hook failed") as exc_info, + ): + wheels.download_prebuilt_wheel( + ctx=tmp_context, + req=req, + version=Version("1.0"), + download_url="https://pkg.test/pkg-1.0-py3-none-any.whl", + direct_download_directory=tmp_context.wheels_build, + ) + + assert exc_info.value.__cause__ is failure + default.assert_not_called() + + @patch("pyproject_hooks.BuildBackendHookCaller.build_wheel") def test_default_build_wheel( mock_build_wheel: Mock, From 69bc40a922ff0f8ae59c17d9afbccb68408522f7 Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Tue, 27 Oct 2026 11:01:29 -0400 Subject: [PATCH 12/35] fix(wheels): reuse the exact wheel the local mirror serves `downloads/` and `prebuilt/` can hold different wheels with the same filename. On a local mirror hit, bootstrap reused the same-named file from the other folder and then switched the mirror to serve it. Add `server.find_mirror_wheel()` to follow the mirror link to the real file. `download_prebuilt_wheel()` returns that file for configured packages, `bg_prepare_prebuilt()` switches the mirror only after a fresh download, and `_is_wheel_built()` uses the same lookup. Co-Authored-By: Claude Closes: #1397 Signed-off-by: Lalatendu Mohanty --- docs/concepts/resolver-architecture.rst | 7 +++-- src/fromager/bootstrapper/_cache.py | 7 ++++- src/fromager/commands/build.py | 25 +++++---------- src/fromager/server.py | 26 ++++++++++++++++ src/fromager/wheels.py | 19 +++++++++--- tests/test_bootstrapper.py | 37 ++++++++++++++++++++-- tests/test_commands.py | 29 +++++++++++++++++ tests/test_server.py | 41 +++++++++++++++++++++++++ tests/test_wheels.py | 39 +++++++++++++++++++++-- 9 files changed, 198 insertions(+), 32 deletions(-) diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index 18d4cf590..6ab7dce1c 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -136,9 +136,10 @@ actual path and kind without changing its default path-only output. Without ``--source-kind``, it keeps the legacy archive behavior. Configured prebuilt resolvers select the prebuilt bootstrap path. Fresh external wheels use their ``download()`` contract and are staged in the -prebuilt wheel directory. Bootstrap, build, and download-sequence use the -direct wheel downloader for local mirror URLs, so they do not rerun the -configured download hook. Test-mode fallback also downloads directly. +prebuilt wheel directory. For local mirror URLs, bootstrap, build, and +download-sequence reuse the exact file the mirror serves, so they do not +rerun the configured download hook or pick a different same-named copy. +Test-mode fallback downloads directly. When a fresh configured wheel shares a filename with an older cached wheel, the local mirror and later build replay use the fresh staged wheel. Legacy build downloads still pass through the build wheel directory before diff --git a/src/fromager/bootstrapper/_cache.py b/src/fromager/bootstrapper/_cache.py index e241a7d63..e058da85a 100644 --- a/src/fromager/bootstrapper/_cache.py +++ b/src/fromager/bootstrapper/_cache.py @@ -258,7 +258,12 @@ def bg_prepare_prebuilt( direct_download_directory=ctx.wheels_prebuilt, ) unpack_dir = _create_unpack_dir(ctx.work_dir, req, resolved_version) - if ctx.package_build_info(req).source_resolver is not None: + # A fresh configured download can share a filename with an older wheel + # in downloads/, so point the mirror at the new file. A local mirror hit + # is already the file the mirror serves; leave the link alone. + configured = ctx.package_build_info(req).source_resolver is not None + local_hit = wheels.is_local_wheel_url(wheel_url, ctx.wheel_server_url) + if configured and not local_hit: server.update_wheel_mirror(ctx, preferred_prebuilt_wheel=wheel_filename) else: server.update_wheel_mirror(ctx) diff --git a/src/fromager/commands/build.py b/src/fromager/commands/build.py index 24ceab603..8e86328e0 100644 --- a/src/fromager/commands/build.py +++ b/src/fromager/commands/build.py @@ -538,8 +538,8 @@ def _is_wheel_built( logger.info("found candidate wheel %s", url) try: wheel_basename = downloads.extract_filename_from_url(url) - normalized_name, _, build_tag_from_name, wheel_tags = ( - parse_wheel_filename(wheel_basename) + _, _, build_tag_from_name, wheel_tags = parse_wheel_filename( + wheel_basename ) except Exception: logger.debug("could not parse candidate wheel %s", url, exc_info=True) @@ -578,21 +578,12 @@ def _is_wheel_built( wheel_filename: pathlib.Path | None = None if wheels.is_local_wheel_url(url, wkctx.wheel_server_url): logging.debug("found wheel on local server") - wheel_filename = wkctx.wheels_downloads / wheel_basename - mirror_wheel = wkctx.wheel_server_dir / normalized_name / wheel_basename - if mirror_wheel.is_symlink(): - mirror_target = mirror_wheel.resolve() - if ( - mirror_target.parent - in ( - wkctx.wheels_downloads, - wkctx.wheels_prebuilt, - ) - and mirror_target.is_file() - ): - # The mirror may point to a fresh prebuilt wheel while - # an older copy remains in downloads. - wheel_filename = mirror_target + # Use the file the mirror link points to; downloads/ and prebuilt/ + # can hold different wheels with the same name. Without a link, + # try downloads/. + wheel_filename = server.find_mirror_wheel(wkctx, wheel_basename) or ( + wkctx.wheels_downloads / wheel_basename + ) if not wheel_filename.exists(): logger.info("wheel not found in local cache, preparing to download") wheel_filename = None diff --git a/src/fromager/server.py b/src/fromager/server.py index 17e03ae8f..d8a61c9e8 100644 --- a/src/fromager/server.py +++ b/src/fromager/server.py @@ -113,6 +113,32 @@ def update_wheel_mirror( temporary_link.unlink(missing_ok=True) +def find_mirror_wheel( + ctx: context.WorkContext, wheel_filename: str +) -> pathlib.Path | None: + """Return the file the local mirror serves for a wheel filename. + + The mirror (``wheel_server_dir``) holds ``/`` symlinks into + the downloads or prebuilt directory. Both can hold different wheels with + the same filename, so follow the link instead of guessing the folder. + + Returns ``None`` when the mirror has no link to an existing wheel in + the downloads or prebuilt directory. + + .. versionadded:: 0.102.0 + """ + (normalized_name, _, _, _) = parse_wheel_filename(wheel_filename) + mirror_wheel = ctx.wheel_server_dir / normalized_name / wheel_filename + if not mirror_wheel.is_symlink(): + return None + target = mirror_wheel.resolve() + if target.parent not in (ctx.wheels_downloads, ctx.wheels_prebuilt): + return None + if not target.is_file(): + return None + return target + + class SimpleHTMLIndex: """Simple HTML Repository API (1.0) diff --git a/src/fromager/wheels.py b/src/fromager/wheels.py index b1a36cb29..2613f8aa7 100644 --- a/src/fromager/wheels.py +++ b/src/fromager/wheels.py @@ -32,6 +32,7 @@ requirements_file, resolver, sbom, + server, sources, ) from .pkgmetadata.pep376 import verbatim_dist_name @@ -541,8 +542,9 @@ def download_prebuilt_wheel( ) -> pathlib.Path: """Download a prebuilt wheel via its resolver or the direct wheel path. - Configured external wheels use the resolver download contract. Legacy - wheels and local-mirror URLs use the direct wheel downloader. + Configured external wheels use the resolver download contract. For a + local-mirror URL, configured wheels reuse the exact file the mirror + serves. Legacy wheels use the direct wheel downloader. .. versionadded:: 0.102.0 """ @@ -552,9 +554,16 @@ def download_prebuilt_wheel( f"{req}: source resolver {source_resolver.provider!r} " "does not resolve prebuilt wheels" ) - if source_resolver is None or is_local_wheel_url( - download_url, ctx.wheel_server_url - ): + if source_resolver is None: + return download_wheel(req, download_url, direct_download_directory) + if is_local_wheel_url(download_url, ctx.wheel_server_url): + # Reuse the exact file the local mirror serves. Fall back to the direct + # downloader when the local mirror has no link for this file. + mirror_wheel = server.find_mirror_wheel( + ctx, downloads.extract_filename_from_url(download_url) + ) + if mirror_wheel is not None: + return mirror_wheel return download_wheel(req, download_url, direct_download_directory) downloaded: packagesettings.DownloadedSource = sources.download_source( diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index fc2cda9d3..bb125cd91 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -13,7 +13,7 @@ from packaging.version import Version from resolvelib.resolvers import ResolverException -from fromager import bootstrapper, log, sources, wheels +from fromager import bootstrapper, log, server, sources, wheels from fromager.bootstrapper._build import Build from fromager.bootstrapper._cache import ( _download_wheel_from_cache, @@ -107,7 +107,7 @@ def test_bootstrap_prebuilt_uses_configured_download( def test_bootstrap_configured_prebuilt_reuses_local_wheel( tmp_context: WorkContext, ) -> None: - """A local mirror URL bypasses the configured download hook.""" + """A local mirror URL bypasses the download hook and the mirror switch.""" _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") req = Requirement("pkg==1.0") version = Version("1.0") @@ -126,7 +126,38 @@ def test_bootstrap_configured_prebuilt_reuses_local_wheel( assert result.wheel_filename == wheel cached.assert_called_once_with(req, url, tmp_context.wheels_prebuilt) configured.assert_not_called() - mirror.assert_called_once_with(tmp_context, preferred_prebuilt_wheel=wheel) + mirror.assert_called_once_with(tmp_context) + + +def test_bootstrap_configured_prebuilt_uses_mirrored_wheel( + tmp_context: WorkContext, +) -> None: + """A local mirror hit uses and keeps serving the exact mirrored file.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + req = Requirement("pkg==1.0") + wheel_name = "pkg-1.0-py3-none-any.whl" + served_wheel = tmp_context.wheels_downloads / wheel_name + served_wheel.write_bytes(b"served wheel") + server.update_wheel_mirror(tmp_context) + other_wheel = tmp_context.wheels_prebuilt / wheel_name + other_wheel.write_bytes(b"other wheel") + tmp_context.wheel_server_url = "https://local.test/simple/" + url = f"https://local.test/simple/pkg/{wheel_name}" + + with ( + patch.object(wheels, "download_wheel", wraps=wheels.download_wheel) as direct, + patch.object(sources, "download_source") as configured, + ): + result = bg_prepare_prebuilt( + tmp_context, req, RequirementType.TOP_LEVEL, Version("1.0"), url + ) + + mirror_wheel = tmp_context.wheel_server_dir / "pkg" / wheel_name + assert result.wheel_filename == served_wheel + assert mirror_wheel.resolve() == served_wheel + assert mirror_wheel.read_bytes() == b"served wheel" + direct.assert_not_called() + configured.assert_not_called() @pytest.mark.parametrize("local_cache", [False, True]) diff --git a/tests/test_commands.py b/tests/test_commands.py index 61b752e60..a268cc9a0 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -80,6 +80,35 @@ def test_build_replay_local_mirror_url_skips_download_hook( hooks.assert_called_once() +def test_forced_build_local_mirror_url_uses_mirrored_wheel( + tmp_context: WorkContext, +) -> None: + """A forced build uses and keeps serving the exact mirrored file.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + req = Requirement("pkg==1.0") + wheel_name = "pkg-1.0-py3-none-any.whl" + served_wheel = tmp_context.wheels_downloads / wheel_name + served_wheel.write_bytes(b"served wheel") + server.update_wheel_mirror(tmp_context) + (tmp_context.wheels_prebuilt / wheel_name).write_bytes(b"other wheel") + tmp_context.wheel_server_url = "https://local.test/simple/" + url = f"https://local.test/simple/pkg/{wheel_name}" + + with ( + patch.object(wheels, "download_wheel") as direct, + patch("fromager.wheels.sources.download_source") as configured, + patch.object(build.hooks, "run_prebuilt_wheel_hooks"), + ): + result = build._build(tmp_context, Version("1.0"), req, url, True, None) + + mirror_wheel = tmp_context.wheel_server_dir / "pkg" / wheel_name + assert result.wheel_filename == served_wheel + assert mirror_wheel.resolve() == served_wheel + assert mirror_wheel.read_bytes() == b"served wheel" + direct.assert_not_called() + configured.assert_not_called() + + def test_fresh_configured_prebuilt_replaces_old_mirror_entry( tmp_context: WorkContext, ) -> None: diff --git a/tests/test_server.py b/tests/test_server.py index c5cf7cec0..8362f2587 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -233,3 +233,44 @@ def test_start_wheel_server_starts_local( mock_mirror.assert_called_once_with(tmp_context) mock_run.assert_called_once_with(tmp_context) + + +@pytest.mark.parametrize( + ("mirror_target", "expected"), + [ + ("downloads", "downloads"), + ("prebuilt", "prebuilt"), + (None, None), + ("outside", None), + ("dangling", None), + ], +) +def test_find_mirror_wheel( + tmp_context: context.WorkContext, + tmp_path: pathlib.Path, + mirror_target: str | None, + expected: str | None, +) -> None: + """Only mirror links to an existing downloads or prebuilt wheel are returned.""" + wheel_name = "pkg-1.0-py3-none-any.whl" + directories = { + "downloads": tmp_context.wheels_downloads, + "prebuilt": tmp_context.wheels_prebuilt, + "outside": tmp_path / "outside", + "dangling": tmp_context.wheels_downloads, + } + if mirror_target is not None: + target = directories[mirror_target] / wheel_name + target.parent.mkdir(parents=True, exist_ok=True) + if mirror_target != "dangling": + target.write_bytes(b"wheel") + mirror_wheel = tmp_context.wheel_server_dir / "pkg" / wheel_name + mirror_wheel.parent.mkdir(parents=True, exist_ok=True) + mirror_wheel.symlink_to(target) + + result = server.find_mirror_wheel(tmp_context, wheel_name) + + if expected is None: + assert result is None + else: + assert result == directories[expected] / wheel_name diff --git a/tests/test_wheels.py b/tests/test_wheels.py index a4d3f2260..6b5306502 100644 --- a/tests/test_wheels.py +++ b/tests/test_wheels.py @@ -9,7 +9,14 @@ from packaging.tags import Tag from packaging.version import Version -from fromager import build_environment, context, downloads, packagesettings, wheels +from fromager import ( + build_environment, + context, + downloads, + packagesettings, + server, + wheels, +) from fromager.packagesettings import SbomSettings, Settings, SettingsFile, WheelSettings @@ -81,11 +88,12 @@ def hook(download_url: str) -> packagesettings.DownloadedSource: default.assert_not_called() -def test_hook_prebuilt_local_mirror_url_uses_direct_download( +def test_hook_prebuilt_local_url_without_mirror_link_uses_direct_download( tmp_context: context.WorkContext, ) -> None: - """A local mirror URL does not invoke the configured download hook.""" + """Without a mirror link, a local URL uses the direct downloader, not the hook.""" _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + # No mirror symlink exists, so find_mirror_wheel() returns None. tmp_context.wheel_server_url = "https://local.test/simple/" req = Requirement("pkg==1.0") url = "https://local.test/simple/pkg/pkg-1.0-py3-none-any.whl" @@ -178,6 +186,31 @@ def test_local_wheel_url_requires_path_boundary() -> None: ) +def test_configured_prebuilt_local_url_reuses_mirrored_wheel( + tmp_context: context.WorkContext, +) -> None: + """A local mirror URL returns the served file, not a same-named copy.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + wheel_name = "pkg-1.0-py3-none-any.whl" + served_wheel = tmp_context.wheels_downloads / wheel_name + served_wheel.write_bytes(b"served wheel") + server.update_wheel_mirror(tmp_context) + (tmp_context.wheels_prebuilt / wheel_name).write_bytes(b"other wheel") + tmp_context.wheel_server_url = "https://local.test/simple/" + + with patch.object(wheels, "download_wheel") as direct: + result = wheels.download_prebuilt_wheel( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + version=Version("1.0"), + download_url=f"https://local.test/simple/pkg/{wheel_name}", + direct_download_directory=tmp_context.wheels_prebuilt, + ) + + assert result == served_wheel + direct.assert_not_called() + + def test_configured_prebuilt_rejects_other_kind( tmp_context: context.WorkContext, ) -> None: From 9c07619fcc8f6323d09bab9cb215c505b5139e73 Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Wed, 28 Oct 2026 13:36:40 -0400 Subject: [PATCH 13/35] fix(resolver): let hook profiles set their own index URL Hook profiles now have an `index_url` setting. `hook-prebuilt` must set it. `hook-sdist` uses PyPI if it is not set. Fromager passes this URL to the `get_resolver_provider` hook, so existing hooks keep working. Profiles in `source:` now always use their own index. If a command passes a different URL, Fromager ignores it and logs a warning. Co-Authored-By: Claude Closes: #1373 Signed-off-by: Lalatendu Mohanty --- docs/concepts/resolver-architecture.rst | 8 +++ docs/proposals/new-resolver-config.md | 13 +++-- docs/reference/hooks.rst | 13 ++++- src/fromager/packagesettings/_resolver.py | 44 +++++---------- src/fromager/sources.py | 17 ++++-- tests/test_bootstrap_requirement_resolver.py | 3 +- tests/test_bootstrapper.py | 16 ++++-- tests/test_commands.py | 22 +++++--- tests/test_packagesettings.py | 5 +- tests/test_packagesettings_resolver.py | 59 +++++++++++++++----- tests/test_sources.py | 44 +++++++++++++-- tests/test_wheels.py | 24 +++++--- 12 files changed, 189 insertions(+), 79 deletions(-) diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index 6ab7dce1c..0098c0caa 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -102,12 +102,20 @@ CLI commands interact with providers through a common to PyPI. Their ``download()`` methods invoke the package's ``download_source`` hook when present and return a validated ``DownloadedSource``. +.. versionchanged:: 0.102.0 + Configured source resolvers own their index URL and ignore caller URLs. + Hook profiles pass their ``index_url`` to the legacy + ``get_resolver_provider`` hook as ``sdist_server_url``. ``hook-sdist`` + defaults to PyPI, while ``hook-prebuilt`` requires an explicit index. + Per-package settings in YAML can select which provider to use and configure its parameters (index URL, tag pattern, etc.). When a package has a ``source:`` resolver configured, that resolver creates the provider. The ``hook-sdist`` and ``hook-prebuilt`` profiles delegate provider creation to the package's required ``get_resolver_provider`` hook and validate that it returns a :class:`~fromager.resolver.BaseProvider`. +The hook may ignore or replace the profile's index URL, but the profile's +value is stable across Fromager commands. Otherwise, override plugins can replace the provider for a specific package via the ``get_resolver_provider`` hook, after which the legacy resolver settings are used. diff --git a/docs/proposals/new-resolver-config.md b/docs/proposals/new-resolver-config.md index f82675df8..72485fe18 100644 --- a/docs/proposals/new-resolver-config.md +++ b/docs/proposals/new-resolver-config.md @@ -145,11 +145,14 @@ variants: - The `hook-sdist` profile calls the `resolver_provider` and `download_source` [hooks](../reference/hooks.rst). The `resolver_provider` hook receives the - `req` argument for access to the current requirement. Hooks can return - any source artifact (sdist, tarball, or git checkout). + `req` argument for access to the current requirement and + `sdist_server_url=str(index_url)`. `index_url` defaults to PyPI. Hooks can + return any source artifact (sdist, tarball, or git checkout). - The `hook-prebuilt` profile is similar to `hook-sdist` but the downloaded - artifact must be a pre-built wheel. + artifact must be a pre-built wheel. Its required `index_url` replaces the + legacy variant `wheel_server_url`; the hook receives it as + `sdist_server_url` and may use, ignore, or replace it. - The `not-available` profile raises an error. It can be used to block a package and only enable it for a single variant. @@ -224,11 +227,11 @@ def matcher_factory_func( - `download_source.destination_filename` is not needed. All sdists use standard `{dist_name}-{version}.tar.gz` file name - `resolver_dist.sdist_server_url` is replaced by `index_url` parameter. - All `pypi-*` profile support a custom index. + All `pypi-*` and `hook-*` profiles support a custom index. - `git_options.submodules` is not needed. Like pip, Fromager will always clone all submodules. - variant settings `wheel_server_url` and `pre_build` are replaced by - `pypi-prebuilt` profile + `pypi-prebuilt` or `hook-prebuilt` profiles ### Migration diff --git a/docs/reference/hooks.rst b/docs/reference/hooks.rst index 5479a9937..261b5bca2 100644 --- a/docs/reference/hooks.rst +++ b/docs/reference/hooks.rst @@ -138,7 +138,7 @@ Resolver hooks The arguments are the ``WorkContext``, the ``Requirement`` being evaluated, a boolean indicating whether source distributions should be included, a boolean indicating whether built wheels should be - included, and the URL for the sdist server. The hook also receives + included, and the URL for the package index. The hook also receives ``req_type`` and ``ignore_platform`` when those parameters are supported by its signature. @@ -158,6 +158,17 @@ Resolver hooks :class:`~fromager.resolver.BaseProvider` is an error. Hook profiles never fall back to the default PyPI provider. + For a configured hook profile, ``sdist_server_url`` is the profile's + ``index_url``. ``hook-sdist`` defaults that field to PyPI; + ``hook-prebuilt`` requires it so a pass-through hook cannot silently + resolve public PyPI wheels. The hook may use, ignore, or replace the URL. + Command-line and caller URLs are ignored for configured ``source:`` + profiles; Fromager logs a warning when it ignores a URL other than PyPI. + + Other legacy calls to this hook, such as prebuilt-wheel fallback and + diagnostic resolution, continue to pass their existing wheel-server or + PyPI URLs. They are not configured source resolution. + The return value must be an instance of :class:`~fromager.resolver.BaseProvider`. diff --git a/src/fromager/packagesettings/_resolver.py b/src/fromager/packagesettings/_resolver.py index 363f536f8..b07f3432d 100644 --- a/src/fromager/packagesettings/_resolver.py +++ b/src/fromager/packagesettings/_resolver.py @@ -25,6 +25,7 @@ logger = logging.getLogger(__name__) _VERSION_QUOTED = "%7Bversion%7D" +_PYPI_INDEX_URL = pydantic.HttpUrl("https://pypi.org/simple/") class BuildSDist(enum.StrEnum): @@ -87,8 +88,6 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, - *, - sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.BaseProvider: """Return a resolver provider for the given requirement.""" raise NotImplementedError @@ -195,7 +194,7 @@ class AbstractPyPIResolver(AbstractResolver, CooldownMixin): """Abstract base class for PyPI resolvers""" index_url: pydantic.HttpUrl = pydantic.Field( - default=pydantic.HttpUrl("https://pypi.org/simple/"), + default=_PYPI_INDEX_URL, description="Python Package Index URL", ) @@ -228,8 +227,6 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, - *, - sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.PyPIProvider: return resolver.PyPIProvider( include_sdists=True, @@ -281,8 +278,6 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, - *, - sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.PyPIProvider: return resolver.PyPIProvider( include_sdists=False, @@ -356,8 +351,6 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, - *, - sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.PyPIProvider: return resolver.PyPIProvider( include_sdists=True, @@ -440,8 +433,6 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, - *, - sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.PyPIProvider: download_url = f"git+{self.clone_url}@refs/tags/{self.tag}" return resolver.PyPIProvider( @@ -609,8 +600,6 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, - *, - sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.GitHubTagProvider: return self._github_provider( ctx=ctx, @@ -652,8 +641,6 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, - *, - sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.GitHubTagProvider: return self._github_provider( ctx=ctx, @@ -693,8 +680,6 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, - *, - sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.GitLabTagProvider: return self._gitlab_provider( ctx=ctx, @@ -736,8 +721,6 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, - *, - sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.GitLabTagProvider: return self._gitlab_provider( ctx=ctx, @@ -766,8 +749,6 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, - *, - sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.BaseProvider: raise ValueError(f"package {req.name} is not available") @@ -835,8 +816,6 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, - *, - sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.BaseProvider: raise NotImplementedError @@ -920,6 +899,7 @@ class HookSDistResolver(AbstractHookResolver): Example:: provider: hook-sdist + index_url: https://pypi.test/simple build_sdist: tarball """ @@ -927,6 +907,11 @@ class HookSDistResolver(AbstractHookResolver): build_sdist: BuildSDist """Source distribution build method.""" + index_url: pydantic.HttpUrl = pydantic.Field( + default=_PYPI_INDEX_URL, + description="Python Package Index URL", + ) + download_kinds: typing.ClassVar[DownloadKindSet] = frozenset( {DownloadKind.sdist, DownloadKind.tarball, DownloadKind.git_checkout} ) @@ -936,15 +921,13 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, - *, - sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.BaseProvider: """Return a source provider from the package resolver hook.""" return self._resolver_provider_from_hook( ctx=ctx, req=req, req_type=req_type, - sdist_server_url=sdist_server_url, + sdist_server_url=str(self.index_url), include_sdists=True, include_wheels=False, ignore_platform=False, @@ -972,11 +955,16 @@ class HookPrebuiltResolver(AbstractHookResolver): Example:: provider: hook-prebuilt + index_url: https://wheels.test/simple """ provider: typing.Literal["hook-prebuilt"] build_sdist: None = None + index_url: pydantic.HttpUrl = pydantic.Field( + description="Python Package Index URL for pre-built wheels", + ) + download_kinds: typing.ClassVar[DownloadKindSet] = frozenset( {DownloadKind.prebuilt_wheel} ) @@ -987,15 +975,13 @@ def resolver_provider( ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, - *, - sdist_server_url: str = resolver.PYPI_SERVER_URL, ) -> resolver.BaseProvider: """Return a pre-built wheel provider from the package resolver hook.""" return self._resolver_provider_from_hook( ctx=ctx, req=req, req_type=req_type, - sdist_server_url=sdist_server_url, + sdist_server_url=str(self.index_url), include_sdists=False, include_wheels=True, ignore_platform=False, diff --git a/src/fromager/sources.py b/src/fromager/sources.py index ec36a4782..82b0fe384 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -131,21 +131,27 @@ def get_source_provider( """Create a provider for resolving source distributions. Returns a provider configured according to the package's resolver settings - (sdist/wheel inclusion, platform matching, server URL override). + (sdist/wheel inclusion and platform matching). Packages with a ``source:`` resolver use that resolver's provider - configuration. Packages without one retain the legacy override-hook and - ``resolver_dist`` behavior. + configuration and ignore the caller's URL, with a warning unless it is the + default PyPI URL. Packages without one retain the + legacy override-hook, ``resolver_dist``, and caller URL behavior. """ pbi = ctx.package_build_info(req) source_resolver = pbi.source_resolver if source_resolver is not None: + if sdist_server_url.rstrip("/") != resolver.PYPI_SERVER_URL: + logger.warning( + "ignoring index URL %s; the %r source resolver uses its own index", + sdist_server_url, + source_resolver.provider, + ) provider = source_resolver.resolver_provider( ctx, req, req_type, - sdist_server_url=sdist_server_url, ) if req_type == RequirementType.TOP_LEVEL and resolver._has_equality_pin(req): provider.cooldown = None @@ -191,7 +197,8 @@ def resolve_source( """ constraint = ctx.constraints.get_constraint(req.name) logger.debug( - f"resolving requirement {req} using {sdist_server_url} with constraint {constraint}" + f"resolving requirement {req} with constraint {constraint}; " + f"legacy index URL: {sdist_server_url}" ) try: diff --git a/tests/test_bootstrap_requirement_resolver.py b/tests/test_bootstrap_requirement_resolver.py index f6fa5b570..a1bc00251 100644 --- a/tests/test_bootstrap_requirement_resolver.py +++ b/tests/test_bootstrap_requirement_resolver.py @@ -476,7 +476,8 @@ def test_configured_hook_prebuilt_uses_hook_provider( req = Requirement("pkg==1.0") tmp_context.settings._package_settings[canonicalize_name("pkg")] = ( packagesettings.PackageSettings.from_string( - "pkg", "source:\n provider: hook-prebuilt\n" + "pkg", + "source:\n provider: hook-prebuilt\n index_url: https://index.test/simple\n", ) ) provider = resolver.PyPIProvider(sdist_server_url="https://hook.test/simple/") diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index bb125cd91..37bd66c50 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -54,6 +54,14 @@ def _configure_package(ctx: WorkContext, yaml_str: str) -> None: ctx.settings._package_settings[ps.name] = ps +def _source_yaml(provider: str) -> str: + """Return package settings that select a configured source provider.""" + yaml_str = f"source:\n provider: {provider}\n" + if provider == "hook-prebuilt": + yaml_str += " index_url: https://index.test/simple\n" + return yaml_str + + def test_phase_subclass_without_phase_attribute_raises() -> None: """Concrete Phase subclass missing 'phase' raises TypeError at class definition.""" with pytest.raises(TypeError, match="must define the 'phase' class attribute"): @@ -80,7 +88,7 @@ def test_bootstrap_prebuilt_uses_configured_download( tmp_context: WorkContext, provider: str ) -> None: """Bootstrap consumes the configured wheel path and updates its mirror.""" - _configure_package(tmp_context, f"source:\n provider: {provider}\n") + _configure_package(tmp_context, _source_yaml(provider)) req = Requirement("pkg==1.0") version = Version("1.0") url = "https://pkg.test/pkg-1.0-py3-none-any.whl" @@ -108,7 +116,7 @@ def test_bootstrap_configured_prebuilt_reuses_local_wheel( tmp_context: WorkContext, ) -> None: """A local mirror URL bypasses the download hook and the mirror switch.""" - _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + _configure_package(tmp_context, _source_yaml("hook-prebuilt")) req = Requirement("pkg==1.0") version = Version("1.0") tmp_context.wheel_server_url = "https://local.test/simple/" @@ -133,7 +141,7 @@ def test_bootstrap_configured_prebuilt_uses_mirrored_wheel( tmp_context: WorkContext, ) -> None: """A local mirror hit uses and keeps serving the exact mirrored file.""" - _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + _configure_package(tmp_context, _source_yaml("hook-prebuilt")) req = Requirement("pkg==1.0") wheel_name = "pkg-1.0-py3-none-any.whl" served_wheel = tmp_context.wheels_downloads / wheel_name @@ -218,7 +226,7 @@ def test_bootstrap_records_configured_prebuilt_in_graph_and_order( tmp_context: WorkContext, provider: str ) -> None: """The selected source makes bootstrap skip build phases and persist its flag.""" - _configure_package(tmp_context, f"source:\n provider: {provider}\n") + _configure_package(tmp_context, _source_yaml(provider)) bt = bootstrapper.Bootstrapper(tmp_context) req = Requirement("pkg==1.0") version = Version("1.0") diff --git a/tests/test_commands.py b/tests/test_commands.py index a268cc9a0..a29c2790b 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -23,12 +23,20 @@ def _configure_package(ctx: WorkContext, yaml_str: str) -> None: ctx.settings._package_settings[ps.name] = ps +def _source_yaml(provider: str) -> str: + """Return package settings that select a configured source provider.""" + yaml_str = f"source:\n provider: {provider}\n" + if provider == "hook-prebuilt": + yaml_str += " index_url: https://index.test/simple\n" + return yaml_str + + @pytest.mark.parametrize("provider", ["pypi-prebuilt", "hook-prebuilt"]) def test_build_order_replay_uses_configured_download( tmp_context: WorkContext, provider: str ) -> None: """A later build run selects the configured wheel download contract.""" - _configure_package(tmp_context, f"source:\n provider: {provider}\n") + _configure_package(tmp_context, _source_yaml(provider)) req = Requirement("pkg==1.0") version = Version("1.0") url = "https://pkg.test/pkg-1.0-py3-none-any.whl" @@ -60,7 +68,7 @@ def test_build_replay_local_mirror_url_skips_download_hook( tmp_context: WorkContext, ) -> None: """Build replay downloads a local mirror URL directly and runs wheel hooks.""" - _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + _configure_package(tmp_context, _source_yaml("hook-prebuilt")) tmp_context.wheel_server_url = "https://local.test/simple/" req = Requirement("pkg==1.0") url = "https://local.test/simple/pkg/pkg-1.0-py3-none-any.whl" @@ -84,7 +92,7 @@ def test_forced_build_local_mirror_url_uses_mirrored_wheel( tmp_context: WorkContext, ) -> None: """A forced build uses and keeps serving the exact mirrored file.""" - _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + _configure_package(tmp_context, _source_yaml("hook-prebuilt")) req = Requirement("pkg==1.0") wheel_name = "pkg-1.0-py3-none-any.whl" served_wheel = tmp_context.wheels_downloads / wheel_name @@ -113,7 +121,7 @@ def test_fresh_configured_prebuilt_replaces_old_mirror_entry( tmp_context: WorkContext, ) -> None: """A forced hook download makes its new wheel visible in the mirror.""" - _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + _configure_package(tmp_context, _source_yaml("hook-prebuilt")) req = Requirement("pkg==1.0") wheel_name = "pkg-1.0-py3-none-any.whl" old_wheel = tmp_context.wheels_downloads / wheel_name @@ -159,7 +167,7 @@ def test_fresh_configured_prebuilt_replaces_old_mirror_entry( def test_configured_prebuilt_reuses_cached_wheel(tmp_context: WorkContext) -> None: """A cache hit avoids a fresh resolver download while retaining hooks.""" - _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + _configure_package(tmp_context, _source_yaml("hook-prebuilt")) req = Requirement("pkg==1.0") version = Version("1.0") url = "https://pkg.test/pkg-1.0-py3-none-any.whl" @@ -182,7 +190,7 @@ def test_download_sequence_replays_configured_prebuilt( cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path ) -> None: """Download sequence uses the saved URL with the selected source resolver.""" - _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + _configure_package(tmp_context, _source_yaml("hook-prebuilt")) url = "https://pkg.test/pkg-1.0-py3-none-any.whl" build_order = tmp_path / "build-order.json" build_order.write_text( @@ -221,7 +229,7 @@ def test_download_sequence_local_mirror_url_skips_download_hook( cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path ) -> None: """Download replay fetches a local mirror wheel without invoking its hook.""" - _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + _configure_package(tmp_context, _source_yaml("hook-prebuilt")) tmp_context.wheel_server_url = "https://local.test/simple/" url = "https://local.test/simple/pkg/pkg-1.0-py3-none-any.whl" build_order = tmp_path / "build-order.json" diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index ce18db756..bfd14c726 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -1259,7 +1259,10 @@ def test_pbi_source_resolver_variant_overrides_package( ("yaml_str", "expected"), [ ("source:\n provider: pypi-prebuilt\n", True), - ("source:\n provider: hook-prebuilt\n", True), + ( + "source:\n provider: hook-prebuilt\n index_url: https://pypi.test/simple\n", + True, + ), ("source:\n provider: pypi-sdist\n", False), ("variants:\n cpu:\n pre_built: true\n", True), ], diff --git a/tests/test_packagesettings_resolver.py b/tests/test_packagesettings_resolver.py index 749285231..e48e0d9cb 100644 --- a/tests/test_packagesettings_resolver.py +++ b/tests/test_packagesettings_resolver.py @@ -822,6 +822,7 @@ class TestHookSDistResolver: YAML = """\ source: provider: hook-sdist + index_url: https://index.test/simple build_sdist: tarball """ @@ -829,6 +830,7 @@ def test_parse(self) -> None: r = _parse(self.YAML) assert isinstance(r, HookSDistResolver) assert r.provider == "hook-sdist" + assert str(r.index_url) == "https://index.test/simple" assert r.build_sdist is BuildSDist.tarball assert r.supports_override_hooks is True assert r.resolves_prebuilt_wheel is False @@ -850,6 +852,13 @@ def test_build_sdist_method_required(self) -> None: with pytest.raises(pydantic.ValidationError, match="build_sdist"): _parse("source:\n provider: hook-sdist\n") + def test_default_index_url(self) -> None: + """Source hook profiles may use the standard PyPI index.""" + r = _parse("source:\n provider: hook-sdist\n build_sdist: tarball\n") + + assert isinstance(r, HookSDistResolver) + assert str(r.index_url) == "https://pypi.org/simple/" + @mock.patch("fromager.packagesettings._resolver.overrides.find_override_method") def test_resolver_provider( self, @@ -882,12 +891,7 @@ def hook( find_override_method.return_value = hook - result = r.resolver_provider( - tmp_context, - _REQ, - _REQ_TYPE, - sdist_server_url="https://index.test/simple", - ) + result = r.resolver_provider(tmp_context, _REQ, _REQ_TYPE) assert result is provider assert captured == { @@ -905,16 +909,23 @@ class TestHookPrebuiltResolver: YAML = """\ source: provider: hook-prebuilt + index_url: https://index.test/simple """ def test_parse(self) -> None: r = _parse(self.YAML) assert isinstance(r, HookPrebuiltResolver) assert r.provider == "hook-prebuilt" + assert str(r.index_url) == "https://index.test/simple" assert r.supports_override_hooks is True assert r.resolves_prebuilt_wheel is True assert r.download_kinds == frozenset({DownloadKind.prebuilt_wheel}) + def test_index_url_required(self) -> None: + """Require an explicit index to avoid silently using public PyPI.""" + with pytest.raises(pydantic.ValidationError, match="index_url"): + _parse("source:\n provider: hook-prebuilt\n") + @mock.patch("fromager.packagesettings._resolver.overrides.find_override_method") def test_resolver_provider( self, @@ -923,6 +934,7 @@ def test_resolver_provider( ) -> None: r = _parse(self.YAML) provider = resolver.GenericProvider(version_source=lambda identifier: []) + captured: dict[str, object] = {} def hook( ctx: WorkContext, @@ -933,18 +945,31 @@ def hook( req_type: RequirementType | None = None, ignore_platform: bool = False, ) -> resolver.BaseProvider: + captured.update( + ctx=ctx, + req=req, + include_sdists=include_sdists, + include_wheels=include_wheels, + sdist_server_url=sdist_server_url, + req_type=req_type, + ignore_platform=ignore_platform, + ) return provider find_override_method.return_value = hook - result = r.resolver_provider( - tmp_context, - _REQ, - _REQ_TYPE, - sdist_server_url="https://index.test/simple", - ) + result = r.resolver_provider(tmp_context, _REQ, _REQ_TYPE) assert result is provider + assert captured == { + "ctx": tmp_context, + "req": _REQ, + "include_sdists": False, + "include_wheels": True, + "sdist_server_url": "https://index.test/simple", + "req_type": _REQ_TYPE, + "ignore_platform": False, + } find_override_method.assert_called_once_with( _REQ.name, "get_resolver_provider", @@ -1248,7 +1273,10 @@ def test_hook_prebuilt_download_fallback_accepts_valid_wheel( f"{wheel.dist_info_path}/METADATA", "Metadata-Version: 2.1\nName: test-pkg\nVersion: 1.2.3\n", ) - r = HookPrebuiltResolver(provider="hook-prebuilt") + r = HookPrebuiltResolver( + provider="hook-prebuilt", + index_url=pydantic.HttpUrl("https://index.test/simple"), + ) with mock.patch("fromager.overrides.find_override_method", return_value=None): result = r.download(tmp_context, _REQ, _CANDIDATE_WHEEL) @@ -1327,7 +1355,10 @@ def hook( return object() find_override_method.return_value = hook - resolver_model = HookPrebuiltResolver(provider="hook-prebuilt") + resolver_model = HookPrebuiltResolver( + provider="hook-prebuilt", + index_url=pydantic.HttpUrl("https://index.test/simple"), + ) with pytest.raises( TypeError, diff --git a/tests/test_sources.py b/tests/test_sources.py index 1059c8156..19e7949bf 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -46,7 +46,6 @@ def test_get_source_provider_uses_configured_source_resolver( tmp_context, req, None, - sdist_server_url="https://caller.test/simple", ) find_and_invoke.assert_not_called() @@ -79,12 +78,48 @@ def test_get_source_provider_uses_pypi_sdist_source_resolver( assert provider.sdist_server_url == "https://pypi.test/simple" +@pytest.mark.parametrize( + ("sdist_server_url", "warns"), + [ + ("https://caller.test/simple", True), + (resolver.PYPI_SERVER_URL, False), + (f"{resolver.PYPI_SERVER_URL}/", False), + ], +) +def test_get_source_provider_warns_about_ignored_url( + tmp_context: context.WorkContext, + caplog: pytest.LogCaptureFixture, + sdist_server_url: str, + warns: bool, +) -> None: + """Warn when a configured source resolver ignores a non-PyPI caller URL.""" + req = Requirement("test-pkg") + source_resolver = packagesettings.PyPISDistResolver( + provider="pypi-sdist", + index_url=pydantic.HttpUrl("https://pypi.test/simple"), + ) + + with patch.object( + packagesettings.PackageBuildInfo, + "source_resolver", + new_callable=PropertyMock, + return_value=source_resolver, + ): + sources.get_source_provider( + ctx=tmp_context, + req=req, + sdist_server_url=sdist_server_url, + ) + + assert ("ignoring index URL" in caplog.text) is warns + + @patch("fromager.packagesettings._resolver.overrides.find_override_method") -def test_get_source_provider_hook_forwards_url_and_applies_cooldown( +def test_get_source_provider_hook_uses_profile_url_and_applies_cooldown( find_override_method: Mock, tmp_context: context.WorkContext, ) -> None: - """Hook providers receive the runtime URL and centralized cooldown.""" + """Hook providers receive the profile URL and centralized cooldown.""" req = Requirement("test-pkg") initial_cooldown = Cooldown(min_age=datetime.timedelta(days=1)) provider = resolver.GenericProvider( @@ -118,6 +153,7 @@ def hook( source_resolver = packagesettings.HookSDistResolver( provider="hook-sdist", build_sdist=packagesettings.BuildSDist.tarball, + index_url=pydantic.HttpUrl("https://profile.test/simple"), min_release_age=14, ) @@ -140,7 +176,7 @@ def hook( "req": req, "include_sdists": True, "include_wheels": False, - "sdist_server_url": "https://caller.test/simple", + "sdist_server_url": "https://profile.test/simple", "req_type": RequirementType.INSTALL, "ignore_platform": False, } diff --git a/tests/test_wheels.py b/tests/test_wheels.py index 6b5306502..8a9cd747b 100644 --- a/tests/test_wheels.py +++ b/tests/test_wheels.py @@ -26,12 +26,20 @@ def _configure_package(ctx: context.WorkContext, yaml_str: str) -> None: ctx.settings._package_settings[ps.name] = ps +def _source_yaml(provider: str) -> str: + """Return package settings that select a configured source provider.""" + yaml_str = f"source:\n provider: {provider}\n" + if provider == "hook-prebuilt": + yaml_str += " index_url: https://index.test/simple\n" + return yaml_str + + @pytest.mark.parametrize("provider", ["pypi-prebuilt", "hook-prebuilt"]) def test_configured_prebuilt_download_uses_resolver_default( tmp_context: context.WorkContext, provider: str ) -> None: """Both profiles use their own default when no download hook exists.""" - _configure_package(tmp_context, f"source:\n provider: {provider}\n") + _configure_package(tmp_context, _source_yaml(provider)) req = Requirement("pkg==1.0") url = "https://pkg.test/pkg-1.0-py3-none-any.whl" expected = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" @@ -57,7 +65,7 @@ def test_hook_prebuilt_download_stages_hook_result( tmp_context: context.WorkContext, ) -> None: """Use the hook's path and make its wheel visible to the local mirror.""" - _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + _configure_package(tmp_context, _source_yaml("hook-prebuilt")) req = Requirement("pkg==1.0") url = "https://pkg.test/pkg-1.0-py3-none-any.whl" hook_path = tmp_context.work_dir / "pkg-1.0-py3-none-any.whl" @@ -92,7 +100,7 @@ def test_hook_prebuilt_local_url_without_mirror_link_uses_direct_download( tmp_context: context.WorkContext, ) -> None: """Without a mirror link, a local URL uses the direct downloader, not the hook.""" - _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + _configure_package(tmp_context, _source_yaml("hook-prebuilt")) # No mirror symlink exists, so find_mirror_wheel() returns None. tmp_context.wheel_server_url = "https://local.test/simple/" req = Requirement("pkg==1.0") @@ -119,7 +127,7 @@ def test_hook_prebuilt_new_result_replaces_staged_wheel( tmp_context: context.WorkContext, ) -> None: """A fresh hook result replaces an older staged wheel with the same name.""" - _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + _configure_package(tmp_context, _source_yaml("hook-prebuilt")) req = Requirement("pkg==1.0") wheel_name = "pkg-1.0-py3-none-any.whl" hook_path = tmp_context.work_dir / wheel_name @@ -148,7 +156,7 @@ def test_hook_prebuilt_staging_failure_preserves_existing_wheel( tmp_context: context.WorkContext, ) -> None: """A failed copy leaves the old staged wheel intact and removes the temp file.""" - _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + _configure_package(tmp_context, _source_yaml("hook-prebuilt")) wheel_name = "pkg-1.0-py3-none-any.whl" hook_path = tmp_context.work_dir / wheel_name hook_path.write_bytes(b"new wheel") @@ -190,7 +198,7 @@ def test_configured_prebuilt_local_url_reuses_mirrored_wheel( tmp_context: context.WorkContext, ) -> None: """A local mirror URL returns the served file, not a same-named copy.""" - _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + _configure_package(tmp_context, _source_yaml("hook-prebuilt")) wheel_name = "pkg-1.0-py3-none-any.whl" served_wheel = tmp_context.wheels_downloads / wheel_name served_wheel.write_bytes(b"served wheel") @@ -215,7 +223,7 @@ def test_configured_prebuilt_rejects_other_kind( tmp_context: context.WorkContext, ) -> None: """An invalid hook result fails without trying a different downloader.""" - _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + _configure_package(tmp_context, _source_yaml("hook-prebuilt")) req = Requirement("pkg==1.0") def hook() -> packagesettings.DownloadedSource: @@ -243,7 +251,7 @@ def test_hook_prebuilt_failure_does_not_fall_back( tmp_context: context.WorkContext, ) -> None: """Keep the hook failure visible and avoid the default wheel fetch.""" - _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + _configure_package(tmp_context, _source_yaml("hook-prebuilt")) req = Requirement("pkg==1.0") failure = OSError("hook download failed") From 0d88414e54f4c3b5c41820eaaa3635506e05a81e Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Wed, 28 Oct 2026 13:37:03 -0400 Subject: [PATCH 14/35] refactor(resolver): remove unused cooldown code from source resolvers `get_source_provider()` sets the cooldown after it creates the provider, so the cooldown that resolvers passed in was always replaced. Remove it. Co-Authored-By: Claude Follow-up to: #1252 Signed-off-by: Lalatendu Mohanty --- src/fromager/packagesettings/_resolver.py | 15 ------------ tests/test_packagesettings_resolver.py | 26 +-------------------- tests/test_sources.py | 28 +++++++++++++++++++++++ 3 files changed, 29 insertions(+), 40 deletions(-) diff --git a/src/fromager/packagesettings/_resolver.py b/src/fromager/packagesettings/_resolver.py index b07f3432d..c41525471 100644 --- a/src/fromager/packagesettings/_resolver.py +++ b/src/fromager/packagesettings/_resolver.py @@ -1,7 +1,6 @@ from __future__ import annotations import dataclasses -import datetime import enum import inspect import logging @@ -13,7 +12,6 @@ import pydantic from .. import downloads, overrides, resolver -from ..candidate import Cooldown from ._typedefs import MODEL_CONFIG if typing.TYPE_CHECKING: @@ -182,13 +180,6 @@ class CooldownMixin(pydantic.BaseModel): Positive integer: override the cooldown with this many days. """ - @property - def _cooldown(self) -> Cooldown | None: - """Convert ``min_release_age`` to a ``Cooldown`` instance.""" - if not self.min_release_age: - return None - return Cooldown(min_age=datetime.timedelta(days=self.min_release_age)) - class AbstractPyPIResolver(AbstractResolver, CooldownMixin): """Abstract base class for PyPI resolvers""" @@ -235,7 +226,6 @@ def resolver_provider( constraints=ctx.constraints, req_type=req_type, ignore_platform=False, - cooldown=self._cooldown, ) def download( @@ -286,7 +276,6 @@ def resolver_provider( constraints=ctx.constraints, req_type=req_type, ignore_platform=False, - cooldown=self._cooldown, ) def download( @@ -362,7 +351,6 @@ def resolver_provider( override_download_url=str(self.download_url).replace( _VERSION_QUOTED, "{version}" ), - cooldown=self._cooldown, ) def download( @@ -443,7 +431,6 @@ def resolver_provider( req_type=req_type, ignore_platform=True, override_download_url=download_url, - cooldown=self._cooldown, ) def download( @@ -555,7 +542,6 @@ def _github_provider( matcher=self._get_matcher(ctx), req_type=req_type, override_download_url=override_download_url, - cooldown=self._cooldown, ) def _gitlab_provider( @@ -574,7 +560,6 @@ def _gitlab_provider( matcher=self._get_matcher(ctx), req_type=req_type, override_download_url=override_download_url, - cooldown=self._cooldown, ) diff --git a/tests/test_packagesettings_resolver.py b/tests/test_packagesettings_resolver.py index e48e0d9cb..f8c9ae3d8 100644 --- a/tests/test_packagesettings_resolver.py +++ b/tests/test_packagesettings_resolver.py @@ -1,7 +1,6 @@ from __future__ import annotations import dataclasses -import datetime import pathlib import re import subprocess @@ -17,7 +16,7 @@ from wheel.wheelfile import WheelFile from fromager import resolver, sources -from fromager.candidate import Candidate, Cooldown +from fromager.candidate import Candidate from fromager.context import WorkContext from fromager.packagesettings._resolver import ( AbstractResolver, @@ -190,19 +189,6 @@ def test_resolver_provider(self, tmp_context: WorkContext) -> None: assert p.override_download_url is None assert p.cooldown is None - def test_resolver_provider_cooldown(self, tmp_context: WorkContext) -> None: - r = PyPISDistResolver(provider="pypi-sdist", min_release_age=7) - p = r.resolver_provider(tmp_context, _REQ, _REQ_TYPE) - assert isinstance(p.cooldown, Cooldown) - assert p.cooldown.min_age == datetime.timedelta(days=7) - - def test_resolver_provider_cooldown_disabled( - self, tmp_context: WorkContext - ) -> None: - r = PyPISDistResolver(provider="pypi-sdist", min_release_age=0) - p = r.resolver_provider(tmp_context, _REQ, _REQ_TYPE) - assert p.cooldown is None - def test_cooldown_negative(self) -> None: with pytest.raises(pydantic.ValidationError, match="min_release_age"): PyPISDistResolver(provider="pypi-sdist", min_release_age=-1) @@ -591,16 +577,6 @@ def test_resolver_provider(self, tmp_context: WorkContext) -> None: assert p.override_download_url is None assert p.cooldown is None - def test_resolver_provider_cooldown(self, tmp_context: WorkContext) -> None: - r = GitHubTagDownloadResolver( - provider="github-tag-download", - project_url="https://github.com/python-wheel-build/fromager", # type: ignore[arg-type] - min_release_age=14, - ) - p = r.resolver_provider(tmp_context, _REQ, _REQ_TYPE) - assert isinstance(p.cooldown, Cooldown) - assert p.cooldown.min_age == datetime.timedelta(days=14) - def test_matcher_factory_rejects_non_callable_return( self, tmp_context: WorkContext ) -> None: diff --git a/tests/test_sources.py b/tests/test_sources.py index 19e7949bf..9b2f42d10 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -240,6 +240,34 @@ def test_get_source_provider_source_resolver_inherits_global_cooldown( assert provider.cooldown is global_cooldown +def test_get_source_provider_source_resolver_zero_disables_cooldown( + tmp_context: context.WorkContext, +) -> None: + """Source resolver with min_release_age=0 disables the global cooldown.""" + req = Requirement("test-pkg") + tmp_context.cooldown = Cooldown(min_age=datetime.timedelta(days=7)) + + source_resolver = packagesettings.PyPISDistResolver( + provider="pypi-sdist", + index_url=pydantic.HttpUrl("https://pypi.test/simple"), + min_release_age=0, + ) + + with patch.object( + packagesettings.PackageBuildInfo, + "source_resolver", + new_callable=PropertyMock, + return_value=source_resolver, + ): + provider = sources.get_source_provider( + ctx=tmp_context, + req=req, + sdist_server_url=resolver.PYPI_SERVER_URL, + ) + + assert provider.cooldown is None + + def test_get_source_provider_source_resolver_toplevel_pin_bypasses_cooldown( tmp_context: context.WorkContext, ) -> None: From c9e99590d36b3a51f4c8b58c5f54b9327d65d749 Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Wed, 28 Oct 2026 13:53:02 -0400 Subject: [PATCH 15/35] refactor(resolver): pass version and URL to source downloads `download()` now takes the version and download URL instead of a `Candidate`. Only these values are saved between commands, so the other `Candidate` fields were always empty at download time. The URL must contain everything the download needs, such as the git tag. Co-Authored-By: Claude Closes: #1384 Signed-off-by: Lalatendu Mohanty --- docs/concepts/resolver-architecture.rst | 10 ++ docs/reference/hooks.rst | 3 + src/fromager/packagesettings/_resolver.py | 113 +++++++++----- src/fromager/sources.py | 12 +- tests/test_packagesettings_resolver.py | 177 +++++++++++----------- tests/test_sources.py | 13 +- 6 files changed, 186 insertions(+), 142 deletions(-) diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index 0098c0caa..5293fba30 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -120,6 +120,16 @@ Otherwise, override plugins can replace the provider for a specific package via the ``get_resolver_provider`` hook, after which the legacy resolver settings are used. +A configured resolver's ``download()`` receives only the version and the +download URL of the selected candidate. Resolution and download can run in +different commands, and only these values are saved in ``graph.json`` and +``build-order.json``. The URL must therefore contain everything the download +needs, such as the git ref in ``git+https://host/repo.git@refs/tags/v1.0``. + +.. versionchanged:: 0.102.0 + ``download()`` takes ``version`` and ``download_url`` instead of a + ``Candidate``. + Hook downloads accept explicit artifact kinds and adapt legacy paths using profile-specific defaults. If no download hook exists, the profile selects the standard sdist or wheel downloader and its destination directory. Hook errors diff --git a/docs/reference/hooks.rst b/docs/reference/hooks.rst index 261b5bca2..dd78794b5 100644 --- a/docs/reference/hooks.rst +++ b/docs/reference/hooks.rst @@ -241,6 +241,9 @@ Source hooks evaluated, version of the package being downloaded, the download URL, and the output directory in which the source should be downloaded. + ``download_url`` is the value from the resolver provider. It can be a + ref, such as a commit SHA, instead of a URL. + For packages without ``source:`` configuration, return a ``pathlib.Path`` to the downloaded source. This legacy contract is unchanged. diff --git a/src/fromager/packagesettings/_resolver.py b/src/fromager/packagesettings/_resolver.py index c41525471..bb260c1db 100644 --- a/src/fromager/packagesettings/_resolver.py +++ b/src/fromager/packagesettings/_resolver.py @@ -10,15 +10,16 @@ import typing import pydantic +from packaging.utils import canonicalize_name from .. import downloads, overrides, resolver from ._typedefs import MODEL_CONFIG if typing.TYPE_CHECKING: from packaging.requirements import Requirement + from packaging.version import Version from .. import context, requirements_file - from ..candidate import Candidate logger = logging.getLogger(__name__) @@ -94,19 +95,31 @@ def download( self, ctx: context.WorkContext, req: Requirement, - candidate: Candidate, + version: Version, + download_url: str, ) -> DownloadedSource: - """Download the resolved artifact for *candidate*.""" + """Download the resolved artifact for *version* from *download_url*. + + Resolution and download can run in different commands, for example + ``bootstrap`` and then ``build-sequence``. Only the version and URL + are saved between them, so they are all a download receives. + *download_url* must contain everything the download needs, such as + the git ref in ``git+https://host/repo.git@refs/tags/v1.0``. It is + passed to download hooks unchanged and is not validated, so it can be + a ref instead of a URL, such as a commit SHA from a + ``VersionMapProvider``. + """ raise NotImplementedError def _download( self, ctx: context.WorkContext, req: Requirement, - candidate: Candidate, + version: Version, + download_url: str, download_kind: DownloadKind, ) -> DownloadedSource: - """Download the resolved artifact for *candidate*. + """Download the resolved artifact for *version* from *download_url*. Returns the local path and the kind of artifact that was downloaded. Dispatches to the appropriate download helper @@ -122,7 +135,7 @@ def _download( destination_dir = ctx.sdists_downloads path = downloads.download_sdist( destination_dir=destination_dir, - url=candidate.url, + url=download_url, ) case DownloadKind.tarball: # TODO: A tarball is not a proper sdist; it should not @@ -133,16 +146,16 @@ def _download( # path, so always provide a destination filename. path = downloads.download_url( destination_dir=destination_dir, - url=candidate.url, + url=download_url, destination_filename=( - f"{candidate.name}-{candidate.version}.tar.gz" + f"{canonicalize_name(req.name)}-{version}.tar.gz" ), ) case DownloadKind.prebuilt_wheel: destination_dir = ctx.wheels_prebuilt path = downloads.download_wheel( destination_dir=destination_dir, - url=candidate.url, + url=download_url, ) case DownloadKind.git_checkout: # TODO: Maybe use dedicated directory like @@ -150,9 +163,7 @@ def _download( # checkout, generate a proper sdist, then unpack to # final destination? destination_dir = ( - ctx.work_dir - / f"{req.name}-{candidate.version}" - / f"{req.name}-{candidate.version}" + ctx.work_dir / f"{req.name}-{version}" / f"{req.name}-{version}" ) if destination_dir.parent.exists(): if ctx.cleanup: @@ -164,7 +175,7 @@ def _download( ) path = downloads.download_git_source( destination_dir=destination_dir, - vcs_url=candidate.url, + vcs_url=download_url, ) case _: # includes any_source and not_available typing.assert_never(download_kind) # type: ignore[arg-type] @@ -232,10 +243,11 @@ def download( self, ctx: context.WorkContext, req: Requirement, - candidate: Candidate, + version: Version, + download_url: str, ) -> DownloadedSource: """Download an sdist from PyPI.""" - return self._download(ctx, req, candidate, DownloadKind.sdist) + return self._download(ctx, req, version, download_url, DownloadKind.sdist) class PyPIPrebuiltResolver(AbstractPyPIResolver): @@ -282,10 +294,13 @@ def download( self, ctx: context.WorkContext, req: Requirement, - candidate: Candidate, + version: Version, + download_url: str, ) -> DownloadedSource: """Download a pre-built wheel from PyPI.""" - return self._download(ctx, req, candidate, DownloadKind.prebuilt_wheel) + return self._download( + ctx, req, version, download_url, DownloadKind.prebuilt_wheel + ) class PyPIDownloadResolver(AbstractPyPIResolver): @@ -357,10 +372,11 @@ def download( self, ctx: context.WorkContext, req: Requirement, - candidate: Candidate, + version: Version, + download_url: str, ) -> DownloadedSource: """Download an sdist or tarball from a custom URL.""" - return self._download(ctx, req, candidate, self.download_kind) + return self._download(ctx, req, version, download_url, self.download_kind) class PyPIGitResolver(AbstractPyPIResolver): @@ -437,10 +453,13 @@ def download( self, ctx: context.WorkContext, req: Requirement, - candidate: Candidate, + version: Version, + download_url: str, ) -> DownloadedSource: """Clone a git repository at a specific tag.""" - return self._download(ctx, req, candidate, DownloadKind.git_checkout) + return self._download( + ctx, req, version, download_url, DownloadKind.git_checkout + ) _PEP440_TAG_PATTERN = re.compile(r"^(v?\d.*)$") @@ -596,10 +615,11 @@ def download( self, ctx: context.WorkContext, req: Requirement, - candidate: Candidate, + version: Version, + download_url: str, ) -> DownloadedSource: """Download a tarball from a GitHub tag.""" - return self._download(ctx, req, candidate, DownloadKind.tarball) + return self._download(ctx, req, version, download_url, DownloadKind.tarball) class GitHubTagCloneResolver(AbstractGitSourceResolver): @@ -637,10 +657,13 @@ def download( self, ctx: context.WorkContext, req: Requirement, - candidate: Candidate, + version: Version, + download_url: str, ) -> DownloadedSource: """Clone a git repository from a GitHub tag.""" - return self._download(ctx, req, candidate, DownloadKind.git_checkout) + return self._download( + ctx, req, version, download_url, DownloadKind.git_checkout + ) class GitLabTagDownloadResolver(AbstractGitSourceResolver): @@ -676,10 +699,11 @@ def download( self, ctx: context.WorkContext, req: Requirement, - candidate: Candidate, + version: Version, + download_url: str, ) -> DownloadedSource: """Download a tarball from a GitLab tag.""" - return self._download(ctx, req, candidate, DownloadKind.tarball) + return self._download(ctx, req, version, download_url, DownloadKind.tarball) class GitLabTagCloneResolver(AbstractGitSourceResolver): @@ -717,10 +741,13 @@ def download( self, ctx: context.WorkContext, req: Requirement, - candidate: Candidate, + version: Version, + download_url: str, ) -> DownloadedSource: """Clone a git repository from a GitLab tag.""" - return self._download(ctx, req, candidate, DownloadKind.git_checkout) + return self._download( + ctx, req, version, download_url, DownloadKind.git_checkout + ) class NotAvailableResolver(AbstractResolver): @@ -741,7 +768,8 @@ def download( self, ctx: context.WorkContext, req: Requirement, - candidate: Candidate, + version: Version, + download_url: str, ) -> DownloadedSource: """Raise because package is not available.""" raise ValueError(f"package {req.name} is not available") @@ -808,13 +836,14 @@ def _download_from_hook( self, ctx: context.WorkContext, req: Requirement, - candidate: Candidate, + version: Version, + download_url: str, legacy_kind: DownloadKind, ) -> DownloadedSource: """Download with the package hook and normalize its result.""" hook = overrides.find_override_method(req.name, "download_source") if hook is None: - return self._download(ctx, req, candidate, legacy_kind) + return self._download(ctx, req, version, download_url, legacy_kind) error_context = f"{req.name}: {self.provider!r} download_source hook" try: @@ -822,8 +851,8 @@ def _download_from_hook( hook, ctx=ctx, req=req, - version=candidate.version, - download_url=candidate.url, + version=version, + download_url=download_url, sdists_downloads_dir=ctx.sdists_downloads, ) except Exception as err: @@ -877,7 +906,7 @@ class HookSDistResolver(AbstractHookResolver): plugin hooks. The downloaded artifact can be a source distribution, a tarball, or a git checkout. Download hooks can return ``DownloadedSource`` with a concrete kind, or a legacy archive ``Path`` treated as an sdist. - Without a download hook, the candidate is downloaded as an sdist. + Without a download hook, the resolved URL is downloaded as an sdist. Specify ``build_sdist`` explicitly: ``tarball`` to repack an existing sdist, or ``pep517`` to build from a git checkout. @@ -922,10 +951,13 @@ def download( self, ctx: context.WorkContext, req: Requirement, - candidate: Candidate, + version: Version, + download_url: str, ) -> DownloadedSource: """Download source via the ``download_source`` override hook.""" - return self._download_from_hook(ctx, req, candidate, DownloadKind.sdist) + return self._download_from_hook( + ctx, req, version, download_url, DownloadKind.sdist + ) class HookPrebuiltResolver(AbstractHookResolver): @@ -934,7 +966,7 @@ class HookPrebuiltResolver(AbstractHookResolver): The ``hook-prebuilt`` provider delegates resolution and download to plugin hooks. The downloaded artifact must be a pre-built wheel. Download hooks can return ``DownloadedSource`` with kind ``prebuilt_wheel`` or a - legacy wheel ``Path``. Without a download hook, the candidate is downloaded + legacy wheel ``Path``. Without a download hook, the resolved URL is downloaded into ``ctx.wheels_prebuilt``. Example:: @@ -976,11 +1008,12 @@ def download( self, ctx: context.WorkContext, req: Requirement, - candidate: Candidate, + version: Version, + download_url: str, ) -> DownloadedSource: """Download a pre-built wheel via the ``download_source`` override hook.""" return self._download_from_hook( - ctx, req, candidate, DownloadKind.prebuilt_wheel + ctx, req, version, download_url, DownloadKind.prebuilt_wheel ) diff --git a/src/fromager/sources.py b/src/fromager/sources.py index 82b0fe384..9cf8e368d 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -33,7 +33,6 @@ tarballs, vendor_rust, ) -from .candidate import Candidate from .requirements_file import RequirementType, SourceType if typing.TYPE_CHECKING: @@ -90,16 +89,19 @@ def download_source( ) -> packagesettings.DownloadedSource: """Download a resolved source and return its path and artifact kind. - Configured source resolvers receive a minimal candidate. Legacy downloads - are adapted to the sdist kind for compatibility. + Configured source resolvers receive the version and download URL. + Legacy downloads are adapted to the sdist kind for compatibility. + + .. versionchanged:: 0.102.0 + Return ``DownloadedSource`` instead of ``pathlib.Path``. Callers use + its ``path`` attribute for the downloaded file. """ logger.info(f"downloading source for {req}") pbi = ctx.package_build_info(req) source_resolver = pbi.source_resolver if source_resolver is not None: - candidate = Candidate(name=req.name, version=version, url=download_url) - return source_resolver.download(ctx, req, candidate) + return source_resolver.download(ctx, req, version, download_url) source_path = overrides.find_and_invoke( req.name, diff --git a/tests/test_packagesettings_resolver.py b/tests/test_packagesettings_resolver.py index f8c9ae3d8..32407f1f2 100644 --- a/tests/test_packagesettings_resolver.py +++ b/tests/test_packagesettings_resolver.py @@ -16,7 +16,6 @@ from wheel.wheelfile import WheelFile from fromager import resolver, sources -from fromager.candidate import Candidate from fromager.context import WorkContext from fromager.packagesettings._resolver import ( AbstractResolver, @@ -67,52 +66,16 @@ def bad_func(x: str) -> None: _REQ = Requirement("test-pkg") _REQ_TYPE = RequirementType.INSTALL _VERSION = Version("1.2.3") -_CANDIDATE_SDIST = Candidate( - name="test-pkg", - version=_VERSION, - url="https://pypi.test/packages/test-pkg-1.2.3.tar.gz", - is_sdist=True, -) -_CANDIDATE_WHEEL = Candidate( - name="test-pkg", - version=_VERSION, - url="https://pypi.test/packages/test_pkg-1.2.3-py3-none-any.whl", - is_sdist=False, -) -_CANDIDATE_TARBALL = Candidate( - name="test-pkg", - version=_VERSION, - url="https://download.test/test-pkg-1.2.3.tar.gz", -) -_CANDIDATE_GIT = Candidate( - name="test-pkg", - version=_VERSION, - url="git+https://code.test/project/repo.git@refs/tags/v1.2.3", -) -_CANDIDATE_GITHUB_TARBALL = Candidate( - name="test-pkg", - version=_VERSION, - url="https://api.github.com/repos/org/repo/tarball/v1.2.3", - remote_tag="v1.2.3", -) -_CANDIDATE_GITHUB_CLONE = Candidate( - name="test-pkg", - version=_VERSION, - url="git+https://github.com/python-wheel-build/fromager@v1.2.3", - remote_tag="v1.2.3", -) -_CANDIDATE_GITLAB_TARBALL = Candidate( - name="test-pkg", - version=_VERSION, - url="https://gitlab.test/group/project/-/archive/v1.2.3/project-v1.2.3.tar.gz", - remote_tag="v1.2.3", -) -_CANDIDATE_GITLAB_CLONE = Candidate( - name="test-pkg", - version=_VERSION, - url="git+https://gitlab.test/python-wheel-build/fromager@v1.2.3", - remote_tag="v1.2.3", +_URL_SDIST = "https://pypi.test/packages/test-pkg-1.2.3.tar.gz" +_URL_WHEEL = "https://pypi.test/packages/test_pkg-1.2.3-py3-none-any.whl" +_URL_TARBALL = "https://download.test/test-pkg-1.2.3.tar.gz" +_URL_GIT = "git+https://code.test/project/repo.git@refs/tags/v1.2.3" +_URL_GITHUB_TARBALL = "https://api.github.com/repos/org/repo/tarball/v1.2.3" +_URL_GITHUB_CLONE = "git+https://github.com/python-wheel-build/fromager@v1.2.3" +_URL_GITLAB_TARBALL = ( + "https://gitlab.test/group/project/-/archive/v1.2.3/project-v1.2.3.tar.gz" ) +_URL_GITLAB_CLONE = "git+https://gitlab.test/python-wheel-build/fromager@v1.2.3" class _SourceWrapper(pydantic.BaseModel): @@ -198,13 +161,13 @@ def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> No expected = tmp_context.sdists_downloads / "test-pkg-1.2.3.tar.gz" mock_dl.return_value = expected r = _parse(self.YAML) - result = r.download(tmp_context, _REQ, _CANDIDATE_SDIST) + result = r.download(tmp_context, _REQ, _VERSION, _URL_SDIST) assert isinstance(result, DownloadedSource) assert result.path == expected assert result.kind is DownloadKind.sdist mock_dl.assert_called_once_with( destination_dir=tmp_context.sdists_downloads, - url=_CANDIDATE_SDIST.url, + url=_URL_SDIST, ) @@ -240,13 +203,13 @@ def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> No expected = tmp_context.wheels_prebuilt / "test_pkg-1.2.3-py3-none-any.whl" mock_dl.return_value = expected r = _parse(self.YAML) - result = r.download(tmp_context, _REQ, _CANDIDATE_WHEEL) + result = r.download(tmp_context, _REQ, _VERSION, _URL_WHEEL) assert isinstance(result, DownloadedSource) assert result.path == expected assert result.kind is DownloadKind.prebuilt_wheel mock_dl.assert_called_once_with( destination_dir=tmp_context.wheels_prebuilt, - url=_CANDIDATE_WHEEL.url, + url=_URL_WHEEL, ) @@ -314,16 +277,31 @@ def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> No expected = tmp_context.sdists_downloads / "test-pkg-1.2.3.tar.gz" mock_dl.return_value = expected r = _parse(self.YAML) - result = r.download(tmp_context, _REQ, _CANDIDATE_TARBALL) + result = r.download(tmp_context, _REQ, _VERSION, _URL_TARBALL) assert isinstance(result, DownloadedSource) assert result.path == expected assert result.kind is DownloadKind.tarball mock_dl.assert_called_once_with( destination_dir=tmp_context.sdists_downloads, - url=_CANDIDATE_TARBALL.url, + url=_URL_TARBALL, destination_filename="test-pkg-1.2.3.tar.gz", ) + @mock.patch("fromager.downloads.download_url") + def test_download_canonicalizes_tarball_name( + self, mock_dl: mock.MagicMock, tmp_context: WorkContext + ) -> None: + """Name the tarball with the canonical package name.""" + r = _parse(self.YAML) + + r.download(tmp_context, Requirement("Foo_Bar"), _VERSION, _URL_TARBALL) + + mock_dl.assert_called_once_with( + destination_dir=tmp_context.sdists_downloads, + url=_URL_TARBALL, + destination_filename="foo-bar-1.2.3.tar.gz", + ) + class TestPyPIGitResolver: YAML = """\ @@ -385,13 +363,13 @@ def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> No ) mock_dl.return_value = expected_dest r = _parse(self.YAML) - result = r.download(tmp_context, _REQ, _CANDIDATE_GIT) + result = r.download(tmp_context, _REQ, _VERSION, _URL_GIT) assert isinstance(result, DownloadedSource) assert result.path == expected_dest assert result.kind is DownloadKind.git_checkout mock_dl.assert_called_once_with( destination_dir=expected_dest, - vcs_url=_CANDIDATE_GIT.url, + vcs_url=_URL_GIT, ) def test_download_rejects_existing_checkout_without_cleanup( @@ -406,7 +384,7 @@ def test_download_rejects_existing_checkout_without_cleanup( (checkout / "keep.txt").write_text("prepared") with pytest.raises(FileExistsError, match="enable cleanup"): - _parse(self.YAML).download(tmp_context, _REQ, _CANDIDATE_GIT) + _parse(self.YAML).download(tmp_context, _REQ, _VERSION, _URL_GIT) assert (checkout / "keep.txt").read_text() == "prepared" @@ -421,13 +399,13 @@ def test_download_cleans_checkout_before_reclone( (checkout.parent / "build-meta.json").write_text("old") def clone(*, destination_dir: pathlib.Path, vcs_url: str) -> pathlib.Path: - assert vcs_url == _CANDIDATE_GIT.url + assert vcs_url == _URL_GIT destination_dir.mkdir(parents=True) (destination_dir / "new.txt").write_text("new") return destination_dir with mock.patch("fromager.downloads.download_git_source", side_effect=clone): - result = _parse(self.YAML).download(tmp_context, _REQ, _CANDIDATE_GIT) + result = _parse(self.YAML).download(tmp_context, _REQ, _VERSION, _URL_GIT) assert result == DownloadedSource(checkout, DownloadKind.git_checkout) assert (checkout / "new.txt").read_text() == "new" @@ -447,7 +425,7 @@ def test_download_preserves_non_checkout_on_collision( meta_file.write_text("existing metadata") with pytest.raises(FileExistsError, match="already exists"): - _parse(self.YAML).download(tmp_context, _REQ, _CANDIDATE_GIT) + _parse(self.YAML).download(tmp_context, _REQ, _VERSION, _URL_GIT) assert (checkout / "keep.txt").read_text() == "existing source" assert meta_file.read_text() == "existing metadata" @@ -477,18 +455,14 @@ def test_local_git_checkout_prepares_and_reclones( capture_output=True, ) subprocess.run(["git", "-C", str(repo), "tag", "v1.2.3"], check=True) - candidate = Candidate( - name=_REQ.name, - version=_VERSION, - url=f"git+file://{repo}@refs/tags/v1.2.3", - ) + url = f"git+file://{repo}@refs/tags/v1.2.3" source_resolver = _parse(self.YAML) tmp_context.cleanup = False package_dir = tmp_context.work_dir / f"{_REQ.name}-{_VERSION}" (package_dir / "other-dir").mkdir(parents=True) (package_dir / "build-meta.json").write_text("stale") - downloaded = source_resolver.download(tmp_context, _REQ, candidate) + downloaded = source_resolver.download(tmp_context, _REQ, _VERSION, url) assert (package_dir / "other-dir").is_dir() assert (package_dir / "build-meta.json").read_text() == "stale" prepared = sources.prepare_source( @@ -500,7 +474,7 @@ def test_local_git_checkout_prepares_and_reclones( ) (prepared / "keep.txt").write_text("prepared") with pytest.raises(FileExistsError, match="already exists"): - source_resolver.download(tmp_context, _REQ, candidate) + source_resolver.download(tmp_context, _REQ, _VERSION, url) assert downloaded.kind is DownloadKind.git_checkout assert prepared == downloaded.path @@ -511,7 +485,7 @@ def test_local_git_checkout_prepares_and_reclones( ) tmp_context.cleanup = True - recloned = source_resolver.download(tmp_context, _REQ, candidate) + recloned = source_resolver.download(tmp_context, _REQ, _VERSION, url) assert recloned.path == prepared assert (recloned.path / ".git").is_dir() @@ -620,13 +594,13 @@ def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> No expected = tmp_context.sdists_downloads / "test-pkg-1.2.3.tar.gz" mock_dl.return_value = expected r = _parse(self.YAML) - result = r.download(tmp_context, _REQ, _CANDIDATE_GITHUB_TARBALL) + result = r.download(tmp_context, _REQ, _VERSION, _URL_GITHUB_TARBALL) assert isinstance(result, DownloadedSource) assert result.path == expected assert result.kind is DownloadKind.tarball mock_dl.assert_called_once_with( destination_dir=tmp_context.sdists_downloads, - url=_CANDIDATE_GITHUB_TARBALL.url, + url=_URL_GITHUB_TARBALL, destination_filename="test-pkg-1.2.3.tar.gz", ) @@ -667,13 +641,13 @@ def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> No ) mock_dl.return_value = expected_dest r = _parse(self.YAML) - result = r.download(tmp_context, _REQ, _CANDIDATE_GITHUB_CLONE) + result = r.download(tmp_context, _REQ, _VERSION, _URL_GITHUB_CLONE) assert isinstance(result, DownloadedSource) assert result.path == expected_dest assert result.kind is DownloadKind.git_checkout mock_dl.assert_called_once_with( destination_dir=expected_dest, - vcs_url=_CANDIDATE_GITHUB_CLONE.url, + vcs_url=_URL_GITHUB_CLONE, ) @@ -709,13 +683,13 @@ def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> No expected = tmp_context.sdists_downloads / "test-pkg-1.2.3.tar.gz" mock_dl.return_value = expected r = _parse(self.YAML) - result = r.download(tmp_context, _REQ, _CANDIDATE_GITLAB_TARBALL) + result = r.download(tmp_context, _REQ, _VERSION, _URL_GITLAB_TARBALL) assert isinstance(result, DownloadedSource) assert result.path == expected assert result.kind is DownloadKind.tarball mock_dl.assert_called_once_with( destination_dir=tmp_context.sdists_downloads, - url=_CANDIDATE_GITLAB_TARBALL.url, + url=_URL_GITLAB_TARBALL, destination_filename="test-pkg-1.2.3.tar.gz", ) @@ -756,13 +730,13 @@ def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> No ) mock_dl.return_value = expected_dest r = _parse(self.YAML) - result = r.download(tmp_context, _REQ, _CANDIDATE_GITLAB_CLONE) + result = r.download(tmp_context, _REQ, _VERSION, _URL_GITLAB_CLONE) assert isinstance(result, DownloadedSource) assert result.path == expected_dest assert result.kind is DownloadKind.git_checkout mock_dl.assert_called_once_with( destination_dir=expected_dest, - vcs_url=_CANDIDATE_GITLAB_CLONE.url, + vcs_url=_URL_GITLAB_CLONE, ) @@ -791,7 +765,7 @@ def test_resolver_provider(self, tmp_context: WorkContext) -> None: def test_download(self, tmp_context: WorkContext) -> None: r = _parse(self.YAML) with pytest.raises(ValueError, match="not available"): - r.download(tmp_context, _REQ, _CANDIDATE_SDIST) + r.download(tmp_context, _REQ, _VERSION, _URL_SDIST) class TestHookSDistResolver: @@ -970,7 +944,7 @@ def test_hook_download_preserves_typed_result_and_arguments( if provider == "hook-sdist" else TestHookPrebuiltResolver.YAML ) - candidate = _CANDIDATE_WHEEL if r.resolves_prebuilt_wheel else _CANDIDATE_SDIST + url = _URL_WHEEL if r.resolves_prebuilt_wheel else _URL_SDIST expected = DownloadedSource(path=tmp_context.work_dir / "artifact", kind=kind) captured: dict[str, object] = {} @@ -996,20 +970,45 @@ def hook( ) as find_hook, mock.patch.object(type(r), "_download") as fallback, ): - result = r.download(tmp_context, _REQ, candidate) + result = r.download(tmp_context, _REQ, _VERSION, url) assert result is expected assert captured == { "ctx": tmp_context, "req": _REQ, - "version": candidate.version, - "download_url": candidate.url, + "version": _VERSION, + "download_url": url, "sdists_downloads_dir": tmp_context.sdists_downloads, } find_hook.assert_called_once_with(_REQ.name, "download_source") fallback.assert_not_called() +def test_hook_download_passes_non_url_ref_unchanged( + tmp_context: WorkContext, +) -> None: + """Pass a bare commit SHA to the download hook as ``download_url``. + + Version-map plugins store a commit SHA instead of a URL. + """ + r = _parse(TestHookSDistResolver.YAML) + sha = "0123456789abcdef0123456789abcdef01234567" + expected = DownloadedSource( + path=tmp_context.work_dir / "checkout", kind=DownloadKind.git_checkout + ) + captured: dict[str, object] = {} + + def hook(version: Version, download_url: str) -> DownloadedSource: + captured.update(version=version, download_url=download_url) + return expected + + with mock.patch("fromager.overrides.find_override_method", return_value=hook): + result = r.download(tmp_context, _REQ, _VERSION, sha) + + assert result is expected + assert captured == {"version": _VERSION, "download_url": sha} + + @pytest.mark.parametrize( ("provider", "filename", "kind"), [ @@ -1039,7 +1038,7 @@ def hook(req: Requirement, version: Version) -> pathlib.Path: return expected with mock.patch("fromager.overrides.find_override_method", return_value=hook): - result = r.download(tmp_context, _REQ, _CANDIDATE_SDIST) + result = r.download(tmp_context, _REQ, _VERSION, _URL_SDIST) assert result == DownloadedSource(path=expected, kind=kind) @@ -1106,7 +1105,7 @@ def hook() -> object: mock.patch.object(type(r), "_download") as fallback, pytest.raises(error_type, match=re.escape(message)) as exc_info, ): - r.download(tmp_context, _REQ, _CANDIDATE_SDIST) + r.download(tmp_context, _REQ, _VERSION, _URL_SDIST) assert f"test-pkg: '{provider}' download_source hook" in str(exc_info.value) fallback.assert_not_called() @@ -1138,7 +1137,7 @@ def hook() -> DownloadedSource: mock.patch("fromager.overrides.find_override_method", return_value=hook), pytest.raises(ValueError, match=f"unsupported kind '{kind.value}'") as exc_info, ): - r.download(tmp_context, _REQ, _CANDIDATE_SDIST) + r.download(tmp_context, _REQ, _VERSION, _URL_SDIST) for allowed_kind in r.download_kinds: assert allowed_kind.value in str(exc_info.value) @@ -1177,7 +1176,7 @@ def hook() -> pathlib.Path: mock.patch("fromager.overrides.find_override_method", return_value=hook), pytest.raises(ValueError, match="Return DownloadedSource"), ): - r.download(tmp_context, _REQ, _CANDIDATE_SDIST) + r.download(tmp_context, _REQ, _VERSION, _URL_SDIST) @pytest.mark.parametrize("provider", ["hook-sdist", "hook-prebuilt"]) @@ -1191,11 +1190,11 @@ def test_hook_download_uses_profile_default_when_hook_is_missing( else TestHookPrebuiltResolver.YAML ) prebuilt = r.resolves_prebuilt_wheel - candidate = _CANDIDATE_WHEEL if prebuilt else _CANDIDATE_SDIST + url = _URL_WHEEL if prebuilt else _URL_SDIST destination = ( tmp_context.wheels_prebuilt if prebuilt else tmp_context.sdists_downloads ) - expected = destination / candidate.url.rsplit("/", 1)[1] + expected = destination / url.rsplit("/", 1)[1] kind = DownloadKind.prebuilt_wheel if prebuilt else DownloadKind.sdist with ( @@ -1203,10 +1202,10 @@ def test_hook_download_uses_profile_default_when_hook_is_missing( mock.patch("fromager.downloads.download_sdist", return_value=expected) as sdist, mock.patch("fromager.downloads.download_wheel", return_value=expected) as wheel, ): - result = r.download(tmp_context, _REQ, candidate) + result = r.download(tmp_context, _REQ, _VERSION, url) selected, unused = (wheel, sdist) if prebuilt else (sdist, wheel) - selected.assert_called_once_with(destination_dir=destination, url=candidate.url) + selected.assert_called_once_with(destination_dir=destination, url=url) unused.assert_not_called() assert result == DownloadedSource(path=expected, kind=kind) @@ -1233,7 +1232,7 @@ def hook() -> pathlib.Path: RuntimeError, match=f"'{provider}' download_source hook failed" ) as exc_info, ): - r.download(tmp_context, _REQ, _CANDIDATE_SDIST) + r.download(tmp_context, _REQ, _VERSION, _URL_SDIST) assert exc_info.value.__cause__ is cause fallback.assert_not_called() @@ -1255,7 +1254,7 @@ def test_hook_prebuilt_download_fallback_accepts_valid_wheel( ) with mock.patch("fromager.overrides.find_override_method", return_value=None): - result = r.download(tmp_context, _REQ, _CANDIDATE_WHEEL) + result = r.download(tmp_context, _REQ, _VERSION, _URL_WHEEL) assert result == DownloadedSource(path=path, kind=DownloadKind.prebuilt_wheel) diff --git a/tests/test_sources.py b/tests/test_sources.py index 9b2f42d10..5b58779e5 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -12,7 +12,7 @@ from packaging.version import Version from fromager import context, packagesettings, resolver, sources -from fromager.candidate import Candidate, Cooldown +from fromager.candidate import Cooldown from fromager.requirements_file import RequirementType @@ -1057,12 +1057,12 @@ def test_download_source_uses_configured_source_resolver( ctx=tmp_context, req=req, version=version, download_url=download_url ) - expected_candidate = Candidate(name=req.name, version=version, url=download_url) assert result == expected_result mock_download.assert_called_once_with( tmp_context, req, - expected_candidate, + version, + download_url, packagesettings.DownloadKind.sdist, ) mock_invoke.assert_not_called() @@ -1234,11 +1234,8 @@ def prepare_override( source_resolver.download.assert_called_once_with( tmp_context, req, - Candidate( - name=req.name, - version=version, - url="git+https://code.test/org/pkg.git@v1.0", - ), + version, + "git+https://code.test/org/pkg.git@v1.0", ) find_override_method.assert_called_once_with(req.name, "prepare_source") assert (checkout.parent / "build-meta.json").is_file() From fb7b1659858311593b45944f31651f5b0b59ef6a Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Wed, 28 Oct 2026 14:33:48 -0400 Subject: [PATCH 16/35] feat(resolver): let hook profiles work with only a download hook If a package has a download hook but no resolver hook, `hook-sdist` and `hook-prebuilt` now find versions on their `index_url`. A package with no hooks gets an error. Closes: #1387 Co-Authored-By: Claude Signed-off-by: Lalatendu Mohanty --- docs/concepts/resolver-architecture.rst | 11 +- docs/proposals/new-resolver-config.md | 9 +- docs/reference/hooks.rst | 30 +++- src/fromager/packagesettings/_resolver.py | 85 +++++++++--- tests/test_packagesettings_resolver.py | 160 +++++++++++++++------- tests/test_sources.py | 41 ++++++ 6 files changed, 259 insertions(+), 77 deletions(-) diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index 5293fba30..c8c9990e6 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -107,15 +107,22 @@ CLI commands interact with providers through a common Hook profiles pass their ``index_url`` to the legacy ``get_resolver_provider`` hook as ``sdist_server_url``. ``hook-sdist`` defaults to PyPI, while ``hook-prebuilt`` requires an explicit index. + A hook profile without a ``get_resolver_provider`` hook finds versions on + its ``index_url`` and uses the ``download_source`` hook. Per-package settings in YAML can select which provider to use and configure its parameters (index URL, tag pattern, etc.). When a package has a ``source:`` resolver configured, that resolver creates the provider. The ``hook-sdist`` and ``hook-prebuilt`` profiles delegate provider creation -to the package's required ``get_resolver_provider`` hook and validate that it -returns a :class:`~fromager.resolver.BaseProvider`. +to the package's ``get_resolver_provider`` hook when it exists and validate +that it returns a :class:`~fromager.resolver.BaseProvider`. The hook may ignore or replace the profile's index URL, but the profile's value is stable across Fromager commands. +Without that hook, the profiles find versions on ``index_url``: +``hook-sdist`` looks at sdists and wheels for any platform, like +``pypi-git``, and ``hook-prebuilt`` looks at wheels for this platform, like +``pypi-prebuilt``. The package's ``download_source`` hook then gets the +artifact. A package with neither hook is an error. Otherwise, override plugins can replace the provider for a specific package via the ``get_resolver_provider`` hook, after which the legacy resolver settings are used. diff --git a/docs/proposals/new-resolver-config.md b/docs/proposals/new-resolver-config.md index 72485fe18..880fe54c3 100644 --- a/docs/proposals/new-resolver-config.md +++ b/docs/proposals/new-resolver-config.md @@ -147,12 +147,17 @@ variants: [hooks](../reference/hooks.rst). The `resolver_provider` hook receives the `req` argument for access to the current requirement and `sdist_server_url=str(index_url)`. `index_url` defaults to PyPI. Hooks can - return any source artifact (sdist, tarball, or git checkout). + return any source artifact (sdist, tarball, or git checkout). Without a + `resolver_provider` hook, Fromager finds versions on `index_url` in sdists + and wheels for any platform, and the `download_source` hook gets the + source. A package needs at least one of the two hooks. - The `hook-prebuilt` profile is similar to `hook-sdist` but the downloaded artifact must be a pre-built wheel. Its required `index_url` replaces the legacy variant `wheel_server_url`; the hook receives it as - `sdist_server_url` and may use, ignore, or replace it. + `sdist_server_url` and may use, ignore, or replace it. Without a + `resolver_provider` hook, Fromager finds versions on `index_url` in + wheels for this platform. - The `not-available` profile raises an error. It can be used to block a package and only enable it for a single variant. diff --git a/docs/reference/hooks.rst b/docs/reference/hooks.rst index dd78794b5..dd58f154b 100644 --- a/docs/reference/hooks.rst +++ b/docs/reference/hooks.rst @@ -142,8 +142,8 @@ Resolver hooks ``req_type`` and ``ignore_platform`` when those parameters are supported by its signature. - The ``hook-sdist`` and ``hook-prebuilt`` source profiles require this hook. - They pass the following keyword arguments: + The ``hook-sdist`` and ``hook-prebuilt`` source profiles call this hook + when it exists. They pass the following keyword arguments: * ``ctx`` * ``req`` @@ -154,9 +154,17 @@ Resolver hooks * ``ignore_platform`` Older hooks may omit newer arguments; unsupported keyword arguments are - filtered for compatibility. A missing hook or a return value that is not a - :class:`~fromager.resolver.BaseProvider` is an error. Hook profiles never - fall back to the default PyPI provider. + filtered for compatibility. A return value that is not a + :class:`~fromager.resolver.BaseProvider` is an error. + + Without this hook, a hook profile finds versions on its ``index_url`` and + the ``download_source`` hook gets the artifact. A package with neither + hook is an error; use ``pypi-sdist`` or ``pypi-prebuilt`` instead. + + .. versionchanged:: 0.102.0 + + ``hook-sdist`` and ``hook-prebuilt`` no longer require this hook when + the package has a ``download_source`` hook. For a configured hook profile, ``sdist_server_url`` is the profile's ``index_url``. ``hook-sdist`` defaults that field to PyPI; @@ -275,8 +283,16 @@ Source hooks URL as an sdist into ``ctx.sdists_downloads``. ``hook-prebuilt`` downloads it as a wheel into ``ctx.wheels_prebuilt``. These defaults use the standard archive and wheel validation. Custom tarball URLs and git - downloads require a download hook. The ``get_resolver_provider`` hook - remains required for both profiles. + downloads require a download hook. + + Without a ``get_resolver_provider`` hook, ``hook-sdist`` finds + versions on its ``index_url`` in sdists and wheels for any platform, + and ``hook-prebuilt`` finds them in wheels for this platform. The + download hook then gets the artifact. ``download_url`` is the URL of + a matching file on the index, which can be a wheel, so the hook + should get the source from its own location, such as a git tag. + If the hook only clones a tag, use ``pypi-git`` instead and drop + the hook. To migrate a raw-tarball hook to ``hook-sdist``, specify ``build_sdist`` in YAML and return its artifact kind explicitly: diff --git a/src/fromager/packagesettings/_resolver.py b/src/fromager/packagesettings/_resolver.py index bb260c1db..a7dcf805f 100644 --- a/src/fromager/packagesettings/_resolver.py +++ b/src/fromager/packagesettings/_resolver.py @@ -781,9 +781,30 @@ class AbstractHookResolver(AbstractResolver, CooldownMixin): supports_override_hooks: typing.ClassVar[bool] = True """Hook resolvers support override hooks.""" + hookless_provider: typing.ClassVar[str] + """Profile to suggest for packages without hooks.""" + + def _find_resolver_hook(self, req: Requirement) -> typing.Callable | None: + """Return the package resolver hook, or None to resolve on the index. + + Resolving on the index needs a ``download_source`` hook to get the + source. Raise when the package has neither hook. + """ + hook = overrides.find_override_method(req.name, "get_resolver_provider") + if hook is None and not overrides.find_override_method( + req.name, "download_source" + ): + raise ValueError( + f"{req.name}: source resolver {self.provider!r} requires a " + "get_resolver_provider or download_source override hook; " + f"use {self.hookless_provider!r} for a package without hooks" + ) + return hook + def _resolver_provider_from_hook( self, *, + hook: typing.Callable, ctx: context.WorkContext, req: Requirement, req_type: requirements_file.RequirementType | None, @@ -792,14 +813,7 @@ def _resolver_provider_from_hook( include_wheels: bool, ignore_platform: bool, ) -> resolver.BaseProvider: - """Invoke the required package resolver hook and validate its result.""" - hook = overrides.find_override_method(req.name, "get_resolver_provider") - if hook is None: - raise ValueError( - f"{req.name}: source resolver {self.provider!r} requires a " - "get_resolver_provider override hook" - ) - + """Invoke the package resolver hook and validate its result.""" try: provider = overrides.invoke( hook, @@ -903,9 +917,11 @@ class HookSDistResolver(AbstractHookResolver): """Call resolver_provider and download_source hook, build from source The ``hook-sdist`` provider delegates resolution and download to - plugin hooks. The downloaded artifact can be a source distribution, - a tarball, or a git checkout. Download hooks can return ``DownloadedSource`` - with a concrete kind, or a legacy archive ``Path`` treated as an sdist. + plugin hooks. Either hook is optional, but not both. Without a resolver + hook, versions come from ``index_url``. The downloaded artifact can be a + source distribution, a tarball, or a git checkout. Download hooks can + return ``DownloadedSource`` with a concrete kind, or a legacy archive + ``Path`` treated as an sdist. Without a download hook, the resolved URL is downloaded as an sdist. Specify ``build_sdist`` explicitly: ``tarball`` to repack an existing sdist, or ``pep517`` to build from a git checkout. @@ -929,6 +945,7 @@ class HookSDistResolver(AbstractHookResolver): download_kinds: typing.ClassVar[DownloadKindSet] = frozenset( {DownloadKind.sdist, DownloadKind.tarball, DownloadKind.git_checkout} ) + hookless_provider: typing.ClassVar[str] = "pypi-sdist" def resolver_provider( self, @@ -936,8 +953,24 @@ def resolver_provider( req: Requirement, req_type: requirements_file.RequirementType | None, ) -> resolver.BaseProvider: - """Return a source provider from the package resolver hook.""" + """Return a source provider from the resolver hook or the index. + + Without a ``get_resolver_provider`` hook, find versions on + ``index_url`` in sdists and wheels for any platform, like + ``pypi-git``. The ``download_source`` hook then gets the source. + """ + hook = self._find_resolver_hook(req) + if hook is None: + return resolver.PyPIProvider( + include_sdists=True, + include_wheels=True, + sdist_server_url=str(self.index_url), + constraints=ctx.constraints, + req_type=req_type, + ignore_platform=True, + ) return self._resolver_provider_from_hook( + hook=hook, ctx=ctx, req=req, req_type=req_type, @@ -964,10 +997,11 @@ class HookPrebuiltResolver(AbstractHookResolver): """Call resolver_provider and download_source hook, use pre-built wheel The ``hook-prebuilt`` provider delegates resolution and download to - plugin hooks. The downloaded artifact must be a pre-built wheel. Download - hooks can return ``DownloadedSource`` with kind ``prebuilt_wheel`` or a - legacy wheel ``Path``. Without a download hook, the resolved URL is downloaded - into ``ctx.wheels_prebuilt``. + plugin hooks. Either hook is optional, but not both. Without a resolver + hook, versions come from ``index_url``. The downloaded artifact must be a + pre-built wheel. Download hooks can return ``DownloadedSource`` with kind + ``prebuilt_wheel`` or a legacy wheel ``Path``. Without a download hook, + the resolved URL is downloaded into ``ctx.wheels_prebuilt``. Example:: @@ -986,6 +1020,7 @@ class HookPrebuiltResolver(AbstractHookResolver): {DownloadKind.prebuilt_wheel} ) resolves_prebuilt_wheel: typing.ClassVar[bool] = True + hookless_provider: typing.ClassVar[str] = "pypi-prebuilt" def resolver_provider( self, @@ -993,8 +1028,24 @@ def resolver_provider( req: Requirement, req_type: requirements_file.RequirementType | None, ) -> resolver.BaseProvider: - """Return a pre-built wheel provider from the package resolver hook.""" + """Return a pre-built wheel provider from the resolver hook or the index. + + Without a ``get_resolver_provider`` hook, find versions on + ``index_url`` in wheels for this platform, like ``pypi-prebuilt``. + The ``download_source`` hook then gets the wheel. + """ + hook = self._find_resolver_hook(req) + if hook is None: + return resolver.PyPIProvider( + include_sdists=False, + include_wheels=True, + sdist_server_url=str(self.index_url), + constraints=ctx.constraints, + req_type=req_type, + ignore_platform=False, + ) return self._resolver_provider_from_hook( + hook=hook, ctx=ctx, req=req, req_type=req_type, diff --git a/tests/test_packagesettings_resolver.py b/tests/test_packagesettings_resolver.py index 32407f1f2..d4030ec1e 100644 --- a/tests/test_packagesettings_resolver.py +++ b/tests/test_packagesettings_resolver.py @@ -926,6 +926,13 @@ def hook( ) +def _parse_hook_profile(provider: str) -> SourceResolver: + """Parse the test YAML for the ``hook-sdist`` or ``hook-prebuilt`` profile.""" + if provider == "hook-sdist": + return _parse(TestHookSDistResolver.YAML) + return _parse(TestHookPrebuiltResolver.YAML) + + @pytest.mark.parametrize( ("provider", "kind"), [ @@ -939,11 +946,7 @@ def test_hook_download_preserves_typed_result_and_arguments( tmp_context: WorkContext, provider: str, kind: DownloadKind ) -> None: """Preserve the hook's explicit kind and forward the legacy arguments.""" - r = _parse( - TestHookSDistResolver.YAML - if provider == "hook-sdist" - else TestHookPrebuiltResolver.YAML - ) + r = _parse_hook_profile(provider) url = _URL_WHEEL if r.resolves_prebuilt_wheel else _URL_SDIST expected = DownloadedSource(path=tmp_context.work_dir / "artifact", kind=kind) captured: dict[str, object] = {} @@ -1025,11 +1028,7 @@ def test_hook_download_adapts_legacy_path_and_narrow_signature( tmp_context: WorkContext, provider: str, filename: str, kind: DownloadKind ) -> None: """Adapt bare paths without requiring hooks to accept every argument.""" - r = _parse( - TestHookSDistResolver.YAML - if provider == "hook-sdist" - else TestHookPrebuiltResolver.YAML - ) + r = _parse_hook_profile(provider) expected = tmp_context.work_dir / filename def hook(req: Requirement, version: Version) -> pathlib.Path: @@ -1091,11 +1090,7 @@ def test_hook_download_rejects_invalid_results_without_fallback( message: str, ) -> None: """Reject malformed hook results with package and provider context.""" - r = _parse( - TestHookSDistResolver.YAML - if provider == "hook-sdist" - else TestHookPrebuiltResolver.YAML - ) + r = _parse_hook_profile(provider) def hook() -> object: return hook_result @@ -1124,11 +1119,7 @@ def test_hook_download_rejects_kind_incompatible_with_profile( tmp_context: WorkContext, provider: str, kind: DownloadKind ) -> None: """Do not relabel an explicit kind to fit the selected profile.""" - r = _parse( - TestHookSDistResolver.YAML - if provider == "hook-sdist" - else TestHookPrebuiltResolver.YAML - ) + r = _parse_hook_profile(provider) def hook() -> DownloadedSource: return DownloadedSource(path=tmp_context.work_dir, kind=kind) @@ -1158,11 +1149,7 @@ def test_hook_download_requires_kind_for_unsupported_legacy_paths( tmp_context: WorkContext, provider: str, filename: str, is_directory: bool ) -> None: """Require an explicit kind when a path cannot use the legacy default.""" - r = _parse( - TestHookSDistResolver.YAML - if provider == "hook-sdist" - else TestHookPrebuiltResolver.YAML - ) + r = _parse_hook_profile(provider) path = tmp_context.work_dir / filename if is_directory: path.mkdir() @@ -1184,11 +1171,7 @@ def test_hook_download_uses_profile_default_when_hook_is_missing( tmp_context: WorkContext, provider: str ) -> None: """Select the correct download helper and destination without a hook.""" - r = _parse( - TestHookSDistResolver.YAML - if provider == "hook-sdist" - else TestHookPrebuiltResolver.YAML - ) + r = _parse_hook_profile(provider) prebuilt = r.resolves_prebuilt_wheel url = _URL_WHEEL if prebuilt else _URL_SDIST destination = ( @@ -1215,11 +1198,7 @@ def test_hook_download_chains_failure_without_fallback( tmp_context: WorkContext, provider: str ) -> None: """Preserve the hook failure and do not attempt a default download.""" - r = _parse( - TestHookSDistResolver.YAML - if provider == "hook-sdist" - else TestHookPrebuiltResolver.YAML - ) + r = _parse_hook_profile(provider) cause = OSError("download failed") def hook() -> pathlib.Path: @@ -1286,29 +1265,112 @@ def legacy_hook( assert result is provider -@mock.patch("fromager.packagesettings._resolver.resolver.default_resolver_provider") +def _find_hooks( + hooks: dict[str, typing.Callable[..., typing.Any]], +) -> typing.Callable[[str, str], typing.Callable[..., typing.Any] | None]: + """Return a ``find_override_method`` replacement for the given hooks.""" + + def find(distname: str, method: str) -> typing.Callable[..., typing.Any] | None: + return hooks.get(method) + + return find + + +@pytest.mark.parametrize( + ("provider", "include_sdists", "ignore_platform", "kind"), + [ + ("hook-sdist", True, True, DownloadKind.git_checkout), + ("hook-prebuilt", False, False, DownloadKind.prebuilt_wheel), + ], +) +def test_hook_resolver_uses_index_without_resolver_hook( + tmp_context: WorkContext, + provider: str, + include_sdists: bool, + ignore_platform: bool, + kind: DownloadKind, +) -> None: + """A download-only package resolves on ``index_url`` and downloads by hook.""" + r = _parse_hook_profile(provider) + expected = DownloadedSource(path=tmp_context.work_dir / "artifact", kind=kind) + + def download_source() -> DownloadedSource: + return expected + + with mock.patch( + "fromager.overrides.find_override_method", + side_effect=_find_hooks({"download_source": download_source}), + ): + p = r.resolver_provider(tmp_context, _REQ, _REQ_TYPE) + result = r.download(tmp_context, _REQ, _VERSION, _URL_WHEEL) + + assert isinstance(p, resolver.PyPIProvider) + assert p.sdist_server_url == "https://index.test/simple" + assert p.include_sdists is include_sdists + assert p.include_wheels is True + assert p.ignore_platform is ignore_platform + assert p.req_type is _REQ_TYPE + assert result is expected + + +@pytest.mark.parametrize("provider", ["hook-sdist", "hook-prebuilt"]) +@pytest.mark.parametrize("has_download_hook", [True, False]) +def test_hook_resolver_prefers_resolver_hook( + tmp_context: WorkContext, + provider: str, + has_download_hook: bool, +) -> None: + """Use the resolver hook whether or not a download hook exists.""" + r = _parse_hook_profile(provider) + hook_provider = resolver.GenericProvider(version_source=lambda identifier: []) + + def get_resolver_provider() -> resolver.BaseProvider: + return hook_provider + + hooks: dict[str, typing.Callable[..., typing.Any]] = { + "get_resolver_provider": get_resolver_provider + } + if has_download_hook: + hooks["download_source"] = lambda: None + + with mock.patch( + "fromager.overrides.find_override_method", side_effect=_find_hooks(hooks) + ): + result = r.resolver_provider(tmp_context, _REQ, _REQ_TYPE) + + assert result is hook_provider + + +@pytest.mark.parametrize( + ("provider", "suggested"), + [("hook-sdist", "pypi-sdist"), ("hook-prebuilt", "pypi-prebuilt")], +) +@mock.patch("fromager.packagesettings._resolver.resolver.PyPIProvider") @mock.patch( "fromager.packagesettings._resolver.overrides.find_override_method", return_value=None, ) -def test_hook_resolver_does_not_fall_back_when_hook_is_missing( +def test_hook_resolver_requires_a_hook( find_override_method: mock.Mock, - default_resolver_provider: mock.Mock, + pypi_provider: mock.Mock, tmp_context: WorkContext, + provider: str, + suggested: str, ) -> None: - """A missing hook fails instead of selecting the default PyPI provider.""" - resolver_model = HookSDistResolver( - provider="hook-sdist", build_sdist=BuildSDist.tarball - ) + """A package with neither hook fails and points to the plain profile.""" + r = _parse_hook_profile(provider) - with pytest.raises(ValueError, match="requires a get_resolver_provider"): - resolver_model.resolver_provider(tmp_context, _REQ, _REQ_TYPE) + with pytest.raises( + ValueError, + match=f"get_resolver_provider or download_source.*'{suggested}'", + ): + r.resolver_provider(tmp_context, _REQ, _REQ_TYPE) - find_override_method.assert_called_once_with( - _REQ.name, - "get_resolver_provider", - ) - default_resolver_provider.assert_not_called() + assert find_override_method.call_args_list == [ + mock.call(_REQ.name, "get_resolver_provider"), + mock.call(_REQ.name, "download_source"), + ] + pypi_provider.assert_not_called() @mock.patch("fromager.packagesettings._resolver.overrides.find_override_method") diff --git a/tests/test_sources.py b/tests/test_sources.py index 5b58779e5..26e94ad57 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -185,6 +185,47 @@ def hook( assert result.cooldown.min_age == datetime.timedelta(days=14) +@patch("fromager.packagesettings._resolver.overrides.find_override_method") +def test_get_source_provider_download_only_hook_uses_index_and_cooldown( + find_override_method: Mock, + tmp_context: context.WorkContext, +) -> None: + """A package with only a download hook resolves on the profile index.""" + req = Requirement("test-pkg") + + def download_source() -> pathlib.Path: + return tmp_context.work_dir / "test-pkg-1.0.tar.gz" + + find_override_method.side_effect = lambda distname, method: ( + download_source if method == "download_source" else None + ) + tmp_context.cooldown = Cooldown(min_age=datetime.timedelta(days=7)) + source_resolver = packagesettings.HookSDistResolver( + provider="hook-sdist", + build_sdist=packagesettings.BuildSDist.tarball, + index_url=pydantic.HttpUrl("https://profile.test/simple"), + min_release_age=14, + ) + + with patch.object( + packagesettings.PackageBuildInfo, + "source_resolver", + new_callable=PropertyMock, + return_value=source_resolver, + ): + result = sources.get_source_provider( + ctx=tmp_context, + req=req, + sdist_server_url=resolver.PYPI_SERVER_URL, + req_type=RequirementType.INSTALL, + ) + + assert isinstance(result, resolver.PyPIProvider) + assert result.sdist_server_url == "https://profile.test/simple" + assert result.cooldown is not None + assert result.cooldown.min_age == datetime.timedelta(days=14) + + def test_get_source_provider_forwards_req_type_to_source_resolver( tmp_context: context.WorkContext, ) -> None: From 8809c850d0ca2bb1222163cfa60014ffe036a4aa Mon Sep 17 00:00:00 2001 From: Justin Larkin Date: Wed, 28 Oct 2026 17:19:33 -0400 Subject: [PATCH 17/35] fix(find-updates): honor configured source resolvers Use `sources.get_source_provider()` for packages with source configuration while preserving legacy provider hooks, distribution options, and index overrides. Keep release-age cooldowns out of update reports so recent releases remain visible. Add CLI regression coverage and document behavior. Closes: #1401 Co-Authored-By: GPT Sol Signed-off-by: Justin Larkin --- docs/concepts/resolver-architecture.rst | 7 + src/fromager/commands/find_updates.py | 47 +++-- tests/test_find_updates.py | 246 +++++++++++++++++++++++- 3 files changed, 277 insertions(+), 23 deletions(-) diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index 5293fba30..72e46d492 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -108,6 +108,13 @@ CLI commands interact with providers through a common ``get_resolver_provider`` hook as ``sdist_server_url``. ``hook-sdist`` defaults to PyPI, while ``hook-prebuilt`` requires an explicit index. + ``find-updates`` uses the configured source resolver when a package has + ``source:`` settings. The source profile determines distribution types and + the index URL. Packages without ``source:`` settings retain the legacy + provider hook, distribution options, and index URL overrides. + Update reports ignore global and per-package release-age cooldowns so + recent releases remain visible with either configuration style. + Per-package settings in YAML can select which provider to use and configure its parameters (index URL, tag pattern, etc.). When a package has a ``source:`` resolver configured, that resolver creates the provider. diff --git a/src/fromager/commands/find_updates.py b/src/fromager/commands/find_updates.py index 6c8ecf6a8..bb4b81bd6 100644 --- a/src/fromager/commands/find_updates.py +++ b/src/fromager/commands/find_updates.py @@ -10,7 +10,7 @@ from packaging.requirements import Requirement from packaging.version import Version -from .. import constraints, context, overrides, resolver +from .. import constraints, context, overrides, resolver, sources from . import package logger = logging.getLogger(__name__) @@ -79,6 +79,10 @@ def find_updates( - "csv": CSV format with "name" and "version" columns Use --output to write results to a file instead of stdout. + + Packages with source configuration use their configured resolver. + Distribution type options apply to legacy configuration. + Update reports include recent releases regardless of release-age cooldowns. """ # Load the constraints file constraint_loader = constraints.Constraints() @@ -185,24 +189,33 @@ def _find_newer_versions( """Find versions that would be updates beyond the constraint for a package.""" # Get package build info to determine distribution preferences pbi = wkctx.package_build_info(constraint) - override_sdist_server_url = pbi.resolver_sdist_server_url(sdist_server_url) + if pbi.source_resolver is not None: + provider = sources.get_source_provider( + ctx=wkctx, + req=constraint, + sdist_server_url=sdist_server_url, + ) + # Update discovery includes releases that are too recent to build. + provider.cooldown = None + else: + override_sdist_server_url = pbi.resolver_sdist_server_url(sdist_server_url) - # Determine include flags based on distribution type - include_sdists, include_wheels = package.parse_distribution_option( - distribution_type, pbi - ) + # Determine include flags based on distribution type + include_sdists, include_wheels = package.parse_distribution_option( + distribution_type, pbi + ) - # Get resolver provider - provider = overrides.find_and_invoke( - constraint.name, - "get_resolver_provider", - resolver.default_resolver_provider, - ctx=wkctx, - req=constraint, - include_sdists=include_sdists, - include_wheels=include_wheels, - sdist_server_url=override_sdist_server_url, - ) + # Get resolver provider + provider = overrides.find_and_invoke( + constraint.name, + "get_resolver_provider", + resolver.default_resolver_provider, + ctx=wkctx, + req=constraint, + include_sdists=include_sdists, + include_wheels=include_wheels, + sdist_server_url=override_sdist_server_url, + ) # Create a requirement without version constraints to get all available versions unconstrained_req = Requirement(constraint.name) diff --git a/tests/test_find_updates.py b/tests/test_find_updates.py index e764ea9fe..c50db996a 100644 --- a/tests/test_find_updates.py +++ b/tests/test_find_updates.py @@ -1,7 +1,245 @@ +"""Tests for constraint parsing and update resolution.""" + +import datetime +import pathlib +from unittest.mock import patch + +import pytest +import requests_mock +from click.testing import CliRunner from packaging.requirements import Requirement from packaging.version import Version -from fromager.commands.find_updates import _get_constraint_version +from fromager import context, packagesettings, resolver +from fromager.candidate import Candidate, Cooldown +from fromager.commands.find_updates import ( + _find_newer_versions, + _get_constraint_version, + find_updates, +) + + +@pytest.mark.parametrize( + ("config", "distribution_type", "index_url", "expected_versions"), + [ + ( + "source:\n provider: pypi-sdist\n index_url: https://source.test/simple/\n", + "default", + "https://source.test/simple/", + ["2.0.0"], + ), + ( + "source:\n provider: pypi-prebuilt\n index_url: https://source.test/simple/\n", + "default", + "https://source.test/simple/", + ["3.0.0"], + ), + ( + "source:\n provider: pypi-sdist\n index_url: https://source.test/simple/\n", + "wheel", + "https://source.test/simple/", + ["2.0.0"], + ), + ( + "source:\n provider: pypi-prebuilt\n index_url: https://source.test/simple/\n", + "sdist", + "https://source.test/simple/", + ["3.0.0"], + ), + ( + "source:\n provider: pypi-sdist\n" + "variants:\n cpu:\n source:\n provider: pypi-prebuilt\n" + " index_url: https://source.test/simple/\n", + "default", + "https://source.test/simple/", + ["3.0.0"], + ), + ("", "default", "https://caller.test/simple/", ["2.0.0"]), + *[ + ( + "resolver_dist:\n sdist_server_url: https://legacy.test/simple/\n" + " include_sdists: false\n include_wheels: true\n", + distribution_type, + "https://legacy.test/simple/", + expected_versions, + ) + for distribution_type, expected_versions in [ + ("default", ["3.0.0"]), + ("sdist", ["2.0.0"]), + ("wheel", ["3.0.0"]), + ("both", ["2.0.0", "3.0.0"]), + ] + ], + ], + ids=[ + "source-sdist", + "source-prebuilt", + "source-sdist-overrides-distribution-option", + "source-prebuilt-overrides-distribution-option", + "source-variant", + "legacy-caller-index", + "legacy-default", + "legacy-sdist", + "legacy-wheel", + "legacy-both", + ], +) +def test_find_updates_resolves_configured_package( + config: str, + distribution_type: str, + index_url: str, + expected_versions: list[str], + tmp_context: context.WorkContext, + tmp_path: pathlib.Path, + cli_runner: CliRunner, + requests_mock: requests_mock.Mocker, +) -> None: + """Resolve updates using source profiles or legacy distribution settings.""" + settings_dir = tmp_path / "settings" + settings_dir.mkdir() + (settings_dir / "test-pkg.yaml").write_text(config) + tmp_context.settings = packagesettings.Settings.from_files( + settings_file=tmp_path / "settings.yaml", + settings_dir=settings_dir, + patches_dir=tmp_path / "patches", + variant="cpu", + max_jobs=None, + ) + constraints_file = tmp_path / "constraints.txt" + constraints_file.write_text("test-pkg==1.0.0\n") + resolver.BaseProvider.clear_cache() + requests_mock.get( + f"{index_url}test-pkg/", + text=( + 'test_pkg-1.0.0.tar.gz' + 'test_pkg-2.0.0.tar.gz' + 'test_pkg-3.0.0-py3-none-any.whl' + ), + ) + + result = cli_runner.invoke( + find_updates, + [ + "--sdist-server-url", + "https://caller.test/simple/", + "--distribution-type", + distribution_type, + str(constraints_file), + ], + obj=tmp_context, + ) + + assert result.exit_code == 0, result.output + assert result.output.splitlines() == [ + f"test-pkg=={version}" for version in expected_versions + ] + assert requests_mock.call_count == 1 + + +@pytest.mark.parametrize("source_provider", [None, "pypi-sdist", "pypi-prebuilt"]) +@pytest.mark.parametrize( + ("global_days", "package_days"), + [(7, None), (None, 7)], + ids=["global-cooldown", "package-cooldown"], +) +@pytest.mark.parametrize( + "include_old_release", [True, False], ids=["mixed", "all-recent"] +) +def test_find_updates_ignores_release_age_cooldown( + source_provider: str | None, + global_days: int | None, + package_days: int | None, + include_old_release: bool, + tmp_context: context.WorkContext, + tmp_path: pathlib.Path, + cli_runner: CliRunner, + requests_mock: requests_mock.Mocker, + caplog: pytest.LogCaptureFixture, +) -> None: + """Report recent updates with either source or legacy configuration.""" + profile: dict[str, str | int] = {} + if source_provider is not None: + profile["provider"] = source_provider + if package_days is not None: + profile["min_release_age"] = package_days + package_settings = packagesettings.PackageSettings.from_mapping( + "test-pkg", + {"source" if source_provider is not None else "resolver_dist": profile}, + source="test", + has_config=True, + ) + tmp_context.settings = packagesettings.Settings( + settings=packagesettings.SettingsFile(), + package_settings=[package_settings], + patches_dir=tmp_path / "patches", + variant="cpu", + max_jobs=None, + ) + now = datetime.datetime.now(datetime.UTC) + if global_days is not None: + tmp_context.cooldown = Cooldown( + min_age=datetime.timedelta(days=global_days), + bootstrap_time=now, + ) + constraints_file = tmp_path / "constraints.txt" + constraints_file.write_text("test-pkg==1.0.0\n") + suffix = "-py3-none-any.whl" if source_provider == "pypi-prebuilt" else ".tar.gz" + releases = [("2.0.0", 1)] + if include_old_release: + releases.append(("1.0.0", 30)) + resolver.BaseProvider.clear_cache() + requests_mock.get( + "https://pypi.org/simple/test-pkg/", + headers={"Content-Type": "application/vnd.pypi.simple.v1+json"}, + json={ + "meta": {"api-version": "1.1"}, + "name": "test-pkg", + "files": [ + { + "filename": f"test_pkg-{version}{suffix}", + "url": f"https://files.test/test_pkg-{version}{suffix}", + "hashes": {}, + "upload-time": (now - datetime.timedelta(days=days)).isoformat(), + } + for version, days in releases + ], + }, + ) + + result = cli_runner.invoke(find_updates, [str(constraints_file)], obj=tmp_context) + + assert result.exit_code == 0, result.output + assert result.output == "test-pkg==2.0.0\n" + assert "Could not check versions" not in caplog.text + assert "cooldown blocked" not in caplog.text + + +def test_find_newer_versions_preserves_legacy_provider_hook( + tmp_context: context.WorkContext, +) -> None: + """Continue resolving legacy packages through their provider override.""" + candidate = Candidate( + name="test-pkg", + version=Version("2.0.0"), + url="https://hook.test/test_pkg-2.0.0.tar.gz", + ) + provider = resolver.GenericProvider( + version_source=lambda identifier: [candidate], + use_resolver_cache=False, + ) + with patch( + "fromager.overrides.find_override_method", + return_value=lambda **kwargs: provider, + ): + versions = _find_newer_versions( + tmp_context, + Requirement("test-pkg==1.0.0"), + Version("1.0.0"), + "default", + "https://caller.test/simple/", + ) + + assert versions == [Version("2.0.0")] class TestGetConstraintVersion: @@ -77,13 +315,9 @@ def test_invalid_version_format(self) -> None: """Test that invalid version formats return None.""" # We need to test the case where Version() construction fails # Let's create a requirement and then mock the Version parsing - import unittest.mock - req = Requirement("package==1.0.0") - with unittest.mock.patch( - "fromager.commands.find_updates.Version" - ) as mock_version: + with patch("fromager.commands.find_updates.Version") as mock_version: mock_version.side_effect = ValueError("Invalid version") result = _get_constraint_version(req) assert result is None From 87cbfb935583c620d2f315fce634b893108beb57 Mon Sep 17 00:00:00 2001 From: Shanmukh Pawan Date: Wed, 28 Oct 2026 18:48:44 -0400 Subject: [PATCH 18/35] fix(download-sequence): replay configured source downloads Record the effective source provider in build order and validate it before sequence downloads. Use the configured resolver for source entries while preserving legacy and runtime fallback behavior. Closes: #1209 Co-Authored-By: Codex Sol 6 Signed-off-by: Shanmukh Pawan --- docs/concepts/resolver-architecture.rst | 22 ++ docs/reference/files.md | 34 ++ src/fromager/bootstrapper/_bootstrapper.py | 5 +- src/fromager/commands/download_sequence.py | 74 +++- src/fromager/requirements_file.py | 2 + src/fromager/sources.py | 8 +- tests/test_bootstrapper.py | 29 ++ tests/test_commands.py | 439 ++++++++++++++++++++- tests/test_sources.py | 31 ++ 9 files changed, 620 insertions(+), 24 deletions(-) diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index c8c9990e6..93bc7e210 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -159,6 +159,28 @@ The separate ``step prepare-source`` invocation accepts ``--source-kind`` and ``step download-source-archive --output-format json`` exposes that result's actual path and kind without changing its default path-only output. Without ``--source-kind``, it keeps the legacy archive behavior. +``download-sequence`` calls the configured resolver's normal ``download()`` +for source entries. Before downloading, it checks ``source_provider`` against +the current provider for each source entry and each ``prebuilt: true`` wheel +requested with ``--include-wheels``. A missing or different provider fails +before any downloads start. Older build orders without ``source_provider`` +still work with legacy settings; regenerate them before replaying a configured +source. This checks the provider name, so replay should still use the same +package settings. A runtime ``prebuilt`` entry is skipped even when the +package has source settings. +Built-in archive resolvers cache the resolved file for a later build. Git +checkout resolvers clone into the work directory instead: with cleanup enabled, +a later build removes that checkout and clones again; without cleanup, the +later download fails because the checkout exists. Hook +download behavior depends on the hook. Legacy packages keep the +``source_url_type`` selection behavior. Build-order entries record the +effective configured ``source_provider`` for provenance; the +``source_url_type`` field remains a legacy compatibility classification, not +the actual downloaded artifact kind. + +.. versionchanged:: 0.102.0 + ``download-sequence`` uses configured source resolvers for source entries. + Configured prebuilt resolvers select the prebuilt bootstrap path. Fresh external wheels use their ``download()`` contract and are staged in the prebuilt wheel directory. For local mirror URLs, bootstrap, build, and diff --git a/docs/reference/files.md b/docs/reference/files.md index 117377c8c..93f95a89d 100644 --- a/docs/reference/files.md +++ b/docs/reference/files.md @@ -67,6 +67,40 @@ The following example shows `build-order.json` file for the top-level dependency ] ``` +`source_url_type` is a legacy compatibility classification (`sdist`, +`override`, or `prebuilt`). It does not report the artifact returned by a +configured source resolver. Entries using `source:` also contain +`source_provider`, the effective package or variant provider name. This names +the configured provider even when a cached wheel avoids a source download. +`pypi-sdist` is also labeled `override` here because it uses `source:`, even +though its download result is an sdist. +For example, a configured Git source is recorded as: + +```json +{ + "source_url": "git+https://github.com/example/pkg@v1.0", + "source_url_type": "override", + "source_provider": "github-tag-git" +} +``` + +`download-sequence` calls the configured resolver's normal `download()` for +these entries. Before downloading, it checks the recorded provider name for +each source entry and each `prebuilt: true` wheel requested with `--include-wheels`. +A missing or different provider is an error. Older build orders without +`source_provider` still work with legacy settings; regenerate them before +replaying a configured source. The check compares only provider names, so +replay should still use the same package settings. Archive +downloads can be reused from the download cache. Git downloads create a +checkout in the work directory and are not cached for a later build. The command +also uses `source_url_type` to skip entries that used a prebuilt fallback at +runtime, even if source settings are configured. + +```{versionchanged} 0.102.0 +Configured source providers are recorded in build order, and +`download-sequence` uses their download operation. +``` + ## graph.json The `graph.json` is an output file that contains all the paths fromager can take to resolve a dependency during building the wheel. The graph represents all of the dependencies encountered during a build of a wheel collection ("collection" is a new term we're introducing). It shows the parent/child relationships of all dependencies, including the type of dependency. diff --git a/src/fromager/bootstrapper/_bootstrapper.py b/src/fromager/bootstrapper/_bootstrapper.py index 0cdeb5cdf..51ce54210 100644 --- a/src/fromager/bootstrapper/_bootstrapper.py +++ b/src/fromager/bootstrapper/_bootstrapper.py @@ -716,7 +716,7 @@ def add_to_build_order( req: The requirement being added to the build order. version: The resolved version. source_url: URL from which the source distribution was obtained. - source_type: The kind of source (sdist, wheel, git, etc.). + source_type: Legacy source classification for compatibility. prebuilt: True if the wheel was taken from a prebuilt cache rather than compiled locally. constraint: The version constraint active for this package, if any. @@ -740,6 +740,9 @@ def add_to_build_order( "source_url": source_url, "source_url_type": str(source_type), } + source_resolver = self.ctx.package_build_info(req).source_resolver + if source_resolver is not None: + info["source_provider"] = source_resolver.provider self._build_stack.append(info) def _check_write_error(self, fut: concurrent.futures.Future[int]) -> None: diff --git a/src/fromager/commands/download_sequence.py b/src/fromager/commands/download_sequence.py index 43876263b..2f54597a1 100644 --- a/src/fromager/commands/download_sequence.py +++ b/src/fromager/commands/download_sequence.py @@ -13,6 +13,14 @@ logger = logging.getLogger(__name__) +def _skip_recorded_url(entry: dict[str, typing.Any], include_wheels: bool) -> bool: + """Return whether a prebuilt path leaves the saved URL unused.""" + if entry["prebuilt"]: + return not include_wheels + source_url_type: str = entry["source_url_type"] + return source_url_type == "prebuilt" + + @click.command() @click.argument("build_order_file") @click.argument("sdist_server_url") @@ -41,14 +49,16 @@ def download_sequence( ignore_missing_sdists: bool, num_threads: int, ) -> None: - """Download a sequence of source distributions in order. + """Download sources and optional wheels from a build order. BUILD_ORDER_FILE is the build-order.json files to build - SDIST_SERVER_URL is the URL for a PyPI-compatible package index hosting sdists + SDIST_SERVER_URL is a PyPI-compatible index used for optional wheel lookup - Performs the equivalent of the 'step download-source-archive' command only for items in - the build order file that have source_url_type as sdist. + Configured sources use their resolver's normal download operation. Entries + that download a configured source or prebuilt wheel require the recorded + provider to match current settings. Legacy entries retain the + ``source_url_type`` selection behavior. """ # The local wheel server is a trusted cache (wheels built in this or @@ -63,11 +73,30 @@ def download_sequence( with read.open_file_or_url(build_order_file) as f: build_order = json.load(f) + # Check recorded providers before workers can change the download cache + # or a Git work directory. Skipped URLs do not use a source resolver. + for entry in build_order: + if _skip_recorded_url(entry, include_wheels): + continue + recorded_provider = entry.get("source_provider") + req = Requirement(f"{entry['dist']}=={entry['version']}") + source_resolver = wkctx.package_build_info(req).source_resolver + current_provider = ( + source_resolver.provider if source_resolver is not None else None + ) + if recorded_provider != current_provider: + raise ValueError( + f"{req}: build order records source provider {recorded_provider!r}, " + f"current settings select {current_provider!r}; " + "use matching settings or regenerate the build order" + ) + def download_one(entry: dict[str, typing.Any]) -> None: req = Requirement(f"{entry['dist']}=={entry['version']}") with req_ctxvar_context(req): + skip_recorded_url = _skip_recorded_url(entry, include_wheels) if entry["prebuilt"]: - if include_wheels: + if not skip_recorded_url: wheels.download_prebuilt_wheel( ctx=wkctx, req=req, @@ -79,24 +108,33 @@ def download_one(entry: dict[str, typing.Any]) -> None: logger.info(f"{entry['dist']}: uses a pre-built wheel, skipping") return - if entry["source_url_type"] == "sdist": + source_url_type = entry["source_url_type"] + if skip_recorded_url: + logger.info( + "%s: used a pre-built wheel, skipping source", entry["dist"] + ) + else: + source_resolver = wkctx.package_build_info(req).source_resolver try: - sources.download_source( - ctx=wkctx, - req=req, - version=Version(entry["version"]), - download_url=entry["source_url"], - ) + if source_resolver is not None or source_url_type == "sdist": + sources.download_source( + ctx=wkctx, + req=req, + version=Version(entry["version"]), + download_url=entry["source_url"], + ) + else: + logger.info( + "%s: uses a %s downloader, skipping", + entry["dist"], + source_url_type, + ) except Exception as err: - logger.error(f"failed to download sdist for {req}: {err}") + logger.error("failed to download source for %s: %s", req, err) if not ignore_missing_sdists: raise RuntimeError( - f"Failed to download sdist for {req}" + f"Failed to download source for {req}" ) from err - else: - logger.info( - f"{entry['dist']}: uses a {entry['source_url_type']} downloader, skipping" - ) if include_wheels: try: diff --git a/src/fromager/requirements_file.py b/src/fromager/requirements_file.py index c7beed399..3c9c7efef 100644 --- a/src/fromager/requirements_file.py +++ b/src/fromager/requirements_file.py @@ -30,6 +30,8 @@ def is_install_requirement(self) -> bool: class SourceType(StrEnum): + """Legacy build-order classification, not the downloaded artifact kind.""" + PREBUILT = "prebuilt" SDIST = "sdist" OVERRIDE = "override" diff --git a/src/fromager/sources.py b/src/fromager/sources.py index 9cf8e368d..f7041415a 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -68,15 +68,17 @@ def download_url( def get_source_type(ctx: context.WorkContext, req: Requirement) -> SourceType: - source_type = SourceType.SDIST + """Return the legacy source classification for a build-order entry.""" pbi = ctx.package_build_info(req) + if pbi.source_resolver is not None: + return SourceType.OVERRIDE if ( overrides.find_override_method(req.name, "download_source") or overrides.find_override_method(req.name, "get_resolver_provider") or pbi.download_source_url(resolve_template=False) ): - source_type = SourceType.OVERRIDE - return source_type + return SourceType.OVERRIDE + return SourceType.SDIST @metrics.timeit(description="download source") diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index 37bd66c50..4f84c64c5 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -410,6 +410,35 @@ def test_build_order(tmp_context: WorkContext) -> None: assert expected == contents +def test_build_order_records_effective_variant_source_provider( + tmp_context: WorkContext, +) -> None: + """Record the active variant's source provider, including prebuilt rows.""" + _configure_package( + tmp_context, + "source:\n" + " provider: pypi-sdist\n" + "variants:\n" + " cpu:\n" + " source:\n" + " provider: pypi-prebuilt\n", + ) + bt = bootstrapper.Bootstrapper(tmp_context) + bt.add_to_build_order( + req=Requirement("pkg==1.0"), + version=Version("1.0"), + source_url="https://pkg.test/pkg-1.0-py3-none-any.whl", + source_type=SourceType.PREBUILT, + prebuilt=True, + ) + bt.finalize() + + build_order = json.loads(bt._build_order_filename.read_text()) + + assert build_order[0]["source_provider"] == "pypi-prebuilt" + assert build_order[0]["source_url_type"] == "prebuilt" + + def test_build_order_repeats(tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) bt.add_to_build_order( diff --git a/tests/test_commands.py b/tests/test_commands.py index a29c2790b..c7a09ce1b 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,12 +1,15 @@ import concurrent.futures +import io import json import pathlib import subprocess +import tarfile import typing from unittest.mock import Mock, patch import click import pytest +import requests_mock from click.testing import CliRunner from packaging.requirements import Requirement from packaging.version import Version @@ -195,7 +198,15 @@ def test_download_sequence_replays_configured_prebuilt( build_order = tmp_path / "build-order.json" build_order.write_text( json.dumps( - [{"dist": "pkg", "version": "1.0", "prebuilt": True, "source_url": url}] + [ + { + "dist": "pkg", + "version": "1.0", + "prebuilt": True, + "source_url": url, + "source_provider": "hook-prebuilt", + } + ] ) ) with ( @@ -225,6 +236,421 @@ def test_download_sequence_replays_configured_prebuilt( legacy.assert_not_called() +@pytest.mark.parametrize("include_wheels", [False, True]) +def test_download_sequence_checks_prebuilt_provider_before_download( + cli_runner: CliRunner, + tmp_context: WorkContext, + tmp_path: pathlib.Path, + include_wheels: bool, +) -> None: + """A configured wheel download requires its recorded provider.""" + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [ + { + "dist": "pkg", + "version": "1.0", + "prebuilt": True, + "source_url": "https://pkg.test/pkg-1.0-py3-none-any.whl", + "source_provider": "hook-prebuilt", + } + ] + ) + ) + args = [str(build_order), "https://index.test/simple", "--num-threads", "1"] + if include_wheels: + args.append("--include-wheels") + + with patch.object(wheels, "download_prebuilt_wheel") as download: + result = cli_runner.invoke( + download_sequence.download_sequence, args, obj=tmp_context + ) + + assert (result.exit_code != 0) is include_wheels + if include_wheels: + assert "hook-prebuilt" in str(result.exception) + download.assert_not_called() + + +@pytest.mark.parametrize( + ("settings_yaml", "url"), + [ + ( + "source:\n provider: pypi-sdist\n", + "https://pkg.test/pkg-1.0.tar.gz", + ), + ( + "source:\n" + " provider: pypi-download\n" + " download_url: https://pkg.test/pkg-{version}.tar.gz\n" + " download_kind: tarball\n", + "https://pkg.test/pkg-1.0.tar.gz", + ), + ( + "source:\n" + " provider: github-tag-download\n" + " project_url: https://github.test/org/pkg\n", + "https://api.github.test/repos/org/pkg/tarball/v1.0", + ), + ( + "source:\n" + " provider: gitlab-tag-download\n" + " project_url: https://gitlab.test/org/pkg\n", + "https://gitlab.test/org/pkg/-/archive/v1.0/pkg-v1.0.tar.gz", + ), + ], +) +def test_download_sequence_prefetches_configured_archives( + cli_runner: CliRunner, + tmp_context: WorkContext, + tmp_path: pathlib.Path, + requests_mock: requests_mock.Mocker, + settings_yaml: str, + url: str, +) -> None: + """A later resolver download reuses the archive prefetched by the command.""" + _configure_package(tmp_context, settings_yaml) + source_resolver = tmp_context.package_build_info( + Requirement("pkg==1.0") + ).source_resolver + assert source_resolver is not None + archive = io.BytesIO() + with tarfile.open(fileobj=archive, mode="w:gz") as tar: + metadata = b"Name: pkg\nVersion: 1.0\n" + member = tarfile.TarInfo("pkg-1.0/PKG-INFO") + member.size = len(metadata) + tar.addfile(member, io.BytesIO(metadata)) + requests_mock.get(url, content=archive.getvalue()) + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [ + { + "dist": "pkg", + "version": "1.0", + "prebuilt": False, + "source_url": url, + "source_url_type": "override", + "source_provider": source_resolver.provider, + } + ] + ) + ) + + result = cli_runner.invoke( + download_sequence.download_sequence, + [str(build_order), "https://index.test/simple", "--num-threads", "1"], + obj=tmp_context, + ) + + assert result.exit_code == 0, result.output + repeated = cli_runner.invoke( + download_sequence.download_sequence, + [str(build_order), "https://index.test/simple", "--num-threads", "1"], + obj=tmp_context, + ) + assert repeated.exit_code == 0, repeated.output + cached = tmp_context.sdists_downloads / "pkg-1.0.tar.gz" + assert cached.exists() + downloaded = source_resolver.download( + tmp_context, Requirement("pkg==1.0"), Version("1.0"), url + ) + assert downloaded.path == cached + assert requests_mock.call_count == 1 + + +def test_download_sequence_calls_configured_git_download( + cli_runner: CliRunner, + tmp_context: WorkContext, + tmp_path: pathlib.Path, +) -> None: + """A configured Git resolver downloads even with a legacy override label.""" + _configure_package( + tmp_context, + "source:\n" + " provider: github-tag-git\n" + " project_url: https://github.test/org/pkg\n", + ) + url = "git+https://github.test/org/pkg@v1.0" + source_resolver = tmp_context.package_build_info( + Requirement("pkg==1.0") + ).source_resolver + assert source_resolver is not None + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [ + { + "dist": "pkg", + "version": "1.0", + "prebuilt": False, + "source_url": url, + "source_url_type": "override", + "source_provider": source_resolver.provider, + } + ] + ) + ) + checkout = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" + with patch( + "fromager.downloads.download_git_source", return_value=checkout + ) as clone: + result = cli_runner.invoke( + download_sequence.download_sequence, + [str(build_order), "https://index.test/simple", "--num-threads", "1"], + obj=tmp_context, + ) + + assert result.exit_code == 0, result.output + clone.assert_called_once_with(destination_dir=checkout, vcs_url=url) + + +def test_download_sequence_calls_configured_hook_download( + cli_runner: CliRunner, + tmp_context: WorkContext, + tmp_path: pathlib.Path, +) -> None: + """A hook source uses its normal download result during sequence replay.""" + _configure_package( + tmp_context, "source:\n provider: hook-sdist\n build_sdist: tarball\n" + ) + url = "https://pkg.test/pkg-1.0.tar.gz" + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [ + { + "dist": "pkg", + "version": "1.0", + "prebuilt": False, + "source_url": url, + "source_url_type": "override", + "source_provider": "hook-sdist", + } + ] + ) + ) + calls: list[tuple[Version, str]] = [] + + def hook(version: Version, download_url: str) -> packagesettings.DownloadedSource: + calls.append((version, download_url)) + return packagesettings.DownloadedSource( + path=tmp_context.work_dir / "pkg-1.0" / "pkg-1.0", + kind=packagesettings.DownloadKind.git_checkout, + ) + + with patch("fromager.overrides.find_override_method", return_value=hook): + result = cli_runner.invoke( + download_sequence.download_sequence, + [str(build_order), "https://index.test/simple", "--num-threads", "1"], + obj=tmp_context, + ) + + assert result.exit_code == 0, result.output + assert calls == [(Version("1.0"), url)] + + +def test_download_sequence_keeps_legacy_sdist_selection( + cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path +) -> None: + """Old build-order entries still prefetch through the legacy downloader.""" + url = "https://pkg.test/pkg-1.0.tar.gz" + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [ + { + "dist": "pkg", + "version": "1.0", + "prebuilt": False, + "source_url": url, + "source_url_type": "sdist", + } + ] + ) + ) + with patch.object(download_sequence.sources, "download_source") as download: + result = cli_runner.invoke( + download_sequence.download_sequence, + [str(build_order), "https://index.test/simple", "--num-threads", "1"], + obj=tmp_context, + ) + + assert result.exit_code == 0, result.output + download.assert_called_once_with( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + version=Version("1.0"), + download_url=url, + ) + + +@pytest.mark.parametrize( + ("configured_provider", "recorded_provider", "ignore_missing"), + [ + (None, "pypi-sdist", False), + (None, "pypi-sdist", True), + ("pypi-sdist", "github-tag-git", False), + ], +) +def test_download_sequence_rejects_missing_or_changed_source_provider( + cli_runner: CliRunner, + tmp_context: WorkContext, + tmp_path: pathlib.Path, + configured_provider: str | None, + recorded_provider: str, + ignore_missing: bool, +) -> None: + """A configured build order must not silently use different settings.""" + if configured_provider is not None: + _configure_package(tmp_context, f"source:\n provider: {configured_provider}\n") + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [ + { + "dist": "pkg", + "version": "1.0", + "prebuilt": False, + "source_url": "https://pkg.test/pkg-1.0.tar.gz", + "source_url_type": "override", + "source_provider": recorded_provider, + } + ] + ) + ) + + with patch.object(download_sequence.sources, "download_source") as download: + args = [str(build_order), "https://index.test/simple", "--num-threads", "1"] + if ignore_missing: + args.append("--ignore-missing-sdists") + result = cli_runner.invoke( + download_sequence.download_sequence, + args, + obj=tmp_context, + ) + + assert result.exit_code != 0 + assert recorded_provider in str(result.exception) + download.assert_not_called() + + +@pytest.mark.parametrize("ignore_missing", [False, True]) +def test_download_sequence_rejects_unrecorded_configured_provider( + cli_runner: CliRunner, + tmp_context: WorkContext, + tmp_path: pathlib.Path, + ignore_missing: bool, +) -> None: + """Old entries cannot use a configured resolver without provenance.""" + _configure_package( + tmp_context, + "source:\n" + " provider: github-tag-git\n" + " project_url: https://github.test/org/pkg\n", + ) + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [ + { + "dist": "pkg", + "version": "1.0", + "prebuilt": False, + "source_url": "https://pkg.test/pkg-1.0.tar.gz", + "source_url_type": "sdist", + } + ] + ) + ) + args = [str(build_order), "https://index.test/simple", "--num-threads", "1"] + if ignore_missing: + args.append("--ignore-missing-sdists") + + with patch.object(download_sequence.sources, "download_source") as download: + result = cli_runner.invoke( + download_sequence.download_sequence, args, obj=tmp_context + ) + + assert isinstance(result.exception, ValueError) + assert "github-tag-git" in str(result.exception) + download.assert_not_called() + + +def test_download_sequence_checks_providers_before_starting_downloads( + cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path +) -> None: + """A later mismatch must fail before an earlier entry downloads.""" + _configure_package(tmp_context, "source:\n provider: pypi-sdist\n") + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [ + { + "dist": "pkg", + "version": "1.0", + "prebuilt": False, + "source_url": "https://pkg.test/pkg-1.0.tar.gz", + "source_url_type": "override", + "source_provider": "pypi-sdist", + }, + { + "dist": "other", + "version": "1.0", + "prebuilt": False, + "source_url": "https://pkg.test/other-1.0.tar.gz", + "source_url_type": "override", + "source_provider": "pypi-sdist", + }, + ] + ) + ) + + with patch.object(download_sequence.sources, "download_source") as download: + result = cli_runner.invoke( + download_sequence.download_sequence, + [str(build_order), "https://index.test/simple", "--num-threads", "1"], + obj=tmp_context, + ) + + assert result.exit_code != 0 + assert "other==1.0" in str(result.exception) + download.assert_not_called() + + +def test_download_sequence_skips_test_mode_prebuilt_fallback_source( + cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path +) -> None: + """A runtime wheel fallback must not retry its failed source download.""" + _configure_package(tmp_context, "source:\n provider: pypi-sdist\n") + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [ + { + "dist": "pkg", + "version": "1.0", + "prebuilt": False, + "source_url": "https://pkg.test/pkg-1.0.tar.gz", + "source_url_type": "prebuilt", + "source_provider": "pypi-sdist", + } + ] + ) + ) + + with patch.object(download_sequence.sources, "download_source") as download: + result = cli_runner.invoke( + download_sequence.download_sequence, + [str(build_order), "https://index.test/simple", "--num-threads", "1"], + obj=tmp_context, + ) + + assert result.exit_code == 0, result.output + download.assert_not_called() + + def test_download_sequence_local_mirror_url_skips_download_hook( cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path ) -> None: @@ -235,7 +661,15 @@ def test_download_sequence_local_mirror_url_skips_download_hook( build_order = tmp_path / "build-order.json" build_order.write_text( json.dumps( - [{"dist": "pkg", "version": "1.0", "prebuilt": True, "source_url": url}] + [ + { + "dist": "pkg", + "version": "1.0", + "prebuilt": True, + "source_url": url, + "source_provider": "hook-prebuilt", + } + ] ) ) with ( @@ -325,6 +759,7 @@ def test_download_sequence_rejects_stale_prebuilt_flag( "version": "1.0", "prebuilt": True, "source_url": "https://pkg.test/pkg-1.0-py3-none-any.whl", + "source_provider": "pypi-sdist", } ] ) diff --git a/tests/test_sources.py b/tests/test_sources.py index 26e94ad57..29dfd4042 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -834,6 +834,37 @@ def test_get_source_type_sdist( assert result == sources.SourceType.SDIST +def test_get_source_type_configured_git_is_override( + tmp_context: context.WorkContext, +) -> None: + """A configured Git source must not be labeled as an sdist.""" + ps = packagesettings.PackageSettings.from_string( + "pkg", + "source:\n" + " provider: github-tag-git\n" + " project_url: https://github.test/org/pkg\n", + ) + tmp_context.settings._package_settings[ps.name] = ps + + result = sources.get_source_type(tmp_context, Requirement("pkg==1.0")) + + assert result == sources.SourceType.OVERRIDE + + +def test_get_source_type_configured_pypi_sdist_is_override( + tmp_context: context.WorkContext, +) -> None: + """The legacy label identifies configured routing, not artifact kind.""" + ps = packagesettings.PackageSettings.from_string( + "pkg", "source:\n provider: pypi-sdist\n" + ) + tmp_context.settings._package_settings[ps.name] = ps + + result = sources.get_source_type(tmp_context, Requirement("pkg==1.0")) + + assert result == sources.SourceType.OVERRIDE + + def test_unpack_source_tar_gz( tmp_context: context.WorkContext, tmp_path: pathlib.Path, From fc1f46469c3fcefe4d860fc3e19a1496fd48dc98 Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Fri, 30 Oct 2026 10:29:39 -0400 Subject: [PATCH 19/35] test(e2e): build packages with configured sources end to end Add an e2e test that runs `bootstrap`, `build`, and `build-sequence` for a package with a `source:` resolver, with no mocks. - `github-tag-git`: resolves stevedore tags on GitHub, builds from a git checkout, records the provider and git URL in build order, and replays the saved URL with `build-sequence`. - `pypi-download`: resolves on PyPI and downloads a tarball from a URL template. Both `build` runs pass an unreachable caller URL to show that the configured resolver uses its own index. Co-Authored-By: Claude Closes: #1403 Signed-off-by: Lalatendu Mohanty --- e2e/ci_build_suite.sh | 1 + e2e/source_settings_archive/stevedore.yaml | 4 + e2e/source_settings_git/stevedore.yaml | 4 + e2e/test_build_source_config.sh | 144 +++++++++++++++++++++ 4 files changed, 153 insertions(+) create mode 100644 e2e/source_settings_archive/stevedore.yaml create mode 100644 e2e/source_settings_git/stevedore.yaml create mode 100755 e2e/test_build_source_config.sh diff --git a/e2e/ci_build_suite.sh b/e2e/ci_build_suite.sh index a0b270d16..04bdcc3b2 100755 --- a/e2e/ci_build_suite.sh +++ b/e2e/ci_build_suite.sh @@ -15,6 +15,7 @@ test_section "core build tests" run_test "build" run_test "build_order" run_test "build_steps" +run_test "build_source_config" test_section "advanced build tests" run_test "build_parallel" diff --git a/e2e/source_settings_archive/stevedore.yaml b/e2e/source_settings_archive/stevedore.yaml new file mode 100644 index 000000000..3b7a475f2 --- /dev/null +++ b/e2e/source_settings_archive/stevedore.yaml @@ -0,0 +1,4 @@ +source: + provider: pypi-download + download_url: https://files.pythonhosted.org/packages/e7/c1/b210bf1071c96ecfcd24c2eeb4c828a2a24bf74b38af13896d02203b1eec/stevedore-{version}.tar.gz + download_kind: tarball diff --git a/e2e/source_settings_git/stevedore.yaml b/e2e/source_settings_git/stevedore.yaml new file mode 100644 index 000000000..da8a5e8aa --- /dev/null +++ b/e2e/source_settings_git/stevedore.yaml @@ -0,0 +1,4 @@ +source: + provider: github-tag-git + project_url: https://github.com/python-wheel-build/stevedore-test-repo + build_sdist: pep517 diff --git a/e2e/test_build_source_config.sh b/e2e/test_build_source_config.sh new file mode 100755 index 000000000..57640f00a --- /dev/null +++ b/e2e/test_build_source_config.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- + +# Test that bootstrap, build, and build-sequence use a configured +# `source:` resolver end-to-end: resolve, download, prepare, and build. +# +# The git profile resolves stevedore tags on GitHub and builds from a +# git checkout. The archive profile resolves on PyPI and downloads a +# tarball from a URL template. + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$SCRIPTDIR/common.sh" + +# What are we building? +DIST="stevedore" +VERSION="5.2.0" +WHEEL="wheels-repo/downloads/${DIST}-${VERSION}-0-py3-none-any.whl" +BUILT_SDIST="sdists-repo/builds/${DIST}-${VERSION}.tar.gz" + +GIT_SETTINGS="$SCRIPTDIR/source_settings_git" +ARCHIVE_SETTINGS="$SCRIPTDIR/source_settings_archive" + +pass=true + +remove_dist() { + rm -rf "$OUTDIR/wheels-repo/downloads/${DIST}"-* + rm -rf "$OUTDIR/wheels-repo/simple/${DIST}" + rm -rf "$OUTDIR/sdists-repo/downloads/${DIST}"-* + rm -rf "$OUTDIR/sdists-repo/builds/${DIST}"-* + rm -rf "$OUTDIR/work-dir/${DIST}"-* +} + +expect_file() { + if [ ! -f "$OUTDIR/$1" ]; then + echo "FAIL: Did not find $OUTDIR/$1" 1>&2 + pass=false + fi +} + +expect_no_source_download() { + if [ -n "$(find "$OUTDIR/sdists-repo/downloads/" -name "${DIST}-*")" ]; then + echo "FAIL: git profile downloaded a source archive for ${DIST}" 1>&2 + pass=false + fi +} + +expect_ignored_index_url() { + if ! grep -q "ignoring index URL unreachable_url" "$1"; then + echo "FAIL: configured resolver did not ignore the caller URL in $1" 1>&2 + pass=false + fi +} + +# 1. Bootstrap with the git profile. This also builds the build +# dependencies that the later steps install from the local wheel server. +fromager \ + $NETWORK_ISOLATION \ + --log-file="$OUTDIR/bootstrap.log" \ + --sdists-repo="$OUTDIR/sdists-repo" \ + --wheels-repo="$OUTDIR/wheels-repo" \ + --work-dir="$OUTDIR/work-dir" \ + --settings-dir="$GIT_SETTINGS" \ + bootstrap "${DIST}==${VERSION}" + +expect_file "$WHEEL" + +# The build order records the configured provider and the git URL. +if ! python3 - "$OUTDIR/work-dir/build-order.json" "$DIST" <<'EOF' +import json +import sys + +path, dist = sys.argv[1:] +with open(path) as f: + entries = [e for e in json.load(f) if e["dist"] == dist] +assert len(entries) == 1, entries +entry = entries[0] +assert entry["source_provider"] == "github-tag-git", entry +assert entry["source_url_type"] == "override", entry +assert entry["source_url"].startswith( + "git+https://github.com/python-wheel-build/stevedore-test-repo@" +), entry +EOF +then + echo "FAIL: build order does not record the configured git source" 1>&2 + pass=false +fi + +# 2. Build with the git profile. The caller URL is unreachable, so the +# configured resolver must use its own index. +remove_dist +fromager \ + --log-file="$OUTDIR/build-git.log" \ + --sdists-repo="$OUTDIR/sdists-repo" \ + --wheels-repo="$OUTDIR/wheels-repo" \ + --work-dir="$OUTDIR/work-dir" \ + --settings-dir="$GIT_SETTINGS" \ + build "${DIST}" "${VERSION}" "unreachable_url" + +expect_file "$WHEEL" +expect_file "$BUILT_SDIST" +expect_no_source_download +expect_ignored_index_url "$OUTDIR/build-git.log" + +# 3. Replay the saved git URL with build-sequence. Only the stevedore +# entry is kept, so the build dependencies are not rebuilt. +remove_dist +python3 - "$OUTDIR/work-dir/build-order.json" "$OUTDIR/source-build-order.json" "$DIST" <<'EOF' +import json +import sys + +src, dest, dist = sys.argv[1:] +with open(src) as f: + entries = [e for e in json.load(f) if e["dist"] == dist] +with open(dest, "w") as f: + json.dump(entries, f) +EOF +fromager \ + --log-file="$OUTDIR/build-sequence-git.log" \ + --sdists-repo="$OUTDIR/sdists-repo" \ + --wheels-repo="$OUTDIR/wheels-repo" \ + --work-dir="$OUTDIR/work-dir" \ + --settings-dir="$GIT_SETTINGS" \ + build-sequence "$OUTDIR/source-build-order.json" + +expect_file "$WHEEL" +expect_file "$BUILT_SDIST" +expect_no_source_download + +# 4. Build with the archive profile. The tarball gets the standard name. +remove_dist +fromager \ + --log-file="$OUTDIR/build-archive.log" \ + --sdists-repo="$OUTDIR/sdists-repo" \ + --wheels-repo="$OUTDIR/wheels-repo" \ + --work-dir="$OUTDIR/work-dir" \ + --settings-dir="$ARCHIVE_SETTINGS" \ + build "${DIST}" "${VERSION}" "unreachable_url" + +expect_file "$WHEEL" +expect_file "$BUILT_SDIST" +expect_file "sdists-repo/downloads/${DIST}-${VERSION}.tar.gz" +expect_ignored_index_url "$OUTDIR/build-archive.log" + +$pass From 12e0902df8d0a9c77f79c4dc40b70faee571a7cb Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Fri, 30 Oct 2026 11:25:34 -0400 Subject: [PATCH 20/35] docs: target 0.103.0 in version notes 0.102.0 was released from `main` without the new resolver config, so the feature branch's version notes must name the next release. - Change every `versionadded` and `versionchanged` added on this branch (0.95.0, 0.96.0 and 0.102.0) to 0.103.0. - Merge notes that now sit next to each other with the same version. - Drop the stale "do not fall back to PyPI" sentence for hook profiles. Since #1409, a hook profile without a `get_resolver_provider` hook finds versions on its `index_url`. Co-Authored-By: Claude Signed-off-by: Lalatendu Mohanty --- docs/concepts/bootstrapper-architecture.rst | 2 +- docs/concepts/hooks-and-overrides.rst | 4 +--- docs/concepts/package-settings.rst | 4 +--- docs/concepts/resolver-architecture.rst | 18 +++++++----------- docs/reference/config-reference.rst | 2 +- docs/reference/files.md | 2 +- docs/reference/hooks.rst | 6 +++--- src/fromager/packagesettings/_resolver.py | 2 +- src/fromager/server.py | 2 +- src/fromager/sources.py | 6 +++--- src/fromager/wheels.py | 4 ++-- 11 files changed, 22 insertions(+), 30 deletions(-) diff --git a/docs/concepts/bootstrapper-architecture.rst b/docs/concepts/bootstrapper-architecture.rst index 86a2cc91b..970d36af9 100644 --- a/docs/concepts/bootstrapper-architecture.rst +++ b/docs/concepts/bootstrapper-architecture.rst @@ -74,7 +74,7 @@ the contract every phase must satisfy: * - ``Complete`` - Clean up build directories (terminal phase) -.. versionchanged:: 0.102.0 +.. versionchanged:: 0.103.0 ``PrepareSource`` passes the downloaded artifact kind through preparation, so Git checkouts can be prepared without unpacking them. diff --git a/docs/concepts/hooks-and-overrides.rst b/docs/concepts/hooks-and-overrides.rst index 6b048bbf2..e030fbf9b 100644 --- a/docs/concepts/hooks-and-overrides.rst +++ b/docs/concepts/hooks-and-overrides.rst @@ -56,14 +56,12 @@ a Python module. See :doc:`/reference/hooks` for the complete list of hooks and their arguments. -.. versionchanged:: 0.96.0 +.. versionchanged:: 0.103.0 Configured ``hook-sdist`` and ``hook-prebuilt`` profiles support typed download results with explicit artifact kinds. See :ref:`download_source_hook` for the contract, legacy path defaults, and a migration example. -.. versionchanged:: 0.102.0 - ``prepare_source`` selects an archive or Git default using the downloaded artifact kind. Package overrides remain active. See :ref:`prepare_source_hook` for their arguments and the public Git checkout helper. diff --git a/docs/concepts/package-settings.rst b/docs/concepts/package-settings.rst index e588b7f96..6d0890e48 100644 --- a/docs/concepts/package-settings.rst +++ b/docs/concepts/package-settings.rst @@ -48,11 +48,9 @@ legacy variant ``pre_built`` setting. Configured prebuilt resolvers download fresh wheels through their ``download()`` contract; legacy prebuilt packages keep the wheel downloader. -.. versionchanged:: 0.102.0 +.. versionchanged:: 0.103.0 Configured prebuilt source resolvers now select the prebuilt pipeline during bootstrap and later builds. - -.. versionchanged:: 0.102.0 Source and legacy settings mixed across package and variant levels now raise a validation error when package settings load. diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index 93bc7e210..f710dd1b8 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -92,17 +92,13 @@ CLI commands interact with providers through a common Used when versions are already known, e.g. from a prior resolution or a settings-provided URL template. -.. versionchanged:: 0.95.0 +.. versionchanged:: 0.103.0 Packages with ``source:`` configuration now select their provider through the configured source resolver. - -.. versionchanged:: 0.96.0 The ``hook-sdist`` and ``hook-prebuilt`` profiles create providers through - the required ``get_resolver_provider`` override hook. They do not fall back - to PyPI. Their ``download()`` methods invoke the package's ``download_source`` - hook when present and return a validated ``DownloadedSource``. - -.. versionchanged:: 0.102.0 + the package's ``get_resolver_provider`` hook when it exists. Their + ``download()`` methods invoke the package's ``download_source`` hook when + present and return a validated ``DownloadedSource``. Configured source resolvers own their index URL and ignore caller URLs. Hook profiles pass their ``index_url`` to the legacy ``get_resolver_provider`` hook as ``sdist_server_url``. ``hook-sdist`` @@ -133,7 +129,7 @@ different commands, and only these values are saved in ``graph.json`` and ``build-order.json``. The URL must therefore contain everything the download needs, such as the git ref in ``git+https://host/repo.git@refs/tags/v1.0``. -.. versionchanged:: 0.102.0 +.. versionchanged:: 0.103.0 ``download()`` takes ``version`` and ``download_url`` instead of a ``Candidate``. @@ -178,7 +174,7 @@ effective configured ``source_provider`` for provenance; the ``source_url_type`` field remains a legacy compatibility classification, not the actual downloaded artifact kind. -.. versionchanged:: 0.102.0 +.. versionchanged:: 0.103.0 ``download-sequence`` uses configured source resolvers for source entries. Configured prebuilt resolvers select the prebuilt bootstrap path. Fresh @@ -193,7 +189,7 @@ Legacy build downloads still pass through the build wheel directory before the mirror moves them to downloads. See :ref:`download_source_hook` for the return contract and compatibility rules. -.. versionchanged:: 0.102.0 +.. versionchanged:: 0.103.0 Explicit ``DownloadKind.git_checkout`` results can be prepared without unpacking an archive. Configured source resolvers also select the sdist build method. Git checkout resolvers use PEP 517 by default; hook sdist diff --git a/docs/reference/config-reference.rst b/docs/reference/config-reference.rst index 4b0432f1e..4d305de85 100644 --- a/docs/reference/config-reference.rst +++ b/docs/reference/config-reference.rst @@ -89,7 +89,7 @@ Source Resolver .. autoattribute:: any_source .. autoattribute:: not_available -.. versionchanged:: 0.96.0 +.. versionchanged:: 0.103.0 Configured hook downloads require a concrete result kind supported by the selected profile. ``any_source`` and ``not_available`` are not accepted as diff --git a/docs/reference/files.md b/docs/reference/files.md index 93f95a89d..a70026cbc 100644 --- a/docs/reference/files.md +++ b/docs/reference/files.md @@ -96,7 +96,7 @@ checkout in the work directory and are not cached for a later build. The command also uses `source_url_type` to skip entries that used a prebuilt fallback at runtime, even if source settings are configured. -```{versionchanged} 0.102.0 +```{versionchanged} 0.103.0 Configured source providers are recorded in build order, and `download-sequence` uses their download operation. ``` diff --git a/docs/reference/hooks.rst b/docs/reference/hooks.rst index dd58f154b..a1d40ce80 100644 --- a/docs/reference/hooks.rst +++ b/docs/reference/hooks.rst @@ -161,7 +161,7 @@ Resolver hooks the ``download_source`` hook gets the artifact. A package with neither hook is an error; use ``pypi-sdist`` or ``pypi-prebuilt`` instead. - .. versionchanged:: 0.102.0 + .. versionchanged:: 0.103.0 ``hook-sdist`` and ``hook-prebuilt`` no longer require this hook when the package has a ``download_source`` hook. @@ -255,7 +255,7 @@ Source hooks For packages without ``source:`` configuration, return a ``pathlib.Path`` to the downloaded source. This legacy contract is unchanged. - .. versionchanged:: 0.96.0 + .. versionchanged:: 0.103.0 The ``hook-sdist`` and ``hook-prebuilt`` source profiles accept :class:`~fromager.packagesettings.DownloadedSource` with a local @@ -352,7 +352,7 @@ Source hooks Git checkouts must use the double-nested package/version layout under ``ctx.work_dir`` so the later bootstrap and step commands can find them. - .. versionchanged:: 0.102.0 + .. versionchanged:: 0.103.0 Passing ``DownloadKind.git_checkout`` prepares a clean Git checkout. .. autofromagerhook:: default_build_sdist diff --git a/src/fromager/packagesettings/_resolver.py b/src/fromager/packagesettings/_resolver.py index a7dcf805f..9e019a967 100644 --- a/src/fromager/packagesettings/_resolver.py +++ b/src/fromager/packagesettings/_resolver.py @@ -56,7 +56,7 @@ class DownloadedSource: ``path`` is the local artifact path. ``kind`` identifies how the artifact should be handled; hook resolvers require a concrete, supported kind. - .. versionadded:: 0.96.0 + .. versionadded:: 0.103.0 """ path: pathlib.Path diff --git a/src/fromager/server.py b/src/fromager/server.py index d8a61c9e8..cd948cf7d 100644 --- a/src/fromager/server.py +++ b/src/fromager/server.py @@ -125,7 +125,7 @@ def find_mirror_wheel( Returns ``None`` when the mirror has no link to an existing wheel in the downloads or prebuilt directory. - .. versionadded:: 0.102.0 + .. versionadded:: 0.103.0 """ (normalized_name, _, _, _) = parse_wheel_filename(wheel_filename) mirror_wheel = ctx.wheel_server_dir / normalized_name / wheel_filename diff --git a/src/fromager/sources.py b/src/fromager/sources.py index f7041415a..bfc2db662 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -94,7 +94,7 @@ def download_source( Configured source resolvers receive the version and download URL. Legacy downloads are adapted to the sdist kind for compatibility. - .. versionchanged:: 0.102.0 + .. versionchanged:: 0.103.0 Return ``DownloadedSource`` instead of ``pathlib.Path``. Callers use its ``path`` attribute for the downloaded file. """ @@ -458,7 +458,7 @@ def prepare_source( Returns: Path to the prepared source root. - .. versionchanged:: 0.102.0 + .. versionchanged:: 0.103.0 Pass ``download_kind`` to prepare a Git checkout. """ logger.info(f"preparing source for {req} from {source_filename}") @@ -538,7 +538,7 @@ def default_prepare_git_checkout( Returns: The checkout path and ``True`` because preparation ran. - .. versionadded:: 0.102.0 + .. versionadded:: 0.103.0 """ if not source_filename.is_dir() or not (source_filename / ".git").exists(): raise ValueError(f"{source_filename} is not a Git checkout") diff --git a/src/fromager/wheels.py b/src/fromager/wheels.py index 2613f8aa7..09d7adff4 100644 --- a/src/fromager/wheels.py +++ b/src/fromager/wheels.py @@ -525,7 +525,7 @@ def download_wheel( def is_local_wheel_url(wheel_url: str, wheel_server_url: str | None) -> bool: """Return whether a wheel URL belongs to the local wheel mirror. - .. versionadded:: 0.102.0 + .. versionadded:: 0.103.0 """ if not wheel_server_url: return False @@ -546,7 +546,7 @@ def download_prebuilt_wheel( local-mirror URL, configured wheels reuse the exact file the mirror serves. Legacy wheels use the direct wheel downloader. - .. versionadded:: 0.102.0 + .. versionadded:: 0.103.0 """ source_resolver = ctx.package_build_info(req).source_resolver if source_resolver is not None and not source_resolver.resolves_prebuilt_wheel: From 8eabbbfc5bce1c423bd12555d055b774eac679d4 Mon Sep 17 00:00:00 2001 From: Rohan Devasthale Date: Fri, 30 Oct 2026 10:36:11 -0400 Subject: [PATCH 21/35] feat(list-overrides): surface source config in details output Add a 'source' column/field to the list-overrides details table, CSV, and JSON exporters showing the package-level source resolver provider (blank for legacy packages) so the column does not change with --variant. Variant columns now also mark variants whose 'source:' resolver returns prebuilt wheels as 'pre-built'. package list-versions now builds its provider through sources.get_source_provider() so 'source:' packages list versions from their configured provider instead of the legacy hook path, and its cooldown status reads the per-package min_release_age through the shared PackageBuildInfo.resolver_min_release_age helper, which now prefers the source resolver's value over resolver_dist. An explicit --distribution-type is rejected for 'source:' packages instead of being silently ignored, since resolution follows the source config. Co-Authored-By: Claude Code Signed-off-by: Rohan Devasthale --- src/fromager/commands/list_overrides.py | 29 ++- src/fromager/commands/package.py | 29 ++- src/fromager/packagesettings/_pbi.py | 11 ++ src/fromager/sources.py | 16 +- tests/test_list_overrides.py | 86 +++++++++ tests/test_list_versions.py | 179 +++++++++++++++++- tests/test_packagesettings.py | 7 +- .../overrides/settings/test_source_pkg.yaml | 10 + 8 files changed, 340 insertions(+), 27 deletions(-) create mode 100644 tests/testdata/context/overrides/settings/test_source_pkg.yaml diff --git a/src/fromager/commands/list_overrides.py b/src/fromager/commands/list_overrides.py index 714ec8a85..4d3a9ff20 100644 --- a/src/fromager/commands/list_overrides.py +++ b/src/fromager/commands/list_overrides.py @@ -70,13 +70,18 @@ def list_overrides( variant_info: dict[str, str] = {} for v in variants: v_info = ps.variants.get(v) - if v_info: - if v_info.pre_built: - variant_info[str(v)] = "pre-built" - else: - variant_info[str(v)] = "yes" - else: + if v_info is None: variant_info[str(v)] = "" + continue + + is_prebuilt = v_info.pre_built + if v_info.source is not None: + # A variant `source:` block replaces the legacy settings, + # so its resolver decides whether wheels are prebuilt + # (e.g. provider: pypi-prebuilt). + is_prebuilt = v_info.source.resolves_prebuilt_wheel + + variant_info[str(v)] = "pre-built" if is_prebuilt else "yes" all_patches: PatchMap = pbi.get_all_patches() global_patches: list[pathlib.Path] = all_patches.get(None, []) @@ -86,7 +91,10 @@ def list_overrides( [v for v in all_patches.keys() if v is not None] ) - min_release_age = ps.resolver_dist.min_release_age + # Show the package-level source so the column does not change with + # --variant; variant-level sources appear in the variant columns. + source = ps.source.provider if ps.source is not None else "" + min_release_age = pbi.resolver_min_release_age min_release_age_str = ( str(min_release_age) if min_release_age is not None else "" ) @@ -97,6 +105,7 @@ def list_overrides( row_data = { "package": name, "version": "", + "source": source, "patches": patches_str, "plugin_hooks": plugin_hooks_str, "min_release_age": min_release_age_str, @@ -114,6 +123,7 @@ def list_overrides( row_data = { "package": name, "version": str(version), + "source": source, "patches": patches_str, "plugin_hooks": plugin_hooks_str, "min_release_age": min_release_age_str, @@ -149,7 +159,7 @@ def _export_csv( """Export data as CSV.""" # Define field names in the order we want them fieldnames = ( - ["package", "version", "patches", "min_release_age"] + ["package", "version", "source", "patches", "min_release_age"] + variants + ["plugin_hooks"] ) @@ -176,6 +186,7 @@ def _export_table( table = Table(title="Package Overrides") table.add_column("Package", justify="left", no_wrap=True) table.add_column("Version", justify="left", no_wrap=True) + table.add_column("Source", justify="left", no_wrap=True) table.add_column("Patches", justify="left", no_wrap=True) table.add_column("Min Release Age (days)", justify="left", no_wrap=True) @@ -186,7 +197,7 @@ def _export_table( table.add_column("Plugin", justify="left") column_keys = ( - ["package", "version", "patches", "min_release_age"] + ["package", "version", "source", "patches", "min_release_age"] + variants + ["plugin_hooks"] ) diff --git a/src/fromager/commands/package.py b/src/fromager/commands/package.py index db75ca6de..ef3c88a05 100644 --- a/src/fromager/commands/package.py +++ b/src/fromager/commands/package.py @@ -24,6 +24,7 @@ packagesettings, request_session, resolver, + sources, ) from ..candidate import Candidate, Cooldown @@ -151,8 +152,9 @@ def list_versions( - "package_name>=1.0" (versions >= 1.0) - "package_name==1.*" (versions matching 1.*) - This command uses the get_resolver_provider hook to retrieve a resolver - provider for the package in case there is a custom provider configured. + This command uses sources.get_source_provider() to retrieve a resolver + provider for the package, so both ``source:`` settings and the legacy + get_resolver_provider hook are honored. Distribution types: - "default": Use package settings for include_sdists/include_wheels @@ -160,6 +162,9 @@ def list_versions( - "wheel": Only include wheels - "both": Include both source distributions and wheels + An explicit distribution type cannot be combined with a ``source:`` + resolver config; version resolution follows the configured resolver. + Output formats: - "versions": one version per line (default) - "requirements": name==version per line (pip-installable pins) @@ -183,26 +188,32 @@ def list_versions( logger.info(f"Filtering versions with specifier: {req.specifier}") pbi = wkctx.package_build_info(req) - override_sdist_server_url = pbi.resolver_sdist_server_url(sdist_server_url) + + dist_type = DistributionType(distribution_type) + if pbi.source_resolver is not None and dist_type is not DistributionType.DEFAULT: + raise click.ClickException( + f"--distribution-type {dist_type.value} cannot be combined with the " + f"'{pbi.source_resolver.provider}' source resolver of {req.name}: " + "version resolution follows the source config" + ) include_sdists, include_wheels = parse_distribution_option( distribution_type, pbi, ) - provider = overrides.find_and_invoke( - req.name, - "get_resolver_provider", - resolver.default_resolver_provider, + provider = sources.get_source_provider( ctx=wkctx, req=req, + sdist_server_url=sdist_server_url, include_sdists=include_sdists, include_wheels=include_wheels, - sdist_server_url=override_sdist_server_url, ) # Get all available candidates from the provider (cooldown is NOT set on - # the provider so we receive every version that matches the specifier). + # the provider so we receive every version that matches the specifier; + # the detail view applies the cooldown separately). + provider.cooldown = None candidates = list( provider.find_matches( identifier=req.name, diff --git a/src/fromager/packagesettings/_pbi.py b/src/fromager/packagesettings/_pbi.py index 74d55a45b..5b3f60fb0 100644 --- a/src/fromager/packagesettings/_pbi.py +++ b/src/fromager/packagesettings/_pbi.py @@ -267,7 +267,18 @@ def resolver_min_release_age(self) -> int | None: Returns None (inherit global), 0 (disabled), or a positive integer (override days). The caller is responsible for converting to a :class:`~fromager.candidate.Cooldown` instance. + + For packages with a ``source:`` resolver, the resolver's + ``min_release_age`` takes precedence over the legacy + ``resolver_dist`` setting. """ + source_resolver = self.source_resolver + if source_resolver is not None: + source_min_release_age: int | None = getattr( + source_resolver, "min_release_age", None + ) + if source_min_release_age is not None: + return source_min_release_age return self._ps.resolver_dist.min_release_age @property diff --git a/src/fromager/sources.py b/src/fromager/sources.py index f7041415a..2f986044c 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -131,12 +131,19 @@ def get_source_provider( req: Requirement, sdist_server_url: str, req_type: RequirementType | None = None, + include_sdists: bool | None = None, + include_wheels: bool | None = None, ) -> resolver.BaseProvider: """Create a provider for resolving source distributions. Returns a provider configured according to the package's resolver settings (sdist/wheel inclusion and platform matching). + *include_sdists* and *include_wheels* override the legacy package + settings for packages without a ``source:`` resolver; they are ignored + for packages with one because the resolver defines its own inclusion + rules. + Packages with a ``source:`` resolver use that resolver's provider configuration and ignore the caller's URL, with a warning unless it is the default PyPI URL. Packages without one retain the @@ -168,6 +175,11 @@ def get_source_provider( override_sdist_server_url = pbi.resolver_sdist_server_url(sdist_server_url) + if include_sdists is None: + include_sdists = pbi.resolver_include_sdists + if include_wheels is None: + include_wheels = pbi.resolver_include_wheels + provider = typing.cast( resolver.BaseProvider, overrides.find_and_invoke( @@ -176,8 +188,8 @@ def get_source_provider( resolver.default_resolver_provider, ctx=ctx, req=req, - include_sdists=pbi.resolver_include_sdists, - include_wheels=pbi.resolver_include_wheels, + include_sdists=include_sdists, + include_wheels=include_wheels, sdist_server_url=override_sdist_server_url, req_type=req_type, ignore_platform=pbi.resolver_ignore_platform, diff --git a/tests/test_list_overrides.py b/tests/test_list_overrides.py index df7edf72d..48bd3ffa9 100644 --- a/tests/test_list_overrides.py +++ b/tests/test_list_overrides.py @@ -275,3 +275,89 @@ def test_list_overrides_min_release_age( # test-other-pkg has no min_release_age override — should be empty test_other_pkg = next(item for item in data if item["package"] == "test-other-pkg") assert test_other_pkg["min_release_age"] == "" + + +def test_list_overrides_details_source( + testdata_path: pathlib.Path, cli_runner: CliRunner +) -> None: + """--details includes the source provider for source-config packages.""" + overrides_dir = testdata_path / "context" / "overrides" + settings_file = overrides_dir / "settings.yaml" + settings_dir = overrides_dir / "settings" + patches_dir = overrides_dir / "patches" + + result = cli_runner.invoke( + fromager, + [ + "--settings-file", + str(settings_file), + "--settings-dir", + str(settings_dir), + "--patches-dir", + str(patches_dir), + "list-overrides", + "--details", + "--format", + "json", + ], + ) + assert result.exit_code == 0 + + json_output = _extract_json_from_output(result.stdout) + data = json.loads(json_output) + + # test-source-pkg has a source: block with a github-tag-git provider + test_source_pkg = next( + item for item in data if item["package"] == "test-source-pkg" + ) + assert test_source_pkg["source"] == "github-tag-git" + # min_release_age is read from the source config + assert test_source_pkg["min_release_age"] == "14" + # the rocm variant resolves a prebuilt wheel through its variant source + assert test_source_pkg["rocm"] == "pre-built" + + # legacy packages have no source config — the field is blank + test_other_pkg = next(item for item in data if item["package"] == "test-other-pkg") + assert test_other_pkg["source"] == "" + + +def test_list_overrides_details_source_variant( + testdata_path: pathlib.Path, cli_runner: CliRunner +) -> None: + """--variant does not change the package-level source column.""" + overrides_dir = testdata_path / "context" / "overrides" + settings_file = overrides_dir / "settings.yaml" + settings_dir = overrides_dir / "settings" + patches_dir = overrides_dir / "patches" + + result = cli_runner.invoke( + fromager, + [ + "--variant", + "rocm", + "--settings-file", + str(settings_file), + "--settings-dir", + str(settings_dir), + "--patches-dir", + str(patches_dir), + "list-overrides", + "--details", + "--format", + "json", + ], + ) + assert result.exit_code == 0 + + json_output = _extract_json_from_output(result.stdout) + data = json.loads(json_output) + + test_source_pkg = next( + item for item in data if item["package"] == "test-source-pkg" + ) + # the source column shows the package-level setting, not the rocm + # variant's pypi-prebuilt resolver + assert test_source_pkg["source"] == "github-tag-git" + # min_release_age comes from the variant's source resolver + assert test_source_pkg["min_release_age"] == "3" + assert test_source_pkg["rocm"] == "pre-built" diff --git a/tests/test_list_versions.py b/tests/test_list_versions.py index 09ff31040..3f665419f 100644 --- a/tests/test_list_versions.py +++ b/tests/test_list_versions.py @@ -377,18 +377,171 @@ def test_list_versions_output_file_requirements_format( # --ignore-per-package-overrides # --------------------------------------------------------------------------- +_ALTERNATE_INDEX_URL = "https://alternate.test/simple/test-pkg/" -def test_list_versions_ignore_per_package_overrides( + +def _invoke_with_source_settings( + cli_runner: CliRunner, + settings_dir: pathlib.Path, + extra_args: list[str] | None = None, + min_release_age: int = 0, +) -> typing.Any: + """Invoke ``package list-versions`` for a package with a ``source:`` resolver. + + The configured resolver uses an alternate index URL; only that URL is + mocked, so a request to pypi.org fails the test. + """ + args: list[str] = ["--settings-dir", str(settings_dir)] + if min_release_age: + args.extend(["--min-release-age", str(min_release_age)]) + args.extend(["package", "list-versions"]) + if extra_args: + args.extend(extra_args) + args.append("test-pkg") + + with requests_mock.Mocker() as m: + m.get( + _ALTERNATE_INDEX_URL, + json=_PYPI_JSON_RESPONSE, + headers={"Content-Type": _PYPI_SIMPLE_JSON_CONTENT_TYPE}, + ) + return cli_runner.invoke(fromager, args) + + +def _write_source_settings( + settings_dir: pathlib.Path, + extra_source_settings: str = "", +) -> None: + """Write a pypi-sdist source config using the alternate index URL.""" + settings_dir.mkdir(parents=True, exist_ok=True) + (settings_dir / "test_pkg.yaml").write_text( + "source:\n" + " provider: pypi-sdist\n" + f" index_url: {(_ALTERNATE_INDEX_URL.removesuffix('test-pkg/'))}\n" + + extra_source_settings + ) + + +def test_list_versions_source_config_provider( + cli_runner: CliRunner, + tmp_path: pathlib.Path, +) -> None: + """A ``source:`` package lists versions from its configured provider.""" + settings_dir = tmp_path / "settings" + _write_source_settings(settings_dir) + + result = _invoke_with_source_settings(cli_runner, settings_dir) + assert result.exit_code == 0 + # Versions come from the alternate index, not pypi.org. + versions = [line.strip() for line in result.stdout.split("\n") if line.strip()] + assert "1.2.2" in versions + assert "1.3.2" in versions + assert "2.0.0" in versions + + +def test_list_versions_source_rejects_distribution_type( + cli_runner: CliRunner, + tmp_path: pathlib.Path, +) -> None: + """An explicit --distribution-type is rejected for source-config packages.""" + settings_dir = tmp_path / "settings" + _write_source_settings(settings_dir) + + for dist_type in ("sdist", "wheel", "both"): + result = _invoke_with_source_settings( + cli_runner, + settings_dir, + extra_args=["--distribution-type", dist_type], + ) + assert result.exit_code != 0 + assert f"--distribution-type {dist_type}" in result.output + assert "source resolver" in result.output + + +def test_list_versions_source_min_release_age( cli_runner: CliRunner, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - """--ignore-per-package-overrides uses global cooldown, ignoring per-package override. + """Cooldown status uses the source resolver's min_release_age.""" + _freeze_cooldown_bootstrap_time(monkeypatch) - Set up a per-package override of min_release_age=0 (exempt) and verify - that --ignore-per-package-overrides still shows cooldown status based - on the global --min-release-age. - """ + settings_dir = tmp_path / "settings" + _write_source_settings(settings_dir, " min_release_age: 3\n") + + # The source override (3 days) wins over the global value (2 days): + # 2.0.0 is 2 days old, so it is blocked. + result = _invoke_with_source_settings( + cli_runner, + settings_dir, + extra_args=["--format", "json"], + min_release_age=2, + ) + assert result.exit_code == 0 + status_by_version = { + row["version"]: row["cooldown"] + for row in json.loads(_extract_json_from_output(result.stdout)) + } + assert status_by_version["2.0.0"] == "blocked" + assert status_by_version["1.3.2"] == "available" + + # --ignore-per-package-overrides uses the global value (2 days): + # 2.0.0 is exactly 2 days old, so it is available. + result = _invoke_with_source_settings( + cli_runner, + settings_dir, + extra_args=["--format", "json", "--ignore-per-package-overrides"], + min_release_age=2, + ) + assert result.exit_code == 0 + status_by_version = { + row["version"]: row["cooldown"] + for row in json.loads(_extract_json_from_output(result.stdout)) + } + assert status_by_version["2.0.0"] == "available" + assert status_by_version["1.3.2"] == "available" + + +def test_list_versions_source_min_release_age_zero( + cli_runner: CliRunner, + tmp_path: pathlib.Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A source min_release_age of 0 disables the cooldown.""" + _freeze_cooldown_bootstrap_time(monkeypatch) + + settings_dir = tmp_path / "settings" + _write_source_settings(settings_dir, " min_release_age: 0\n") + + # The source override (0) disables the cooldown entirely. + result = _invoke_with_source_settings( + cli_runner, + settings_dir, + extra_args=["--format", "json"], + min_release_age=_COOLDOWN_7_DAYS, + ) + assert result.exit_code == 0 + for row in json.loads(_extract_json_from_output(result.stdout)): + assert row["cooldown"] == "" + + # --ignore-per-package-overrides restores the global cooldown. + result = _invoke_with_source_settings( + cli_runner, + settings_dir, + extra_args=["--format", "json", "--ignore-per-package-overrides"], + min_release_age=_COOLDOWN_7_DAYS, + ) + assert result.exit_code == 0 + status_by_version = { + row["version"]: row["cooldown"] + for row in json.loads(_extract_json_from_output(result.stdout)) + } + assert status_by_version["2.0.0"] == "blocked" + assert status_by_version["1.3.2"] == "available" + + +def _freeze_cooldown_bootstrap_time(monkeypatch: pytest.MonkeyPatch) -> None: + """Pin Cooldown.bootstrap_time to _BOOTSTRAP_TIME for deterministic ages.""" original_init = candidate.Cooldown.__init__ def patched_init( @@ -399,6 +552,20 @@ def patched_init( monkeypatch.setattr(candidate.Cooldown, "__init__", patched_init) + +def test_list_versions_ignore_per_package_overrides( + cli_runner: CliRunner, + tmp_path: pathlib.Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """--ignore-per-package-overrides uses global cooldown, ignoring per-package override. + + Set up a per-package override of min_release_age=0 (exempt) and verify + that --ignore-per-package-overrides still shows cooldown status based + on the global --min-release-age. + """ + _freeze_cooldown_bootstrap_time(monkeypatch) + # Create per-package override with min_release_age=0 (exempted) settings_dir = tmp_path / "settings" settings_dir.mkdir(parents=True) diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index bfd14c726..6a37a8266 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -37,6 +37,7 @@ TEST_RELATED_PKG = "test-pkg-library" TEST_PREBUILT_PKG = "test-prebuilt-pkg" TEST_COOLDOWN_PKG = "test-cooldown-pkg" +TEST_SOURCE_PKG = "test-source-pkg" FULL_EXPECTED: dict[str, typing.Any] = { "annotations": { @@ -552,6 +553,7 @@ def test_settings_overrides(testdata_context: context.WorkContext) -> None: TEST_RELATED_PKG, TEST_PREBUILT_PKG, TEST_COOLDOWN_PKG, + TEST_SOURCE_PKG, } @@ -603,11 +605,14 @@ def test_settings_list(testdata_context: context.WorkContext) -> None: TEST_OTHER_PKG, TEST_RELATED_PKG, TEST_PREBUILT_PKG, + TEST_SOURCE_PKG, } assert testdata_context.settings.list_pre_built() == {TEST_PREBUILT_PKG} assert testdata_context.settings.variant_changelog() == [] testdata_context.settings.variant = Variant("rocm") - assert testdata_context.settings.list_pre_built() == {TEST_PKG} + # TEST_SOURCE_PKG's rocm variant resolves a prebuilt wheel via its + # variant-level `source:` resolver. + assert testdata_context.settings.list_pre_built() == {TEST_PKG, TEST_SOURCE_PKG} assert testdata_context.settings.variant_changelog() == [ "setuptools upgraded to 82.0.0" ] diff --git a/tests/testdata/context/overrides/settings/test_source_pkg.yaml b/tests/testdata/context/overrides/settings/test_source_pkg.yaml new file mode 100644 index 000000000..b6c74abe4 --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_source_pkg.yaml @@ -0,0 +1,10 @@ +source: + provider: github-tag-git + project_url: https://github.com/python-wheel-build/fromager + min_release_age: 14 +variants: + rocm: + source: + provider: pypi-prebuilt + index_url: https://wheel.test/simple + min_release_age: 3 From 5494a601397ff7b2fda389e5001f2f3256de4bf3 Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Fri, 30 Oct 2026 15:45:38 -0400 Subject: [PATCH 22/35] fix(sources): exclude `.git` from configured tarball sdists With `build_sdist: tarball`, a git checkout's `.git` ended up in the sdist. Pass `exclude_vcs=True` on the configured tarball path. Packages without `source:` are unchanged. Co-Authored-By: Claude Signed-off-by: Lalatendu Mohanty --- docs/concepts/resolver-architecture.rst | 3 ++- src/fromager/sources.py | 9 +++++++ tests/test_sources.py | 36 ++++++++++++++++++++++++- 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index 1800120b8..b9c0bb89a 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -202,7 +202,8 @@ See :ref:`download_source_hook` for the return contract and compatibility rules. build method. Git checkout resolvers use PEP 517 by default; hook sdist resolvers require an explicit ``build_sdist: pep517`` or ``build_sdist: tarball``. PyPI sdist resolvers - repack a tarball. Both configured strategies take precedence over an installed + repack a tarball. A configured ``tarball`` build leaves out VCS metadata + such as ``.git``. Both configured strategies take precedence over an installed ``build_sdist`` override and log a warning when it is ignored. Legacy settings continue to use the override. Configured prebuilt profiles use their source provider after a local cache check, and local mirror URLs diff --git a/src/fromager/sources.py b/src/fromager/sources.py index 6164feb18..2dc527f12 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -692,6 +692,8 @@ def build_sdist( sdist_root_dir=sdist_root_dir, build_dir=build_dir, build_env=build_env, + # Git checkouts keep their `.git`; keep it out of the sdist. + exclude_vcs=True, ) elif source_resolver is None: sdist_file = overrides.find_and_invoke( @@ -734,7 +736,13 @@ def default_build_sdist( sdist_root_dir: pathlib.Path, build_env: build_environment.BuildEnvironment, build_dir: pathlib.Path, + *, + exclude_vcs: bool = False, ) -> pathlib.Path: + """Repack *build_dir* as an sdist. + + With *exclude_vcs*, VCS metadata such as ``.git`` is left out. + """ # It seems like the "correct" way to do this would be to run the # PEP 517 API in the source tree we have modified. However, quite # a few packages assume their source distribution is being built @@ -763,6 +771,7 @@ def default_build_sdist( tar=sdist, basedir=build_dir, prefix=build_dir.parent, + exclude_vcs=exclude_vcs, arcname_root=f"{dist_name}-{version}", ) return sdist_filename diff --git a/tests/test_sources.py b/tests/test_sources.py index 29dfd4042..54732d583 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -1944,7 +1944,7 @@ def test_build_sdist_uses_configured_tarball_or_legacy_hook( tarball.assert_not_called() find_override.assert_not_called() else: - tarball.assert_called_once_with(**build_kwargs) + tarball.assert_called_once_with(**build_kwargs, exclude_vcs=True) find_and_invoke.assert_not_called() find_override.assert_called_once_with(req.name, "build_sdist") if override is not None: @@ -2118,3 +2118,37 @@ def test_default_build_sdist_normalizes_name_and_root( top_levels = {name.split("/")[0] for name in names} assert top_levels == {"foo_bar-1.0"} assert "foo_bar-1.0/setup.py" in names + + +@pytest.mark.parametrize("exclude_vcs", [False, True]) +def test_default_build_sdist_exclude_vcs( + exclude_vcs: bool, + tmp_context: context.WorkContext, + tmp_path: pathlib.Path, +) -> None: + """Leave VCS metadata out of the sdist only when asked.""" + build_dir = tmp_path / "pkg-1.0" + (build_dir / ".git" / "objects").mkdir(parents=True) + (build_dir / ".git" / "HEAD").write_text("ref: refs/heads/main\n") + (build_dir / "sub").mkdir() + (build_dir / "sub" / ".git").write_text("gitdir: ../.git/modules/sub\n") + (build_dir / "setup.py").write_text("from setuptools import setup; setup()\n") + + sdist_file = sources.default_build_sdist( + ctx=tmp_context, + extra_environ={}, + req=Requirement("pkg==1.0"), + version=Version("1.0"), + sdist_root_dir=build_dir, + build_env=Mock(), + build_dir=build_dir, + exclude_vcs=exclude_vcs, + ) + + with tarfile.open(sdist_file, "r:gz") as tar: + names = tar.getnames() + assert "pkg-1.0/setup.py" in names + has_vcs = {"pkg-1.0/.git/HEAD", "pkg-1.0/sub/.git"} <= set(names) + assert has_vcs is not exclude_vcs + if exclude_vcs: + assert not any("/.git" in name for name in names) From 4b888fe2ab5d8f4e76de2f06d5f3f46d70a0322b Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Mon, 2 Nov 2026 08:48:23 -0500 Subject: [PATCH 23/35] test(bootstrapper): fix a race in the resolver test Raise the test error in the fake resolver. This lets the background task finish before the test removes its mock. Co-Authored-By: Codex Signed-off-by: Lalatendu Mohanty --- tests/test_bootstrapper_iterative.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/test_bootstrapper_iterative.py b/tests/test_bootstrapper_iterative.py index d68d45d4e..2aa739b9a 100644 --- a/tests/test_bootstrapper_iterative.py +++ b/tests/test_bootstrapper_iterative.py @@ -1405,15 +1405,14 @@ def test_multiple_versions_resolve_failure_continues( """RESOLVE failure for one dependency does not crash the loop.""" bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) - original_resolve_run = Resolve.run completed: list[str] = [] - def resolve_run( - self: Resolve, bt_arg: bootstrapper.Bootstrapper - ) -> list[Phase]: - if str(self.work_item.req.name) == "bad-dep": + def resolve( + req: Requirement, **kwargs: typing.Any + ) -> list[tuple[str, Version]]: + if req.name == "bad-dep": raise RuntimeError("Could not resolve any versions for bad-dep") - return original_resolve_run(self, bt_arg) + return [("https://pkg.test/pkg-1.0.tar.gz", Version("1.0"))] def prepare_source_run( self: PrepareSource, bt_arg: bootstrapper.Bootstrapper @@ -1422,13 +1421,12 @@ def prepare_source_run( return [] with ( - patch.object(Resolve, "run", resolve_run), patch.object(PrepareSource, "background_work", return_value=None), patch.object(PrepareSource, "run", prepare_source_run), patch.object( bt._resolver, "resolve", - return_value=[("url-1.0", Version("1.0"))], + side_effect=resolve, ), patch.object(bt, "has_been_seen", return_value=False), ): From 3fc6de3f3b8f7da59d78916f5d110b154517cc1a Mon Sep 17 00:00:00 2001 From: Shanmukh Pawan Date: Mon, 2 Nov 2026 11:14:02 -0500 Subject: [PATCH 24/35] fix(build-sequence): validate recorded source providers Check build-order provenance before replay starts, sharing the provider check with download-sequence. Preserve skipped download URLs and document the replay contract. Closes: #1418 Co-Authored-By: Codex Signed-off-by: Shanmukh Pawan --- docs/concepts/resolver-architecture.rst | 21 +-- src/fromager/commands/build.py | 58 +++---- src/fromager/commands/build_order.py | 31 +++- src/fromager/commands/download_sequence.py | 28 +--- tests/test_commands.py | 166 ++++++++++++++++++++- 5 files changed, 239 insertions(+), 65 deletions(-) diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index b9c0bb89a..46a81a6f9 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -132,7 +132,7 @@ settings are used. A configured resolver's ``download()`` receives only the version and the download URL of the selected candidate. Resolution and download can run in -different commands, and only these values are saved in ``graph.json`` and +different commands, and these values are saved in ``graph.json`` and ``build-order.json``. The URL must therefore contain everything the download needs, such as the git ref in ``git+https://host/repo.git@refs/tags/v1.0``. @@ -163,14 +163,15 @@ The separate ``step prepare-source`` invocation accepts ``--source-kind`` and actual path and kind without changing its default path-only output. Without ``--source-kind``, it keeps the legacy archive behavior. ``download-sequence`` calls the configured resolver's normal ``download()`` -for source entries. Before downloading, it checks ``source_provider`` against -the current provider for each source entry and each ``prebuilt: true`` wheel -requested with ``--include-wheels``. A missing or different provider fails -before any downloads start. Older build orders without ``source_provider`` -still work with legacy settings; regenerate them before replaying a configured -source. This checks the provider name, so replay should still use the same -package settings. A runtime ``prebuilt`` entry is skipped even when the -package has source settings. +for source entries. Both sequence commands check each entry's recorded +``source_provider`` against current settings, including prebuilt entries whose +URLs ``download-sequence`` will skip. A missing or different provider fails +before any downloads or builds start. Older build orders without +``source_provider`` still work with legacy settings; regenerate them before +replaying a configured source. +These checks compare provider names, so replay should still use the same +package settings. ``download-sequence`` still skips downloading a runtime +``prebuilt`` entry even when the package has source settings. Built-in archive resolvers cache the resolved file for a later build. Git checkout resolvers clone into the work directory instead: with cleanup enabled, a later build removes that checkout and clones again; without cleanup, the @@ -182,7 +183,7 @@ effective configured ``source_provider`` for provenance; the the actual downloaded artifact kind. .. versionchanged:: 0.103.0 - ``download-sequence`` uses configured source resolvers for source entries. + Sequence commands check recorded providers before replaying a build order. Configured prebuilt resolvers select the prebuilt bootstrap path. Fresh external wheels use their ``download()`` contract and are staged in the diff --git a/src/fromager/commands/build.py b/src/fromager/commands/build.py index 8e86328e0..a41b10d44 100644 --- a/src/fromager/commands/build.py +++ b/src/fromager/commands/build.py @@ -35,6 +35,7 @@ ) from ..log import VERBOSE_LOG_FMT, ThreadLogFilter, req_ctxvar_context +from .build_order import check_source_providers logger = logging.getLogger(__name__) @@ -177,36 +178,37 @@ def build_sequence( logger.info("reading build order from %s", build_order_file) with read.open_file_or_url(build_order_file) as f: - for entry in progress.progress(json.load(f)): - dist_name = entry["dist"] - resolved_version = Version(entry["version"]) - source_download_url = entry["source_url"] - - req = Requirement(f"{dist_name}=={resolved_version}") - - with req_ctxvar_context(req, resolved_version): - logger.info("building %s", resolved_version) - entry = _build( - wkctx=wkctx, - resolved_version=resolved_version, - req=req, - source_download_url=source_download_url, - force=force, - cache_wheel_server_url=cache_wheel_server_url, + build_order = json.load(f) + # Check provider consistency for every entry, including prebuilt wheels. + check_source_providers(wkctx, build_order) + for entry in progress.progress(build_order): + dist_name = entry["dist"] + resolved_version = Version(entry["version"]) + source_download_url = entry["source_url"] + + req = Requirement(f"{dist_name}=={resolved_version}") + + with req_ctxvar_context(req, resolved_version): + logger.info("building %s", resolved_version) + entry = _build( + wkctx=wkctx, + resolved_version=resolved_version, + req=req, + source_download_url=source_download_url, + force=force, + cache_wheel_server_url=cache_wheel_server_url, + ) + if entry.prebuilt: + logger.info("downloaded prebuilt wheel %s", entry.wheel_filename.name) + elif entry.skipped: + logger.info( + "skipping building wheel since %s already exists", + entry.wheel_filename.name, ) - if entry.prebuilt: - logger.info( - "downloaded prebuilt wheel %s", entry.wheel_filename.name - ) - elif entry.skipped: - logger.info( - "skipping building wheel since %s already exists", - entry.wheel_filename.name, - ) - else: - logger.info("built %s", entry.wheel_filename.name) + else: + logger.info("built %s", entry.wheel_filename.name) - entries.append(entry) + entries.append(entry) metrics.summarize(wkctx, "Building") diff --git a/src/fromager/commands/build_order.py b/src/fromager/commands/build_order.py index 88782c4ea..f74ff3257 100644 --- a/src/fromager/commands/build_order.py +++ b/src/fromager/commands/build_order.py @@ -3,10 +3,39 @@ import json import pathlib import sys +import typing import click +from packaging.requirements import Requirement -from .. import clickext, overrides +from .. import clickext, context, overrides + + +def check_source_providers( + wkctx: context.WorkContext, entries: typing.Iterable[dict[str, typing.Any]] +) -> None: + """Reject build-order entries whose source provider differs from settings. + + Args: + wkctx: Context containing the current package settings. + entries: Build-order entries to validate. + + Raises: + ValueError: A recorded provider does not match current settings. + """ + for entry in entries: + recorded_provider = entry.get("source_provider") + req = Requirement(f"{entry['dist']}=={entry['version']}") + source_resolver = wkctx.package_build_info(req).source_resolver + current_provider = ( + source_resolver.provider if source_resolver is not None else None + ) + if recorded_provider != current_provider: + raise ValueError( + f"{req}: build order records source provider {recorded_provider!r}, " + f"current settings select {current_provider!r}; " + "use matching settings or regenerate the build order" + ) @click.group() diff --git a/src/fromager/commands/download_sequence.py b/src/fromager/commands/download_sequence.py index 2f54597a1..1fb32a6e5 100644 --- a/src/fromager/commands/download_sequence.py +++ b/src/fromager/commands/download_sequence.py @@ -9,6 +9,7 @@ from .. import context, progress, read, sources, threading_utils, wheels from ..log import req_ctxvar_context +from .build_order import check_source_providers logger = logging.getLogger(__name__) @@ -55,10 +56,10 @@ def download_sequence( SDIST_SERVER_URL is a PyPI-compatible index used for optional wheel lookup - Configured sources use their resolver's normal download operation. Entries - that download a configured source or prebuilt wheel require the recorded - provider to match current settings. Legacy entries retain the - ``source_url_type`` selection behavior. + Configured sources use their resolver's normal download operation. Every + entry requires its recorded provider to match current settings, even when + this command skips its URL. Legacy entries retain the ``source_url_type`` + selection behavior. """ # The local wheel server is a trusted cache (wheels built in this or @@ -73,23 +74,8 @@ def download_sequence( with read.open_file_or_url(build_order_file) as f: build_order = json.load(f) - # Check recorded providers before workers can change the download cache - # or a Git work directory. Skipped URLs do not use a source resolver. - for entry in build_order: - if _skip_recorded_url(entry, include_wheels): - continue - recorded_provider = entry.get("source_provider") - req = Requirement(f"{entry['dist']}=={entry['version']}") - source_resolver = wkctx.package_build_info(req).source_resolver - current_provider = ( - source_resolver.provider if source_resolver is not None else None - ) - if recorded_provider != current_provider: - raise ValueError( - f"{req}: build order records source provider {recorded_provider!r}, " - f"current settings select {current_provider!r}; " - "use matching settings or regenerate the build order" - ) + # Validate the whole order before workers alter the cache or Git work dirs. + check_source_providers(wkctx, build_order) def download_one(entry: dict[str, typing.Any]) -> None: req = Requirement(f"{entry['dist']}=={entry['version']}") diff --git a/tests/test_commands.py b/tests/test_commands.py index c7a09ce1b..b68b777e0 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -237,13 +237,13 @@ def test_download_sequence_replays_configured_prebuilt( @pytest.mark.parametrize("include_wheels", [False, True]) -def test_download_sequence_checks_prebuilt_provider_before_download( +def test_download_sequence_checks_prebuilt_provider_even_when_skipped( cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path, include_wheels: bool, ) -> None: - """A configured wheel download requires its recorded provider.""" + """A prebuilt entry needs matching settings even without a download.""" build_order = tmp_path / "build-order.json" build_order.write_text( json.dumps( @@ -253,6 +253,7 @@ def test_download_sequence_checks_prebuilt_provider_before_download( "version": "1.0", "prebuilt": True, "source_url": "https://pkg.test/pkg-1.0-py3-none-any.whl", + "source_url_type": "prebuilt", "source_provider": "hook-prebuilt", } ] @@ -267,9 +268,8 @@ def test_download_sequence_checks_prebuilt_provider_before_download( download_sequence.download_sequence, args, obj=tmp_context ) - assert (result.exit_code != 0) is include_wheels - if include_wheels: - assert "hook-prebuilt" in str(result.exception) + assert isinstance(result.exception, ValueError) + assert "hook-prebuilt" in str(result.exception) download.assert_not_called() @@ -619,6 +619,130 @@ def test_download_sequence_checks_providers_before_starting_downloads( download.assert_not_called() +def test_build_sequence_rejects_mismatched_source_provider_before_build( + cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path +) -> None: + """A later mismatch prevents every build in the recorded order.""" + _configure_package(tmp_context, "source:\n provider: pypi-sdist\n") + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [ + { + "dist": "other", + "version": "1.0", + "prebuilt": False, + "source_url": "https://pkg.test/other-1.0.tar.gz", + "source_url_type": "sdist", + }, + { + "dist": "pkg", + "version": "1.0", + "prebuilt": False, + "source_url": "git+https://github.test/org/pkg.git@refs/tags/v1.0", + "source_url_type": "override", + "source_provider": "github-tag-git", + }, + ] + ) + ) + + with ( + patch.object(build.server, "start_wheel_server"), + patch.object(build, "_build") as run_build, + patch.object(build.metrics, "summarize"), + patch.object(build, "_summary"), + ): + result = cli_runner.invoke( + build.build_sequence, [str(build_order)], obj=tmp_context + ) + + assert isinstance(result.exception, ValueError) + assert "pkg==1.0" in str(result.exception) + assert "github-tag-git" in str(result.exception) + assert "pypi-sdist" in str(result.exception) + run_build.assert_not_called() + + +def test_build_sequence_rejects_prebuilt_entry_without_current_source_provider( + cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path +) -> None: + """A prebuilt entry still needs matching settings during build replay.""" + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [ + { + "dist": "pkg", + "version": "1.0", + "prebuilt": True, + "source_url": "https://pkg.test/pkg-1.0-py3-none-any.whl", + "source_url_type": "prebuilt", + "source_provider": "hook-prebuilt", + } + ] + ) + ) + + with ( + patch.object(build.server, "start_wheel_server"), + patch.object(build, "_build") as run_build, + ): + result = cli_runner.invoke( + build.build_sequence, [str(build_order)], obj=tmp_context + ) + + assert isinstance(result.exception, ValueError) + assert "pkg==1.0" in str(result.exception) + assert "hook-prebuilt" in str(result.exception) + assert "current settings select None" in str(result.exception) + run_build.assert_not_called() + + +@pytest.mark.parametrize("configured", [False, True]) +def test_build_sequence_accepts_matching_source_provider( + cli_runner: CliRunner, + tmp_context: WorkContext, + tmp_path: pathlib.Path, + configured: bool, +) -> None: + """Build replay accepts legacy entries and matching configured sources.""" + if configured: + _configure_package(tmp_context, "source:\n provider: pypi-sdist\n") + build_order = tmp_path / "build-order.json" + source_url = "https://pkg.test/pkg-1.0.tar.gz" + recorded_entry = { + "dist": "pkg", + "version": "1.0", + "prebuilt": False, + "source_url": source_url, + "source_url_type": "override" if configured else "sdist", + } + if configured: + recorded_entry["source_provider"] = "pypi-sdist" + build_order.write_text(json.dumps([recorded_entry])) + built_entry = build.BuildSequenceEntry( + name="pkg", + version=Version("1.0"), + prebuilt=False, + download_url=source_url, + wheel_filename=tmp_context.wheels_build / "pkg-1.0-py3-none-any.whl", + ) + + with ( + patch.object(build.server, "start_wheel_server"), + patch.object(build, "_build", return_value=built_entry) as run_build, + patch.object(build.metrics, "summarize"), + patch.object(build, "_summary"), + ): + result = cli_runner.invoke( + build.build_sequence, [str(build_order)], obj=tmp_context + ) + + assert result.exit_code == 0, result.output + run_build.assert_called_once() + + def test_download_sequence_skips_test_mode_prebuilt_fallback_source( cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path ) -> None: @@ -651,6 +775,38 @@ def test_download_sequence_skips_test_mode_prebuilt_fallback_source( download.assert_not_called() +def test_download_sequence_rejects_mismatched_fallback_provider( + cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path +) -> None: + """A skipped fallback URL still requires matching package settings.""" + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [ + { + "dist": "pkg", + "version": "1.0", + "prebuilt": False, + "source_url": "https://pkg.test/pkg-1.0.tar.gz", + "source_url_type": "prebuilt", + "source_provider": "pypi-sdist", + } + ] + ) + ) + + with patch.object(download_sequence.sources, "download_source") as download: + result = cli_runner.invoke( + download_sequence.download_sequence, + [str(build_order), "https://index.test/simple", "--num-threads", "1"], + obj=tmp_context, + ) + + assert isinstance(result.exception, ValueError) + assert "pypi-sdist" in str(result.exception) + download.assert_not_called() + + def test_download_sequence_local_mirror_url_skips_download_hook( cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path ) -> None: From 78f119dba573f97e1af7ac70e8cbf94c50aabda4 Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Mon, 2 Nov 2026 14:14:23 -0500 Subject: [PATCH 25/35] fix(sources): reuse matching Git checkouts Reuse an existing checkout when it is a clean clone of the same repository at the same commit. Builds after download-sequence no longer clone again and can run without the Git host. Other checkouts are still cloned again with cleanup or rejected without it. Fixes #1431 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Lalatendu Mohanty --- docs/concepts/resolver-architecture.rst | 9 +- docs/using.md | 8 +- src/fromager/gitutils.py | 89 ++++++++++++++ src/fromager/packagesettings/_resolver.py | 35 ++++-- src/fromager/sources.py | 23 +--- tests/test_gitutils.py | 142 +++++++++++++++++++++- tests/test_packagesettings_resolver.py | 119 ++++++++++++++---- 7 files changed, 367 insertions(+), 58 deletions(-) diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index 46a81a6f9..c8782ca72 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -173,9 +173,12 @@ These checks compare provider names, so replay should still use the same package settings. ``download-sequence`` still skips downloading a runtime ``prebuilt`` entry even when the package has source settings. Built-in archive resolvers cache the resolved file for a later build. Git -checkout resolvers clone into the work directory instead: with cleanup enabled, -a later build removes that checkout and clones again; without cleanup, the -later download fails because the checkout exists. Hook +checkout resolvers clone into the work directory instead. A later build in the +same work directory reuses that checkout if it is still a clean clone of the +same repository at the same commit, so it can run without access to the Git +host. It only checks local state, so a tag moved on the remote is not noticed. +Any other existing checkout is removed and cloned again with cleanup enabled, +and the download fails without cleanup. Hook download behavior depends on the hook. Legacy packages keep the ``source_url_type`` selection behavior. Build-order entries record the effective configured ``source_provider`` for provenance; the diff --git a/docs/using.md b/docs/using.md index 7ffab10e8..76fa9198c 100644 --- a/docs/using.md +++ b/docs/using.md @@ -266,9 +266,11 @@ download command's optional `--output-format json` prints both `path` and `kind` when a hook can return different artifact kinds. Its default output remains the path alone. Git preparation applies patches to the selected checkout itself, so use a -disposable clone for `--source-path`. Git downloads do not reuse an existing -checkout with `--no-cleanup`; the files remain available for inspection. Enable -cleanup or use a new work directory to download it again. +disposable clone for `--source-path`. Git downloads reuse an existing checkout +only when it is a clean clone of the same repository at the same commit. Any +other existing checkout is cloned again when cleanup is enabled. With +`--no-cleanup` the download fails instead, so the files remain available for +inspection. Enable cleanup or use a new work directory to download it again. The `step prepare-build` command creates a virtualenv with the build dependencies for building the wheel. It expects a `--wheel-server-url` diff --git a/src/fromager/gitutils.py b/src/fromager/gitutils.py index 2d554a6a1..20af13351 100644 --- a/src/fromager/gitutils.py +++ b/src/fromager/gitutils.py @@ -2,6 +2,7 @@ import logging import pathlib +import subprocess import typing from urllib.parse import urlparse @@ -158,6 +159,94 @@ def git_clean(source_dir: pathlib.Path) -> None: external_commands.run(cmd, cwd=str(source_dir)) +_GIT_STATUS_COMMAND: typing.Final = ( + "git", + "status", + "--porcelain=v1", + "--untracked-files=all", + "--ignored=matching", + "--ignore-submodules=none", +) + + +def git_checkout_changes(checkout_dir: pathlib.Path) -> str: + """Return ``git status`` lines for local changes in a checkout. + + Lists tracked, untracked, and ignored changes in *checkout_dir* and its + initialized submodules. Returns an empty string when there are none. + Raises ``subprocess.CalledProcessError`` if Git cannot inspect it. + + .. versionadded:: 0.103.0 + """ + changes = external_commands.run(_GIT_STATUS_COMMAND, cwd=str(checkout_dir)) + if not changes: + # A parent repository can hide changes inside ignored submodules. + changes = external_commands.run( + [ + "git", + "submodule", + "--quiet", + "foreach", + "--recursive", + " ".join(_GIT_STATUS_COMMAND), + ], + cwd=str(checkout_dir), + ) + return changes + + +def git_checkout_mismatch( + *, + checkout_dir: pathlib.Path, + repo_url: str, + ref: str, +) -> str | None: + """Explain why a checkout differs from a fresh clone of *repo_url* at *ref*. + + Returns ``None`` when *checkout_dir* is the top of a Git checkout cloned + from *repo_url*, ``HEAD`` is at *ref*, all submodules are initialized at + their recorded commits, and there are no local changes. Otherwise returns + a short reason. Only local state is checked, so a tag that moved on the + remote after the clone is not detected. + + .. versionadded:: 0.103.0 + """ + if not (checkout_dir / ".git").exists(): + return "it is not a Git checkout" + cwd = str(checkout_dir) + try: + # Git searches parent directories when ".git" is broken. + toplevel = external_commands.run( + ["git", "rev-parse", "--show-toplevel"], cwd=cwd + ).strip() + if pathlib.Path(toplevel).resolve() != checkout_dir.resolve(): + return "it is not the top of a Git checkout" + origin = external_commands.run( + ["git", "remote", "get-url", "origin"], cwd=cwd + ).strip() + if origin != repo_url: + # Do not log URLs; they can contain credentials. + return "it was cloned from a different repository" + head = external_commands.run( + ["git", "rev-parse", "--verify", "HEAD"], cwd=cwd + ).strip() + wanted = external_commands.run( + ["git", "rev-parse", "--verify", f"{ref}^{{commit}}"], cwd=cwd + ).strip() + if head != wanted: + return f"HEAD is not at {ref!r}" + submodules = external_commands.run( + ["git", "submodule", "status", "--recursive"], cwd=cwd + ) + if any(line.startswith(("-", "+", "U")) for line in submodules.splitlines()): + return "a submodule is not initialized at its recorded commit" + if git_checkout_changes(checkout_dir): + return "it has local changes" + except subprocess.CalledProcessError: + return "Git cannot inspect it" + return None + + def parse_vcs_url(vcs_url: str, *, require_ref: bool = True) -> tuple[str, str]: """Parse a `pip VCS URL`_ into a repo clone URL and git ref. diff --git a/src/fromager/packagesettings/_resolver.py b/src/fromager/packagesettings/_resolver.py index 9e019a967..79d010819 100644 --- a/src/fromager/packagesettings/_resolver.py +++ b/src/fromager/packagesettings/_resolver.py @@ -12,7 +12,7 @@ import pydantic from packaging.utils import canonicalize_name -from .. import downloads, overrides, resolver +from .. import downloads, gitutils, overrides, resolver from ._typedefs import MODEL_CONFIG if typing.TYPE_CHECKING: @@ -124,6 +124,10 @@ def _download( Returns the local path and the kind of artifact that was downloaded. Dispatches to the appropriate download helper based on *download_kind*. + + An existing Git checkout is reused if it is a clean clone of the + same repository at the same commit. Otherwise it is removed and + cloned again when cleanup is enabled, or rejected when it is not. """ if download_kind not in self.download_kinds: raise ValueError( @@ -165,14 +169,31 @@ def _download( destination_dir = ( ctx.work_dir / f"{req.name}-{version}" / f"{req.name}-{version}" ) - if destination_dir.parent.exists(): - if ctx.cleanup: - shutil.rmtree(destination_dir.parent) - elif destination_dir.exists(): + if destination_dir.exists(): + # Reuse a checkout from an earlier download, for example + # from download-sequence, so builds can run offline. + repo_url, ref = gitutils.parse_vcs_url(download_url) + mismatch = gitutils.git_checkout_mismatch( + checkout_dir=destination_dir, repo_url=repo_url, ref=ref + ) + if mismatch is None: + logger.info("reusing Git checkout %s", destination_dir) + return DownloadedSource( + path=destination_dir, kind=download_kind + ) + if not ctx.cleanup: raise FileExistsError( - f"Git checkout {destination_dir} already exists; " - "enable cleanup or use a new work directory to download it again" + f"Git checkout {destination_dir} already exists and " + f"cannot be reused because {mismatch}; enable cleanup " + "or use a new work directory to download it again" ) + logger.info( + "cloning again: cannot reuse Git checkout %s because %s", + destination_dir, + mismatch, + ) + if destination_dir.parent.exists() and ctx.cleanup: + shutil.rmtree(destination_dir.parent) path = downloads.download_git_source( destination_dir=destination_dir, vcs_url=download_url, diff --git a/src/fromager/sources.py b/src/fromager/sources.py index 2dc527f12..1822db83f 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -554,29 +554,8 @@ def default_prepare_git_checkout( """ if not source_filename.is_dir() or not (source_filename / ".git").exists(): raise ValueError(f"{source_filename} is not a Git checkout") - status_command = ( - "git", - "status", - "--porcelain=v1", - "--untracked-files=all", - "--ignored=matching", - "--ignore-submodules=none", - ) try: - changes = external_commands.run(status_command, cwd=str(source_filename)) - if not changes: - # A parent repository can hide changes inside ignored submodules. - changes = external_commands.run( - [ - "git", - "submodule", - "--quiet", - "foreach", - "--recursive", - " ".join(status_command), - ], - cwd=str(source_filename), - ) + changes = gitutils.git_checkout_changes(source_filename) except subprocess.CalledProcessError as err: raise ValueError(f"cannot inspect Git checkout {source_filename}") from err if changes: diff --git a/tests/test_gitutils.py b/tests/test_gitutils.py index c13c32b11..b85801625 100644 --- a/tests/test_gitutils.py +++ b/tests/test_gitutils.py @@ -7,7 +7,13 @@ import pytest from packaging.version import Version -from fromager.gitutils import GIT_HEAD, git_clean, git_clone_fast, parse_vcs_url +from fromager.gitutils import ( + GIT_HEAD, + git_checkout_mismatch, + git_clean, + git_clone_fast, + parse_vcs_url, +) needs_git_command = pytest.mark.skipif( shutil.which("git") is None, reason="requires 'git' command" @@ -128,3 +134,137 @@ def test_git_clone_real(tmp_path: pathlib.Path) -> None: # detect version from .git assert setuptools_scm_version(tmp_path) == Version("0.73.0") + + +def _git(repo: pathlib.Path, *args: str) -> str: + """Run git in *repo* with a test identity and local file submodules.""" + result = subprocess.run( + [ + "git", + "-C", + str(repo), + "-c", + "user.name=Test", + "-c", + "user.email=test@pkg.test", + "-c", + "protocol.file.allow=always", + *args, + ], + check=True, + capture_output=True, + text=True, + ) + return result.stdout + + +def _make_repo(repo: pathlib.Path) -> pathlib.Path: + """Create a Git repository with one commit tagged ``v1.0``.""" + repo.mkdir() + _git(repo, "init") + (repo / "README").write_text("source") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "Initial source") + _git(repo, "tag", "v1.0") + return repo + + +def _clone(repo: pathlib.Path, checkout: pathlib.Path, ref: str) -> pathlib.Path: + """Clone *repo* to *checkout* and check out *ref* with submodules.""" + _git(repo.parent, "clone", "--recurse-submodules", str(repo), str(checkout)) + _git(checkout, "checkout", "--recurse-submodules", ref) + return checkout + + +@needs_git_command +def test_git_checkout_mismatch_accepts_clean_clone(tmp_path: pathlib.Path) -> None: + repo = _make_repo(tmp_path / "repo") + checkout = _clone(repo, tmp_path / "checkout", "refs/tags/v1.0") + + result = git_checkout_mismatch( + checkout_dir=checkout, repo_url=str(repo), ref="refs/tags/v1.0" + ) + + assert result is None + + +@needs_git_command +@pytest.mark.parametrize( + ("change", "reason"), + [ + ("untracked", "it has local changes"), + ("ignored", "it has local changes"), + ("other_ref", "HEAD is not at 'refs/tags/v1.0'"), + ("unknown_ref", "Git cannot inspect it"), + ("other_repo", "it was cloned from a different repository"), + ], +) +def test_git_checkout_mismatch_reports_reason( + tmp_path: pathlib.Path, change: str, reason: str +) -> None: + repo = _make_repo(tmp_path / "repo") + checkout = _clone(repo, tmp_path / "checkout", "refs/tags/v1.0") + ref = "refs/tags/v1.0" + repo_url = str(repo) + if change == "untracked": + (checkout / "new.txt").write_text("new") + elif change == "ignored": + (checkout / ".git" / "info" / "exclude").write_text("build/\n") + (checkout / "build").mkdir() + (checkout / "build" / "out.txt").write_text("built") + elif change == "other_ref": + _git(checkout, "commit", "--allow-empty", "-m", "Local commit") + elif change == "unknown_ref": + ref = "refs/tags/missing" + else: + repo_url = str(tmp_path / "other-repo") + + result = git_checkout_mismatch(checkout_dir=checkout, repo_url=repo_url, ref=ref) + + assert result == reason + + +@needs_git_command +def test_git_checkout_mismatch_rejects_uninitialized_submodule( + tmp_path: pathlib.Path, +) -> None: + sub = _make_repo(tmp_path / "sub") + repo = _make_repo(tmp_path / "repo") + _git(repo, "submodule", "add", str(sub), "sub") + _git(repo, "commit", "-m", "Add submodule") + _git(repo, "tag", "v2.0") + checkout = _clone(repo, tmp_path / "checkout", "refs/tags/v2.0") + assert ( + git_checkout_mismatch( + checkout_dir=checkout, repo_url=str(repo), ref="refs/tags/v2.0" + ) + is None + ) + _git(checkout, "submodule", "deinit", "--force", "sub") + + result = git_checkout_mismatch( + checkout_dir=checkout, repo_url=str(repo), ref="refs/tags/v2.0" + ) + + assert result == "a submodule is not initialized at its recorded commit" + + +@needs_git_command +def test_git_checkout_mismatch_rejects_non_checkout(tmp_path: pathlib.Path) -> None: + repo = _make_repo(tmp_path / "repo") + plain_dir = tmp_path / "plain" + plain_dir.mkdir() + # A broken ".git" makes Git search parent directories. + nested = repo / "nested" + nested.mkdir() + (nested / ".git").mkdir() + + plain_result = git_checkout_mismatch( + checkout_dir=plain_dir, repo_url=str(repo), ref="refs/tags/v1.0" + ) + nested_result = git_checkout_mismatch( + checkout_dir=nested, repo_url=str(repo), ref="refs/tags/v1.0" + ) + + assert plain_result == "it is not a Git checkout" + assert nested_result == "it is not the top of a Git checkout" diff --git a/tests/test_packagesettings_resolver.py b/tests/test_packagesettings_resolver.py index d4030ec1e..8f0e6370e 100644 --- a/tests/test_packagesettings_resolver.py +++ b/tests/test_packagesettings_resolver.py @@ -3,6 +3,7 @@ import dataclasses import pathlib import re +import shutil import subprocess import textwrap import typing @@ -303,6 +304,37 @@ def test_download_canonicalizes_tarball_name( ) +def _git(repo: pathlib.Path, *args: str) -> str: + """Run git in *repo* with a test identity and return its output.""" + result = subprocess.run( + [ + "git", + "-C", + str(repo), + "-c", + "user.name=Test", + "-c", + "user.email=test@pkg.test", + *args, + ], + check=True, + capture_output=True, + text=True, + ) + return result.stdout + + +def _make_tagged_repo(repo: pathlib.Path) -> pathlib.Path: + """Create a Git repository with one commit tagged ``v1.2.3``.""" + repo.mkdir() + _git(repo, "init") + (repo / "pyproject.toml").write_text("[build-system]\nrequires = []\n") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "Initial source") + _git(repo, "tag", "v1.2.3") + return repo + + class TestPyPIGitResolver: YAML = """\ source: @@ -433,28 +465,7 @@ def test_download_preserves_non_checkout_on_collision( def test_local_git_checkout_prepares_and_reclones( self, tmp_context: WorkContext, tmp_path: pathlib.Path ) -> None: - repo = tmp_path / "repo" - repo.mkdir() - subprocess.run(["git", "init", str(repo)], check=True, capture_output=True) - (repo / "pyproject.toml").write_text("[build-system]\nrequires = []\n") - subprocess.run(["git", "-C", str(repo), "add", "."], check=True) - subprocess.run( - [ - "git", - "-C", - str(repo), - "-c", - "user.name=Test", - "-c", - "user.email=test@pkg.test", - "commit", - "-m", - "Initial source", - ], - check=True, - capture_output=True, - ) - subprocess.run(["git", "-C", str(repo), "tag", "v1.2.3"], check=True) + repo = _make_tagged_repo(tmp_path / "repo") url = f"git+file://{repo}@refs/tags/v1.2.3" source_resolver = _parse(self.YAML) tmp_context.cleanup = False @@ -493,6 +504,70 @@ def test_local_git_checkout_prepares_and_reclones( assert not (package_dir / "other-dir").exists() assert not (recloned.path.parent / "build-meta.json").exists() + @pytest.mark.parametrize("cleanup", [True, False]) + def test_download_reuses_clean_checkout_offline( + self, tmp_context: WorkContext, tmp_path: pathlib.Path, cleanup: bool + ) -> None: + repo = _make_tagged_repo(tmp_path / "repo") + url = f"git+file://{repo}@refs/tags/v1.2.3" + source_resolver = _parse(self.YAML) + first = source_resolver.download(tmp_context, _REQ, _VERSION, url) + # Git status does not report files inside ".git". + marker = first.path / ".git" / "fromager-test-marker" + marker.write_text("first clone") + # Without the source repository, any clone attempt fails. + shutil.rmtree(repo) + tmp_context.cleanup = cleanup + + second = source_resolver.download(tmp_context, _REQ, _VERSION, url) + + assert second == first + assert marker.read_text() == "first clone" + assert (second.path / "pyproject.toml").is_file() + + @pytest.mark.parametrize( + ("change", "reason"), + [ + ("other_ref", "HEAD is not at 'refs/tags/v2.0'"), + ("other_repo", "cloned from a different repository"), + ], + ) + def test_download_does_not_reuse_mismatched_checkout( + self, + tmp_context: WorkContext, + tmp_path: pathlib.Path, + change: str, + reason: str, + ) -> None: + repo = _make_tagged_repo(tmp_path / "repo") + _git(repo, "commit", "--allow-empty", "-m", "Second commit") + _git(repo, "tag", "v2.0") + source_resolver = _parse(self.YAML) + first = source_resolver.download( + tmp_context, _REQ, _VERSION, f"git+file://{repo}@refs/tags/v1.2.3" + ) + marker = first.path / ".git" / "fromager-test-marker" + marker.write_text("first clone") + if change == "other_ref": + url = f"git+file://{repo}@refs/tags/v2.0" + else: + other_repo = tmp_path / "other-repo" + _git(tmp_path, "clone", str(repo), str(other_repo)) + url = f"git+file://{other_repo}@refs/tags/v1.2.3" + + tmp_context.cleanup = False + with pytest.raises(FileExistsError, match=re.escape(reason)): + source_resolver.download(tmp_context, _REQ, _VERSION, url) + assert marker.is_file() + + tmp_context.cleanup = True + recloned = source_resolver.download(tmp_context, _REQ, _VERSION, url) + + assert recloned.path == first.path + assert not marker.exists() + expected_head = _git(repo, "rev-parse", url.rpartition("@")[2]).strip() + assert _git(recloned.path, "rev-parse", "HEAD").strip() == expected_head + # -- Git source resolvers ---------------------------------------------------- From 88da44f9457d4389a911d18de9c6f12647711758 Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Mon, 2 Nov 2026 14:24:15 -0500 Subject: [PATCH 26/35] fix(sources): save sdists with a standard filename Downloaded sdists used the filename from the URL. Some URLs have no proper filename, so the download failed or fromager could not find the file later. Now they are saved as `{name}-{version}.tar.gz`, like tarballs. Fixes #1430 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Lalatendu Mohanty --- docs/reference/hooks.rst | 9 ++- src/fromager/downloads.py | 13 ++++ src/fromager/packagesettings/_resolver.py | 19 +++-- src/fromager/sources.py | 7 +- tests/test_downloads.py | 13 ++++ tests/test_packagesettings_resolver.py | 93 ++++++++++++++++++++++- 6 files changed, 135 insertions(+), 19 deletions(-) diff --git a/docs/reference/hooks.rst b/docs/reference/hooks.rst index a1d40ce80..b1e9e6175 100644 --- a/docs/reference/hooks.rst +++ b/docs/reference/hooks.rst @@ -280,10 +280,11 @@ Source hooks neither failures nor invalid results trigger a fallback download. If the download hook is absent, ``hook-sdist`` downloads the candidate - URL as an sdist into ``ctx.sdists_downloads``. ``hook-prebuilt`` downloads - it as a wheel into ``ctx.wheels_prebuilt``. These defaults use the - standard archive and wheel validation. Custom tarball URLs and git - downloads require a download hook. + URL as an sdist into ``ctx.sdists_downloads``, saved as + ``{canonical name}-{version}.tar.gz`` (or ``.zip``) whatever the URL's + filename. ``hook-prebuilt`` downloads it as a wheel into + ``ctx.wheels_prebuilt``. These defaults use the standard archive and + wheel validation. Git downloads require a download hook. Without a ``get_resolver_provider`` hook, ``hook-sdist`` finds versions on its ``index_url`` in sdists and wheels for any platform, diff --git a/src/fromager/downloads.py b/src/fromager/downloads.py index e6ffa28a3..43eb14b76 100644 --- a/src/fromager/downloads.py +++ b/src/fromager/downloads.py @@ -27,6 +27,19 @@ def extract_filename_from_url(url: str) -> str: return unquote(filename) +def sdist_extension_for_url(url: str) -> str: + """Return the sdist extension to save *url* with. + + Returns ``.zip`` if the URL's filename ends with ``.zip``, otherwise + ``.tar.gz``, including for a URL without a filename. + + .. versionadded:: 0.103.0 + """ + if extract_filename_from_url(url).endswith(".zip"): + return ".zip" + return ".tar.gz" + + def download_url( *, destination_dir: pathlib.Path, diff --git a/src/fromager/packagesettings/_resolver.py b/src/fromager/packagesettings/_resolver.py index 79d010819..815b8ad69 100644 --- a/src/fromager/packagesettings/_resolver.py +++ b/src/fromager/packagesettings/_resolver.py @@ -123,7 +123,9 @@ def _download( Returns the local path and the kind of artifact that was downloaded. Dispatches to the appropriate download helper - based on *download_kind*. + based on *download_kind*. Sdists and tarballs are saved as + ``{canonical name}-{version}``. Tarballs always use ``.tar.gz``; + sdists use ``.zip`` if the URL ends with it, otherwise ``.tar.gz``. An existing Git checkout is reused if it is a clean clone of the same repository at the same commit. Otherwise it is removed and @@ -134,26 +136,31 @@ def _download( f"provider {self.provider!r} does not support download kind " f"{download_kind!r}, expected one of {self.download_kinds!r}" ) + # URL filenames can be missing or nonstandard (for example a + # "/tarball/v1.0" tag URL), so save archives as "{name}-{version}" + # where find_sdist() and later downloads can find them. + # Legacy downloads use the underscore name ("test_pkg") instead. + # find_sdist() accepts both, and one package cannot use both paths + # because mixing source: with legacy settings is rejected. + archive_basename = f"{canonicalize_name(req.name)}-{version}" match download_kind: case DownloadKind.sdist: destination_dir = ctx.sdists_downloads + extension = downloads.sdist_extension_for_url(download_url) path = downloads.download_sdist( destination_dir=destination_dir, url=download_url, + destination_filename=f"{archive_basename}{extension}", ) case DownloadKind.tarball: # TODO: A tarball is not a proper sdist; it should not # live in sdists_downloads. Using it here for now until # a dedicated tarballs directory is introduced. destination_dir = ctx.sdists_downloads - # GitHub tarball URLs have no usable filename in the - # path, so always provide a destination filename. path = downloads.download_url( destination_dir=destination_dir, url=download_url, - destination_filename=( - f"{canonicalize_name(req.name)}-{version}.tar.gz" - ), + destination_filename=f"{archive_basename}.tar.gz", ) case DownloadKind.prebuilt_wheel: destination_dir = ctx.wheels_prebuilt diff --git a/src/fromager/sources.py b/src/fromager/sources.py index 1822db83f..226542ab6 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -256,11 +256,8 @@ def default_download_source( if url is None: raise ValueError(f"Could not determine download URL for {req}") if destination_filename is None: - url_filename = downloads.extract_filename_from_url(url) - if url_filename.endswith(".zip"): - destination_filename = f"{pbi.override_module_name}-{version}.zip" - else: - destination_filename = f"{pbi.override_module_name}-{version}.tar.gz" + extension = downloads.sdist_extension_for_url(url) + destination_filename = f"{pbi.override_module_name}-{version}{extension}" logger.debug( "config has no destination_filename, use default %r", destination_filename ) diff --git a/tests/test_downloads.py b/tests/test_downloads.py index 94053ae5a..0b51a79cd 100644 --- a/tests/test_downloads.py +++ b/tests/test_downloads.py @@ -20,6 +20,7 @@ download_url, download_wheel, extract_filename_from_url, + sdist_extension_for_url, ) # -- extract_filename_from_url ------------------------------------------------ @@ -37,6 +38,18 @@ def test_extract_filename_from_url(url: str, expected: str) -> None: assert extract_filename_from_url(url) == expected +@pytest.mark.parametrize( + "url, expected", + [ + ("https://pkg.test/pkg-1.0.zip", ".zip"), + ("https://pkg.test/pkg-1.0.tar.gz", ".tar.gz"), + ("https://api.pkg.test/repos/org/repo/tarball/v1.0", ".tar.gz"), + ], +) +def test_sdist_extension_for_url(url: str, expected: str) -> None: + assert sdist_extension_for_url(url) == expected + + # -- download_url ------------------------------------------------------------- _PKG_URL = "https://pkg.test/pkg-1.0.tar.gz" diff --git a/tests/test_packagesettings_resolver.py b/tests/test_packagesettings_resolver.py index 8f0e6370e..6734746fd 100644 --- a/tests/test_packagesettings_resolver.py +++ b/tests/test_packagesettings_resolver.py @@ -1,12 +1,15 @@ from __future__ import annotations import dataclasses +import io import pathlib import re import shutil import subprocess +import tarfile import textwrap import typing +import zipfile from unittest import mock import pydantic @@ -16,7 +19,7 @@ from packaging.version import Version from wheel.wheelfile import WheelFile -from fromager import resolver, sources +from fromager import finders, resolver, sources from fromager.context import WorkContext from fromager.packagesettings._resolver import ( AbstractResolver, @@ -169,6 +172,7 @@ def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> No mock_dl.assert_called_once_with( destination_dir=tmp_context.sdists_downloads, url=_URL_SDIST, + destination_filename="test-pkg-1.2.3.tar.gz", ) @@ -1262,12 +1266,93 @@ def test_hook_download_uses_profile_default_when_hook_is_missing( ): result = r.download(tmp_context, _REQ, _VERSION, url) - selected, unused = (wheel, sdist) if prebuilt else (sdist, wheel) - selected.assert_called_once_with(destination_dir=destination, url=url) - unused.assert_not_called() + if prebuilt: + wheel.assert_called_once_with(destination_dir=destination, url=url) + sdist.assert_not_called() + else: + sdist.assert_called_once_with( + destination_dir=destination, + url=url, + destination_filename="test-pkg-1.2.3.tar.gz", + ) + wheel.assert_not_called() assert result == DownloadedSource(path=expected, kind=kind) +def _fake_archive_download( + *, destination_dir: pathlib.Path, url: str, destination_filename: str +) -> pathlib.Path: + """Write a small archive in place of a network download.""" + path = destination_dir / destination_filename + destination_dir.mkdir(parents=True, exist_ok=True) + if destination_filename.endswith(".zip"): + with zipfile.ZipFile(path, "w") as zf: + zf.writestr("pkg/setup.py", "") + else: + with tarfile.open(path, "w:gz") as tf: + info = tarfile.TarInfo("pkg/setup.py") + tf.addfile(info, io.BytesIO(b"")) + return path + + +@pytest.mark.parametrize( + ("yaml_text", "url", "expected_name"), + [ + pytest.param( + """\ + source: + provider: hook-sdist + build_sdist: pep517 + """, + _URL_GITHUB_TARBALL, + "test-pkg-1.2.3.tar.gz", + id="hook-sdist-tag-url", + ), + pytest.param( + """\ + source: + provider: pypi-download + download_url: https://download.test/pkg-v{version}.tar.gz + download_kind: sdist + """, + "https://download.test/pkg-v1.2.3.tar.gz", + "test-pkg-1.2.3.tar.gz", + id="pypi-download-prefixed-version", + ), + pytest.param( + """\ + source: + provider: pypi-sdist + """, + "https://pypi.test/packages/Test_Pkg-1.2.3.zip", + "test-pkg-1.2.3.zip", + id="pypi-sdist-zip", + ), + ], +) +def test_sdist_download_uses_standard_filename( + tmp_context: WorkContext, yaml_text: str, url: str, expected_name: str +) -> None: + """Save sdists so ``find_sdist()`` finds them by name and version.""" + r = _parse(yaml_text) + + with ( + mock.patch("fromager.overrides.find_override_method", return_value=None), + mock.patch( + "fromager.downloads.download_url", side_effect=_fake_archive_download + ), + ): + result = r.download(tmp_context, _REQ, _VERSION, url) + + assert result == DownloadedSource( + path=tmp_context.sdists_downloads / expected_name, kind=DownloadKind.sdist + ) + found = finders.find_sdist( + tmp_context, tmp_context.sdists_downloads, _REQ, str(_VERSION) + ) + assert found == result.path + + @pytest.mark.parametrize("provider", ["hook-sdist", "hook-prebuilt"]) def test_hook_download_chains_failure_without_fallback( tmp_context: WorkContext, provider: str From c625217871198a8c76f6cd2a5293ed2d90a8cf5f Mon Sep 17 00:00:00 2001 From: Jenna Date: Wed, 28 Oct 2026 14:28:27 -0400 Subject: [PATCH 27/35] feat(package): honor configured source resolvers Display source resolver details and resolve through the configured provider. Preserve distribution type overrides for legacy package settings. Closes: #1400 Co-Authored-By: Codex Signed-off-by: Jenna --- src/fromager/commands/package.py | 95 +++++++++++++++----- tests/test_package.py | 147 +++++++++++++++++++++++++++++++ 2 files changed, 219 insertions(+), 23 deletions(-) create mode 100644 tests/test_package.py diff --git a/src/fromager/commands/package.py b/src/fromager/commands/package.py index ef3c88a05..2aebd4b3c 100644 --- a/src/fromager/commands/package.py +++ b/src/fromager/commands/package.py @@ -7,6 +7,7 @@ import sys import typing from collections import defaultdict +from urllib.parse import urlsplit import click import pypi_simple @@ -31,6 +32,15 @@ logger = logging.getLogger(__name__) +def _redact_url_credentials(url: object) -> str: + """Remove credentials from a URL before displaying it.""" + url_string = str(url) + parsed_url = urlsplit(url_string) + if "@" not in parsed_url.netloc: + return url_string + return parsed_url._replace(netloc=parsed_url.netloc.rsplit("@", 1)[1]).geturl() + + def sdist_server_url_option(func: typing.Callable) -> typing.Callable: """Add --sdist-server-url Click option""" return click.option( @@ -459,8 +469,7 @@ def _resolve_fromager( ) -> set[Version] | None: # print package build info settings (overrides) pbi = ctx.package_build_info(req) - override_sdist_server_url = pbi.resolver_sdist_server_url(sdist_server_url) - include_sdists, include_wheels = parse_distribution_option(distribution_type, pbi) + source_resolver = pbi.source_resolver if pbi.has_customizations: click.secho( @@ -469,15 +478,41 @@ def _resolve_fromager( ) else: click.secho("Package uses standard settings.") - click.secho(f"- sdist server url: {override_sdist_server_url}") - click.secho(f"- resolver includes sdists: {pbi.resolver_include_sdists}") - click.secho(f"- resolver includes wheels: {pbi.resolver_include_wheels}") - click.secho( - f"- using distribution type '{distribution_type.value}' (sdists: {include_sdists}, wheels: {include_wheels})" - ) - click.secho(f"- wheel server url: {pbi.wheel_server_url}") - click.secho(f"- download url: {pbi.download_source_url(resolve_template=False)}") - click.secho(f"- prebuilt wheel: {pbi.pre_built}") + if source_resolver is not None: + click.secho("- source:") + click.secho(f" - provider: {source_resolver.provider}") + click.secho( + f" - resolves_prebuilt_wheel: {source_resolver.resolves_prebuilt_wheel}" + ) + click.secho( + f" - min_release_age: {getattr(source_resolver, 'min_release_age', None)}" + ) + for url_field in ("index_url", "project_url", "clone_url", "download_url"): + url = getattr(source_resolver, url_field, None) + if url is not None: + click.secho(f" - {url_field}: {_redact_url_credentials(url)}") + if distribution_type is not DistributionType.DEFAULT: + click.secho( + f"- distribution type '{distribution_type.value}' is ignored " + "for configured source resolvers" + ) + else: + override_sdist_server_url = pbi.resolver_sdist_server_url(sdist_server_url) + include_sdists, include_wheels = parse_distribution_option( + distribution_type, pbi + ) + click.secho(f"- sdist server url: {override_sdist_server_url}") + click.secho(f"- resolver includes sdists: {pbi.resolver_include_sdists}") + click.secho(f"- resolver includes wheels: {pbi.resolver_include_wheels}") + click.secho( + f"- using distribution type '{distribution_type.value}' " + f"(sdists: {include_sdists}, wheels: {include_wheels})" + ) + click.secho(f"- wheel server url: {pbi.wheel_server_url}") + click.secho( + f"- download url: {pbi.download_source_url(resolve_template=False)}" + ) + click.secho(f"- prebuilt wheel: {pbi.pre_built}") if constraint is not None: click.secho( f"- package is constrained by '--constraints-file': {constraint}.", @@ -486,18 +521,28 @@ def _resolve_fromager( else: click.secho("- package is not constrained by '--constraints-file'") - # resolve package with Fromager's resolver and settings. - provider = overrides.find_and_invoke( - req.name, - "get_resolver_provider", - resolver.default_resolver_provider, - ctx=ctx, - req=req, - include_sdists=include_sdists, - include_wheels=include_wheels, - sdist_server_url=override_sdist_server_url, - ignore_platform=pbi.resolver_ignore_platform, - ) + # Resolve with configured source settings or the unchanged legacy path. + if source_resolver is not None: + provider = sources.get_source_provider( + ctx=ctx, + req=req, + sdist_server_url=sdist_server_url, + ) + else: + provider = overrides.find_and_invoke( + req.name, + "get_resolver_provider", + resolver.default_resolver_provider, + ctx=ctx, + req=req, + include_sdists=include_sdists, + include_wheels=include_wheels, + sdist_server_url=override_sdist_server_url, + ignore_platform=pbi.resolver_ignore_platform, + ) + + # Diagnostic resolution includes releases that are too recent to build. + provider.cooldown = None click.echo() click.secho(f"get_resolver_provider returns provider '{type(provider).__name__}'") @@ -645,6 +690,10 @@ def resolve( way. First, it resolves with Fromager's resolver. Second, it performs a simple query against PyPI and compares the results. + Packages with source configuration use their configured resolver. + Distribution type options apply to legacy configuration. Diagnostic + resolution includes recent releases regardless of release-age cooldowns. + Usage:: $ fromager --variant cpu package resolve 'fromager>=0.70' diff --git a/tests/test_package.py b/tests/test_package.py new file mode 100644 index 000000000..8c25f25c8 --- /dev/null +++ b/tests/test_package.py @@ -0,0 +1,147 @@ +"""Tests for package resolver configuration output.""" + +from unittest.mock import Mock, patch + +import pytest +from packaging.requirements import Requirement +from packaging.version import Version + +from fromager import context, packagesettings, resolver +from fromager.candidate import Candidate +from fromager.commands import package as package_command + + +def _configure_package(ctx: context.WorkContext, yaml_str: str) -> None: + """Apply one package setting to the test context.""" + ps = packagesettings.PackageSettings.from_string("pkg", yaml_str) + ctx.settings._package_settings[ps.name] = ps + + +def test_resolve_fromager_displays_and_uses_source_config( + tmp_context: context.WorkContext, + capsys: pytest.CaptureFixture[str], +) -> None: + """Display source settings and resolve through the configured provider.""" + _configure_package( + tmp_context, + "source:\n" + " provider: pypi-prebuilt\n" + " index_url: https://index-user:index-password@pkg.test/simple/\n" + " min_release_age: 7\n", + ) + req = Requirement("pkg") + candidate = Candidate( + name=req.name, + version=Version("1.0"), + url="https://pkg.test/pkg-1.0-py3-none-any.whl", + ) + + with ( + patch.object( + package_command.sources, + "get_source_provider", + wraps=package_command.sources.get_source_provider, + ) as get_source_provider, + patch.object( + resolver.PyPIProvider, + "find_matches", + autospec=True, + return_value=[candidate], + ) as find_matches, + ): + versions = package_command._resolve_fromager( + tmp_context, + req, + constraint=None, + distribution_type=package_command.DistributionType.SDIST, + sdist_server_url="https://caller.test/simple/", + ) + + output = capsys.readouterr().out + assert "- source:" in output + assert " - provider: pypi-prebuilt" in output + assert " - resolves_prebuilt_wheel: True" in output + assert " - min_release_age: 7" in output + assert " - index_url: https://pkg.test/simple/" in output + assert "index-user" not in output + assert "index-password" not in output + assert ( + "- distribution type 'sdist' is ignored for configured source resolvers" + in output + ) + assert "- resolver includes sdists:" not in output + assert "- download url:" not in output + assert versions == {Version("1.0")} + get_source_provider.assert_called_once_with( + ctx=tmp_context, + req=req, + sdist_server_url="https://caller.test/simple/", + ) + find_matches.assert_called_once() + source_provider = find_matches.call_args.args[0] + assert source_provider.cooldown is None + + +def test_resolve_fromager_preserves_legacy_config_output( + tmp_context: context.WorkContext, + capsys: pytest.CaptureFixture[str], +) -> None: + """Keep legacy resolver and download settings visible.""" + _configure_package( + tmp_context, + "download_source:\n" + " url: https://pkg.test/pkg-${version}.tar.gz\n" + "resolver_dist:\n" + " sdist_server_url: https://legacy.test/simple/\n" + " include_sdists: true\n" + " include_wheels: false\n" + "variants:\n" + " cpu:\n" + " wheel_server_url: https://wheels.test/simple/\n" + " pre_built: false\n", + ) + req = Requirement("pkg") + candidate = Candidate( + name=req.name, + version=Version("1.0"), + url="https://pkg.test/pkg-1.0.tar.gz", + ) + provider = Mock() + provider.cooldown = object() + provider.find_matches.return_value = [candidate] + + with patch.object( + package_command.sources.overrides, + "find_and_invoke", + return_value=provider, + ) as find_and_invoke: + versions = package_command._resolve_fromager( + tmp_context, + req, + constraint=None, + distribution_type=package_command.DistributionType.WHEEL, + sdist_server_url="https://caller.test/simple/", + ) + + output = capsys.readouterr().out + assert "- source:" not in output + assert "- sdist server url: https://legacy.test/simple/" in output + assert "- resolver includes sdists: True" in output + assert "- resolver includes wheels: False" in output + assert "- using distribution type 'wheel' (sdists: False, wheels: True)" in output + assert "- wheel server url: https://wheels.test/simple/" in output + assert "- download url: https://pkg.test/pkg-${version}.tar.gz" in output + assert "- prebuilt wheel: False" in output + assert versions == {Version("1.0")} + assert provider.cooldown is None + find_and_invoke.assert_called_once_with( + req.name, + "get_resolver_provider", + resolver.default_resolver_provider, + ctx=tmp_context, + req=req, + include_sdists=False, + include_wheels=True, + sdist_server_url="https://legacy.test/simple/", + ignore_platform=False, + ) From 6bab2ee1d748420c67acb0dc4953e978f27fae0c Mon Sep 17 00:00:00 2001 From: Shanmukh Pawan Date: Tue, 3 Nov 2026 08:23:38 -0500 Subject: [PATCH 28/35] fix(sources): canonicalize Git checkout paths Share the checkout path between configured Git downloads and step replay. Reuse clean checkouts when requirement spelling changes, and document the path for source hooks. Closes: #1434 Co-Authored-By: Codex Signed-off-by: Shanmukh Pawan --- docs/concepts/resolver-architecture.rst | 3 ++- docs/reference/hooks.rst | 10 +++++++-- src/fromager/commands/step.py | 5 +++-- src/fromager/gitutils.py | 23 ++++++++++++++++++++ src/fromager/packagesettings/_resolver.py | 7 +++--- tests/test_commands.py | 26 +++++++++++++++++++++++ tests/test_finders.py | 1 + tests/test_gitutils.py | 17 +++++++++++++++ tests/test_packagesettings_resolver.py | 14 ++++++++++-- 9 files changed, 96 insertions(+), 10 deletions(-) diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index c8782ca72..6610a0776 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -147,7 +147,8 @@ propagate with context instead of triggering a fallback. In-process callers pass the downloaded path and kind to preparation: ``sdist`` and ``tarball`` are unpacked, while ``git_checkout`` uses the checkout directory. The checkout uses the same double-nested work directory pattern as an unpacked archive; -the directory names can differ for normalized package names. +its directory names use the canonical package name, even when the requirement +uses another spelling. The default Git preparer rejects a checkout with local changes before applying patches, project overrides, and Rust vendoring. Git status does not prove that the checkout is the original clone or preserve it after preparation. Build metadata diff --git a/docs/reference/hooks.rst b/docs/reference/hooks.rst index b1e9e6175..f98830450 100644 --- a/docs/reference/hooks.rst +++ b/docs/reference/hooks.rst @@ -115,6 +115,9 @@ Finder hooks The return value should be a string with the name of the source root directory relative to the ``ctx.work_dir`` where it was prepared. + For a configured Git checkout, this path uses the canonical distribution + name. Return the actual prepared path; an override bypasses the standard + source directory search. Resolver hooks -------------- @@ -352,6 +355,8 @@ Source hooks The override returns the source root ``Path`` or a ``(Path, bool)`` pair. Git checkouts must use the double-nested package/version layout under ``ctx.work_dir`` so the later bootstrap and step commands can find them. + Use :func:`~fromager.gitutils.git_checkout_dir` to construct this path; + it canonicalizes the package name before building the directory names. .. versionchanged:: 0.103.0 Passing ``DownloadKind.git_checkout`` prepares a clean Git checkout. @@ -371,8 +376,9 @@ Source hooks Source helper functions ~~~~~~~~~~~~~~~~~~~~~~~ -The following helper functions are available in the ``fromager.sources`` module -for use in custom source hooks: +The following helpers are available for use in custom source hooks: + +.. autofunction:: fromager.gitutils.git_checkout_dir .. autofunction:: fromager.sources.default_prepare_git_checkout diff --git a/src/fromager/commands/step.py b/src/fromager/commands/step.py index f9ea3ef04..36b5b45d4 100644 --- a/src/fromager/commands/step.py +++ b/src/fromager/commands/step.py @@ -11,6 +11,7 @@ clickext, context, finders, + gitutils, packagesettings, server, sources, @@ -120,8 +121,8 @@ def prepare_source( token = requirement_ctxvar.set(req) download_kind = packagesettings.DownloadKind(source_kind) sdists_downloads = pathlib.Path(wkctx.sdists_repo) / "downloads" - checkout = ( - wkctx.work_dir / f"{req.name}-{dist_version}" / f"{req.name}-{dist_version}" + checkout = gitutils.git_checkout_dir( + work_dir=wkctx.work_dir, distribution_name=req.name, version=dist_version ) source_filename: pathlib.Path | None if source_path is not None: diff --git a/src/fromager/gitutils.py b/src/fromager/gitutils.py index 20af13351..167602b90 100644 --- a/src/fromager/gitutils.py +++ b/src/fromager/gitutils.py @@ -1,3 +1,5 @@ +"""Helpers for cloning, checking, and locating Git source trees.""" + from __future__ import annotations import logging @@ -7,6 +9,8 @@ from urllib.parse import urlparse from packaging.requirements import Requirement +from packaging.utils import canonicalize_name +from packaging.version import Version from fromager import external_commands @@ -19,6 +23,25 @@ GIT_HEAD: typing.Final = "HEAD" +def git_checkout_dir( + *, work_dir: pathlib.Path, distribution_name: str, version: Version +) -> pathlib.Path: + """Return the standard checkout path using the canonical distribution name. + + Args: + work_dir: Directory that contains package work directories. + distribution_name: Package name in any valid spelling. + version: Resolved package version. + + Returns: + Double-nested checkout path for the package and version. + + .. versionadded:: 0.103.0 + """ + dirname = f"{canonicalize_name(distribution_name)}-{version}" + return work_dir / dirname / dirname + + def git_clone( *, ctx: context.WorkContext, diff --git a/src/fromager/packagesettings/_resolver.py b/src/fromager/packagesettings/_resolver.py index 815b8ad69..6a05a5d81 100644 --- a/src/fromager/packagesettings/_resolver.py +++ b/src/fromager/packagesettings/_resolver.py @@ -142,7 +142,8 @@ def _download( # Legacy downloads use the underscore name ("test_pkg") instead. # find_sdist() accepts both, and one package cannot use both paths # because mixing source: with legacy settings is rejected. - archive_basename = f"{canonicalize_name(req.name)}-{version}" + name = canonicalize_name(req.name) + archive_basename = f"{name}-{version}" match download_kind: case DownloadKind.sdist: destination_dir = ctx.sdists_downloads @@ -173,8 +174,8 @@ def _download( # 'work_dir / f"{name}-{version}" / "checkout"' for # checkout, generate a proper sdist, then unpack to # final destination? - destination_dir = ( - ctx.work_dir / f"{req.name}-{version}" / f"{req.name}-{version}" + destination_dir = gitutils.git_checkout_dir( + work_dir=ctx.work_dir, distribution_name=name, version=version ) if destination_dir.exists(): # Reuse a checkout from an earlier download, for example diff --git a/tests/test_commands.py b/tests/test_commands.py index b68b777e0..b00f12cf2 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1085,6 +1085,32 @@ def test_step_prepare_source_prepares_clean_git_checkout( assert (checkout.parent / "build-meta.json").is_file() +def test_step_prepare_source_finds_canonical_git_checkout( + cli_runner: CliRunner, tmp_context: WorkContext +) -> None: + """Find a checkout when the CLI package name uses another spelling.""" + checkout = tmp_context.work_dir / "test-pkg-1.0" / "test-pkg-1.0" + checkout.mkdir(parents=True) + + with patch.object( + step.sources, "prepare_source", return_value=checkout + ) as prepare_source: + result = cli_runner.invoke( + step.step, + ["prepare-source", "--source-kind", "git_checkout", "test_pkg", "1.0"], + obj=tmp_context, + ) + + assert result.exit_code == 0, result.output + prepare_source.assert_called_once_with( + ctx=tmp_context, + req=Requirement("test_pkg==1.0"), + source_filename=checkout, + version=Version("1.0"), + download_kind=packagesettings.DownloadKind.git_checkout, + ) + + def test_step_prepare_source_kind_selects_checkout_over_archive( cli_runner: CliRunner, tmp_context: WorkContext ) -> None: diff --git a/tests/test_finders.py b/tests/test_finders.py index 691fdfebf..b7752792d 100644 --- a/tests/test_finders.py +++ b/tests/test_finders.py @@ -89,6 +89,7 @@ def test_find_wheel_returns_none_when_only_non_wheel_files( [ ("mypkg", "1.2", "mypkg-1.2", "mypkg-1.2"), ("oslo.messaging", "14.7.0", "oslo.messaging-14.7.0", "oslo.messaging-14.7.0"), + ("zope_event", "1.0", "zope-event-1.0", "zope-event-1.0"), ("cython", "3.0.10", "Cython-3.0.10", "Cython-3.0.10"), ("ruamel-yaml", "0.18.6", "ruamel.yaml-0.18.6", "ruamel.yaml-0.18.6"), ], diff --git a/tests/test_gitutils.py b/tests/test_gitutils.py index b85801625..2dac91d5d 100644 --- a/tests/test_gitutils.py +++ b/tests/test_gitutils.py @@ -9,6 +9,7 @@ from fromager.gitutils import ( GIT_HEAD, + git_checkout_dir, git_checkout_mismatch, git_clean, git_clone_fast, @@ -20,6 +21,22 @@ ) +@pytest.mark.parametrize( + "distribution_name", ["zope_event", "zope.event", "Zope-Event"] +) +def test_git_checkout_dir_uses_canonical_name( + tmp_path: pathlib.Path, distribution_name: str +) -> None: + """Equivalent distribution names share one checkout path.""" + checkout = git_checkout_dir( + work_dir=tmp_path, + distribution_name=distribution_name, + version=Version("1.0"), + ) + + assert checkout == tmp_path / "zope-event-1.0" / "zope-event-1.0" + + def setuptools_scm_version(root_dir: pathlib.Path) -> Version: out = subprocess.check_output( [sys.executable, "-m", "setuptools_scm"], diff --git a/tests/test_packagesettings_resolver.py b/tests/test_packagesettings_resolver.py index 6734746fd..5f05ec123 100644 --- a/tests/test_packagesettings_resolver.py +++ b/tests/test_packagesettings_resolver.py @@ -509,13 +509,20 @@ def test_local_git_checkout_prepares_and_reclones( assert not (recloned.path.parent / "build-meta.json").exists() @pytest.mark.parametrize("cleanup", [True, False]) + @pytest.mark.parametrize("first_name", ["test-pkg", "test_pkg"]) def test_download_reuses_clean_checkout_offline( - self, tmp_context: WorkContext, tmp_path: pathlib.Path, cleanup: bool + self, + tmp_context: WorkContext, + tmp_path: pathlib.Path, + cleanup: bool, + first_name: str, ) -> None: repo = _make_tagged_repo(tmp_path / "repo") url = f"git+file://{repo}@refs/tags/v1.2.3" source_resolver = _parse(self.YAML) - first = source_resolver.download(tmp_context, _REQ, _VERSION, url) + first = source_resolver.download( + tmp_context, Requirement(first_name), _VERSION, url + ) # Git status does not report files inside ".git". marker = first.path / ".git" / "fromager-test-marker" marker.write_text("first clone") @@ -526,6 +533,9 @@ def test_download_reuses_clean_checkout_offline( second = source_resolver.download(tmp_context, _REQ, _VERSION, url) assert second == first + assert second.path == ( + tmp_context.work_dir / "test-pkg-1.2.3" / "test-pkg-1.2.3" + ) assert marker.read_text() == "first clone" assert (second.path / "pyproject.toml").is_file() From 30c637ee41a4d91c98cde00d0d7680d6e22e2d14 Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Tue, 3 Nov 2026 21:05:32 -0500 Subject: [PATCH 29/35] fix(sources): reuse checkouts and infer source preparation kinds Read the first origin URL from the checkout config without URL rewrites or credential logging. Cover global configuration, rewrites, and multiple origins; isolate tests from user Git settings. Find existing source roots in a consistent order and infer the sole supported source kind for preparation. Keep the sdist fallback for legacy and ambiguous settings, excluding prebuilt wheels from inference. Update the tests and usage documentation. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Codex Signed-off-by: Lalatendu Mohanty --- docs/concepts/resolver-architecture.rst | 5 +- docs/using.md | 9 ++- src/fromager/commands/step.py | 32 +++++++-- src/fromager/finders.py | 9 ++- src/fromager/gitutils.py | 14 ++-- tests/conftest.py | 8 +++ tests/test_commands.py | 88 +++++++++++++++++++++++++ tests/test_finders.py | 29 ++++++++ tests/test_gitutils.py | 69 +++++++++++++++++++ 9 files changed, 248 insertions(+), 15 deletions(-) diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index 6610a0776..58a17506f 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -162,7 +162,10 @@ The separate ``step prepare-source`` invocation accepts ``--source-kind`` and ``--source-path`` because it cannot retrieve the earlier Python download result. ``step download-source-archive --output-format json`` exposes that result's actual path and kind without changing its default path-only output. -Without ``--source-kind``, it keeps the legacy archive behavior. +Without ``--source-kind``, it uses the provider's only supported source kind: +``sdist``, ``tarball``, or ``git_checkout``. Legacy settings and providers with +multiple kinds keep the ``sdist`` default. Prebuilt wheels are not source kinds +and are excluded from inference. ``download-sequence`` calls the configured resolver's normal ``download()`` for source entries. Both sequence commands check each entry's recorded ``source_provider`` against current settings, including prebuilt entries whose diff --git a/docs/using.md b/docs/using.md index 76fa9198c..3ddf68b68 100644 --- a/docs/using.md +++ b/docs/using.md @@ -258,9 +258,12 @@ private index server is more robust. The `step prepare-source` command unpacks a source archive and applies any patches (refer to [customization](customization.md) for details about patching). -It defaults to the archive kind for existing scripts. For a Git checkout, pass -`--source-kind git_checkout`. Pass `--source-path` when the downloaded source -cannot be found at its standard location. Git checkouts must use the standard +Without `--source-kind`, it uses the provider's only supported source kind +(`sdist`, `tarball`, or `git_checkout`). Legacy settings and providers with +multiple kinds default to `sdist`; prebuilt wheels are excluded from inference. +Pass `--source-kind` when the kind cannot be inferred, such as a Git checkout +returned by a hook. Pass `--source-path` when the downloaded source cannot be +found at its standard location. Git checkouts must use the standard double-nested work directory layout so later build steps can find them. The download command's optional `--output-format json` prints both `path` and `kind` when a hook can return diff --git a/src/fromager/commands/step.py b/src/fromager/commands/step.py index 36b5b45d4..554e16f01 100644 --- a/src/fromager/commands/step.py +++ b/src/fromager/commands/step.py @@ -81,6 +81,23 @@ def download_source_archive( print(downloaded_source.path) +def _default_source_kind( + wkctx: context.WorkContext, req: Requirement +) -> packagesettings.DownloadKind: + """Return the source kind to use when ``--source-kind`` is not given. + + Use the provider's only supported source kind. Legacy settings, providers + with multiple kinds, and prebuilt profiles keep the ``sdist`` default. + """ + source_resolver = wkctx.package_build_info(req).source_resolver + kinds = ( + source_resolver.download_kinds if source_resolver is not None else frozenset() + ) + if len(kinds) == 1 and packagesettings.DownloadKind.prebuilt_wheel not in kinds: + return next(iter(kinds)) + return packagesettings.DownloadKind.sdist + + @step.command() @click.option( "--source-kind", @@ -91,9 +108,11 @@ def download_source_archive( packagesettings.DownloadKind.git_checkout.value, ] ), - default=packagesettings.DownloadKind.sdist.value, - show_default=True, - help="Kind returned by the source download step.", + default=None, + help=( + "Kind returned by the source download step. Defaults to the provider's " + "only supported source kind, otherwise sdist. Prebuilt wheels are excluded." + ), ) @click.option( "--source-path", @@ -105,7 +124,7 @@ def download_source_archive( @click.pass_obj def prepare_source( wkctx: context.WorkContext, - source_kind: str, + source_kind: str | None, source_path: pathlib.Path | None, dist_name: str, dist_version: Version, @@ -119,7 +138,10 @@ def prepare_source( """ req = Requirement(f"{dist_name}=={dist_version}") token = requirement_ctxvar.set(req) - download_kind = packagesettings.DownloadKind(source_kind) + if source_kind is not None: + download_kind = packagesettings.DownloadKind(source_kind) + else: + download_kind = _default_source_kind(wkctx, req) sdists_downloads = pathlib.Path(wkctx.sdists_repo) / "downloads" checkout = gitutils.git_checkout_dir( work_dir=wkctx.work_dir, distribution_name=req.name, version=dist_version diff --git a/src/fromager/finders.py b/src/fromager/finders.py index 619f87e74..1800c3d12 100644 --- a/src/fromager/finders.py +++ b/src/fromager/finders.py @@ -266,7 +266,8 @@ def find_source_dir( name_based = f"{req.name}-{dist_version}" dotted_name = f"{req.name.replace('-', '.')}-{dist_version}" - candidate_bases = set( + # Keep the order below and drop duplicates. + candidate_bases = dict.fromkeys( [ # First check if the file is there using the canonically # transformed name. @@ -297,6 +298,10 @@ def find_source_dir( # what case they have, but the pattern matched, so # use the base name of the unpack directory to # extend the path 1 level. - return dirname / dirname.name + source_dir = dirname / dirname.name + # Skip a match without a source root, such as a + # wheel-cache directory named with another spelling. + if source_dir.is_dir(): + return source_dir return None diff --git a/src/fromager/gitutils.py b/src/fromager/gitutils.py index 167602b90..e9eabb86e 100644 --- a/src/fromager/gitutils.py +++ b/src/fromager/gitutils.py @@ -244,11 +244,17 @@ def git_checkout_mismatch( ).strip() if pathlib.Path(toplevel).resolve() != checkout_dir.resolve(): return "it is not the top of a Git checkout" - origin = external_commands.run( - ["git", "remote", "get-url", "origin"], cwd=cwd - ).strip() + # Read the checkout's first stored URL without global settings or + # "insteadOf" rewrites. Call subprocess directly because + # external_commands.run() logs output, and URLs can contain credentials. + origin = subprocess.run( + ["git", "config", "--local", "--null", "--get-all", "remote.origin.url"], + cwd=cwd, + check=True, + capture_output=True, + text=True, + ).stdout.split("\0", 1)[0] if origin != repo_url: - # Do not log URLs; they can contain credentials. return "it was cloned from a different repository" head = external_commands.run( ["git", "rev-parse", "--verify", "HEAD"], cwd=cwd diff --git a/tests/conftest.py b/tests/conftest.py index 5550ffcd9..152865ef0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +import os import pathlib import typing import warnings @@ -66,6 +67,13 @@ def pytest_terminal_summary(terminalreporter: pytest.TerminalReporter) -> None: terminalreporter.write_line(f"{nodeid}: {message}") +@pytest.fixture(autouse=True) +def isolate_git_config(monkeypatch: pytest.MonkeyPatch) -> None: + """Ignore the user's and system Git config, such as commit signing.""" + monkeypatch.setenv("GIT_CONFIG_GLOBAL", os.devnull) + monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1") + + @pytest.fixture def testdata_path() -> typing.Generator[pathlib.Path, None, None]: yield TESTDATA_PATH diff --git a/tests/test_commands.py b/tests/test_commands.py index b00f12cf2..bb1113bbd 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1163,6 +1163,94 @@ def test_step_prepare_source_defaults_to_archive_kind( ) +@pytest.mark.parametrize("provider", ["pypi-git", "github-tag-git", "gitlab-tag-git"]) +def test_step_prepare_source_defaults_to_checkout_for_git_provider( + cli_runner: CliRunner, tmp_context: WorkContext, provider: str +) -> None: + """Use the Git checkout when the provider only clones Git repositories.""" + source_options = ( + " clone_url: https://git.test/org/pkg.git\n tag: 'v{version}'\n" + if provider == "pypi-git" + else " project_url: https://git.test/org/pkg\n" + ) + _configure_package( + tmp_context, + f"source:\n provider: {provider}\n" + source_options, + ) + checkout = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" + checkout.mkdir(parents=True) + + with patch.object( + step.sources, "prepare_source", return_value=checkout + ) as prepare_source: + result = cli_runner.invoke( + step.step, ["prepare-source", "pkg", "1.0"], obj=tmp_context + ) + + assert result.exit_code == 0, result.output + prepare_source.assert_called_once_with( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + source_filename=checkout, + version=Version("1.0"), + download_kind=packagesettings.DownloadKind.git_checkout, + ) + + +@pytest.mark.parametrize("provider", ["github-tag-download", "gitlab-tag-download"]) +def test_step_prepare_source_defaults_to_tarball_for_download_provider( + cli_runner: CliRunner, tmp_context: WorkContext, provider: str +) -> None: + """Infer tarball for providers that only download repository archives.""" + _configure_package( + tmp_context, + f"source:\n provider: {provider}\n project_url: https://git.test/org/pkg\n", + ) + archive = tmp_context.sdists_downloads / "pkg-1.0.tar.gz" + archive.write_bytes(b"archive") + source_root = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" + + with patch.object( + step.sources, "prepare_source", return_value=source_root + ) as prepare_source: + result = cli_runner.invoke( + step.step, ["prepare-source", "pkg", "1.0"], obj=tmp_context + ) + + assert result.exit_code == 0, result.output + prepare_source.assert_called_once_with( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + source_filename=archive, + version=Version("1.0"), + download_kind=packagesettings.DownloadKind.tarball, + ) + + +@pytest.mark.parametrize( + "source_yaml", + [ + "source:\n provider: pypi-sdist\n", + "source:\n provider: hook-sdist\n build_sdist: tarball\n", + "source:\n provider: pypi-download\n" + " download_url: https://pkg.test/pkg-{version}.tar.gz\n" + " download_kind: tarball\n", + "source:\n provider: pypi-prebuilt\n", + "source:\n provider: hook-prebuilt\n index_url: https://pkg.test/simple/\n", + ], + ids=["sdist", "hook-source", "mixed-archives", "prebuilt", "hook-prebuilt"], +) +def test_default_source_kind_keeps_archive_fallback( + tmp_context: WorkContext, source_yaml: str +) -> None: + """Keep sdist when no unique source kind can be inferred, including wheels.""" + _configure_package(tmp_context, source_yaml) + + result = step._default_source_kind(tmp_context, Requirement("pkg==1.0")) + + assert result is packagesettings.DownloadKind.sdist + + def test_step_prepare_source_accepts_explicit_hook_path( cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path ) -> None: diff --git a/tests/test_finders.py b/tests/test_finders.py index b7752792d..7dcd991a5 100644 --- a/tests/test_finders.py +++ b/tests/test_finders.py @@ -114,6 +114,35 @@ def test_find_source_dir( assert str(source_dir) == str(actual) +def test_find_source_dir_skips_match_without_source_root( + tmp_path: pathlib.Path, tmp_context: context.WorkContext +) -> None: + """Ignore a matching directory that has no nested source root.""" + work_dir = tmp_path + (work_dir / "test_pkg-1.0").mkdir() + source_dir = work_dir / "test-pkg-1.0" / "test-pkg-1.0" + source_dir.mkdir(parents=True) + + actual = finders.find_source_dir( + tmp_context, work_dir, Requirement("test_pkg"), "1.0" + ) + + assert actual == source_dir + + +def test_find_source_dir_returns_none_without_source_root( + tmp_path: pathlib.Path, tmp_context: context.WorkContext +) -> None: + """Return None when a matching directory has no nested source root.""" + (tmp_path / "test_pkg-1.0").mkdir() + + actual = finders.find_source_dir( + tmp_context, tmp_path, Requirement("test_pkg"), "1.0" + ) + + assert actual is None + + def test_pypi_cache_provider() -> None: url = "https://cache.test/simple/" diff --git a/tests/test_gitutils.py b/tests/test_gitutils.py index 2dac91d5d..a725c5dac 100644 --- a/tests/test_gitutils.py +++ b/tests/test_gitutils.py @@ -1,3 +1,4 @@ +import logging import pathlib import shutil import subprocess @@ -241,6 +242,74 @@ def test_git_checkout_mismatch_reports_reason( assert result == reason +@needs_git_command +def test_git_checkout_mismatch_uses_stored_origin_url( + tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture +) -> None: + """Compare the stored origin URL, not the one rewritten by insteadOf.""" + repo = _make_repo(tmp_path / "repo") + checkout = _clone(repo, tmp_path / "checkout", "refs/tags/v1.0") + repo_url = "https://user:origin-secret@upstream.test/repo" + rewritten_url = "https://user:mirror-secret@mirror.test/repo" + _git(checkout, "remote", "set-url", "origin", repo_url) + _git(checkout, "config", f"url.{rewritten_url}.insteadOf", repo_url) + + with caplog.at_level(logging.DEBUG): + result = git_checkout_mismatch( + checkout_dir=checkout, repo_url=repo_url, ref="refs/tags/v1.0" + ) + + assert "origin-secret" not in caplog.text + assert "mirror-secret" not in caplog.text + assert result is None + + +@needs_git_command +@pytest.mark.parametrize( + ("origin_index", "reason"), + [(0, None), (1, "it was cloned from a different repository")], +) +def test_git_checkout_mismatch_uses_first_origin_url( + tmp_path: pathlib.Path, + origin_index: int, + reason: str | None, +) -> None: + """Compare the first stored URL even with multiple URLs and a rewrite.""" + repo = _make_repo(tmp_path / "repo") + checkout = _clone(repo, tmp_path / "checkout", "refs/tags/v1.0") + origin_urls = [str(repo), "https://user:secret@other.test/repo"] + _git(checkout, "remote", "set-url", "--add", "origin", origin_urls[1]) + _git(checkout, "config", "url.https://mirror.test/repo.insteadOf", origin_urls[0]) + + result = git_checkout_mismatch( + checkout_dir=checkout, + repo_url=origin_urls[origin_index], + ref="refs/tags/v1.0", + ) + + assert result == reason + + +@needs_git_command +def test_git_checkout_mismatch_ignores_global_origin_url( + tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Use the checkout's stored origin even when the global config sets one.""" + repo = _make_repo(tmp_path / "repo") + checkout = _clone(repo, tmp_path / "checkout", "refs/tags/v1.0") + global_config = tmp_path / "gitconfig" + global_config.write_text( + '[remote "origin"]\n url = https://global.test/other-repo\n' + ) + monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(global_config)) + + result = git_checkout_mismatch( + checkout_dir=checkout, repo_url=str(repo), ref="refs/tags/v1.0" + ) + + assert result is None + + @needs_git_command def test_git_checkout_mismatch_rejects_uninitialized_submodule( tmp_path: pathlib.Path, From dbabba41d4207a7763e7e68619654920e9eded5c Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Tue, 3 Nov 2026 21:05:33 -0500 Subject: [PATCH 30/35] fix(package): redact credentials from displayed source URLs Use `_display_url()` to remove user information and fragments while preserving query parameter names and masking every value. Apply it to configured and legacy index, wheel server, and download URLs. Cover duplicate and blank query parameters, credential-bearing output, and the original URLs passed to resolution. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Codex Signed-off-by: Lalatendu Mohanty --- src/fromager/commands/package.py | 33 ++++++++----- tests/test_package.py | 84 +++++++++++++++++++++++++++++--- 2 files changed, 98 insertions(+), 19 deletions(-) diff --git a/src/fromager/commands/package.py b/src/fromager/commands/package.py index 2aebd4b3c..464fd1f5f 100644 --- a/src/fromager/commands/package.py +++ b/src/fromager/commands/package.py @@ -7,7 +7,7 @@ import sys import typing from collections import defaultdict -from urllib.parse import urlsplit +from urllib.parse import parse_qsl, urlencode, urlsplit import click import pypi_simple @@ -32,13 +32,21 @@ logger = logging.getLogger(__name__) -def _redact_url_credentials(url: object) -> str: - """Remove credentials from a URL before displaying it.""" - url_string = str(url) - parsed_url = urlsplit(url_string) - if "@" not in parsed_url.netloc: - return url_string - return parsed_url._replace(netloc=parsed_url.netloc.rsplit("@", 1)[1]).geturl() +def _display_url(url: object) -> str: + """Format a URL for display without exposing credentials. + + Keep query parameter names but mask every value. Drop user info and fragments. + """ + parsed_url = urlsplit(str(url)) + netloc = parsed_url.netloc.rsplit("@", 1)[-1] + query = urlencode( + [ + (name, "***") + for name, _ in parse_qsl(parsed_url.query, keep_blank_values=True) + ], + safe="*", + ) + return parsed_url._replace(netloc=netloc, query=query, fragment="").geturl() def sdist_server_url_option(func: typing.Callable) -> typing.Callable: @@ -490,7 +498,7 @@ def _resolve_fromager( for url_field in ("index_url", "project_url", "clone_url", "download_url"): url = getattr(source_resolver, url_field, None) if url is not None: - click.secho(f" - {url_field}: {_redact_url_credentials(url)}") + click.secho(f" - {url_field}: {_display_url(url)}") if distribution_type is not DistributionType.DEFAULT: click.secho( f"- distribution type '{distribution_type.value}' is ignored " @@ -501,16 +509,17 @@ def _resolve_fromager( include_sdists, include_wheels = parse_distribution_option( distribution_type, pbi ) - click.secho(f"- sdist server url: {override_sdist_server_url}") + click.secho(f"- sdist server url: {_display_url(override_sdist_server_url)}") click.secho(f"- resolver includes sdists: {pbi.resolver_include_sdists}") click.secho(f"- resolver includes wheels: {pbi.resolver_include_wheels}") click.secho( f"- using distribution type '{distribution_type.value}' " f"(sdists: {include_sdists}, wheels: {include_wheels})" ) - click.secho(f"- wheel server url: {pbi.wheel_server_url}") + click.secho(f"- wheel server url: {_display_url(pbi.wheel_server_url)}") click.secho( - f"- download url: {pbi.download_source_url(resolve_template=False)}" + "- download url: " + f"{_display_url(pbi.download_source_url(resolve_template=False))}" ) click.secho(f"- prebuilt wheel: {pbi.pre_built}") if constraint is not None: diff --git a/tests/test_package.py b/tests/test_package.py index 8c25f25c8..106b27889 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -17,6 +17,43 @@ def _configure_package(ctx: context.WorkContext, yaml_str: str) -> None: ctx.settings._package_settings[ps.name] = ps +@pytest.mark.parametrize( + ("url", "expected"), + [ + ("https://pkg.test/simple/", "https://pkg.test/simple/"), + ("https://user:pass@pkg.test/simple/", "https://pkg.test/simple/"), + ( + "https://pkg.test/archive?private_token=SECRET", + "https://pkg.test/archive?private_token=***", + ), + ("https://pkg.test/archive#token=SECRET", "https://pkg.test/archive"), + ( + "https://user:pass@pkg.test/archive/v{version}.tar.gz?token=SECRET", + "https://pkg.test/archive/v{version}.tar.gz?token=***", + ), + ( + "https://user:pass@pkg.test/archive?token=SECRET#token=SECRET", + "https://pkg.test/archive?token=***", + ), + ( + "https://pkg.test/archive.tar.gz?sha=v{version}&private_token=SECRET", + "https://pkg.test/archive.tar.gz?sha=***&private_token=***", + ), + ( + "https://pkg.test/archive?ref=one&ref=two&flag&empty=", + "https://pkg.test/archive?ref=***&ref=***&flag=***&empty=***", + ), + ( + "https://pkg.test/archive?private%5Ftoken=SECRET%26OTHER", + "https://pkg.test/archive?private_token=***", + ), + ], +) +def test_display_url(url: str, expected: str) -> None: + """Keep query names while hiding user info, values, and fragments.""" + assert package_command._display_url(url) == expected + + def test_resolve_fromager_displays_and_uses_source_config( tmp_context: context.WorkContext, capsys: pytest.CaptureFixture[str], @@ -26,7 +63,7 @@ def test_resolve_fromager_displays_and_uses_source_config( tmp_context, "source:\n" " provider: pypi-prebuilt\n" - " index_url: https://index-user:index-password@pkg.test/simple/\n" + " index_url: https://index-user:index-password@pkg.test/simple/?token=SECRET#fragment-secret\n" " min_release_age: 7\n", ) req = Requirement("pkg") @@ -62,9 +99,11 @@ def test_resolve_fromager_displays_and_uses_source_config( assert " - provider: pypi-prebuilt" in output assert " - resolves_prebuilt_wheel: True" in output assert " - min_release_age: 7" in output - assert " - index_url: https://pkg.test/simple/" in output + assert " - index_url: https://pkg.test/simple/?token=***\n" in output assert "index-user" not in output assert "index-password" not in output + assert "SECRET" not in output + assert "fragment-secret" not in output assert ( "- distribution type 'sdist' is ignored for configured source resolvers" in output @@ -82,22 +121,38 @@ def test_resolve_fromager_displays_and_uses_source_config( assert source_provider.cooldown is None +@pytest.mark.parametrize( + ("credentials", "suffix"), + [ + ("", ""), + ( + "SECRET_USER:SECRET_PASSWORD@", + "?private_token=QUERY_SECRET#token=FRAGMENT_SECRET", + ), + ], + ids=["public", "credentials"], +) def test_resolve_fromager_preserves_legacy_config_output( tmp_context: context.WorkContext, capsys: pytest.CaptureFixture[str], + credentials: str, + suffix: str, ) -> None: - """Keep legacy resolver and download settings visible.""" + """Display legacy URLs safely while resolving with their original values.""" + index_url = f"https://{credentials}legacy.test/simple/{suffix}" + wheel_url = f"https://{credentials}wheels.test/simple/{suffix}" + download_url = f"https://{credentials}pkg.test/pkg-${{version}}.tar.gz{suffix}" _configure_package( tmp_context, "download_source:\n" - " url: https://pkg.test/pkg-${version}.tar.gz\n" + f" url: {download_url}\n" "resolver_dist:\n" - " sdist_server_url: https://legacy.test/simple/\n" + f" sdist_server_url: {index_url}\n" " include_sdists: true\n" " include_wheels: false\n" "variants:\n" " cpu:\n" - " wheel_server_url: https://wheels.test/simple/\n" + f" wheel_server_url: {wheel_url}\n" " pre_built: false\n", ) req = Requirement("pkg") @@ -132,6 +187,21 @@ def test_resolve_fromager_preserves_legacy_config_output( assert "- wheel server url: https://wheels.test/simple/" in output assert "- download url: https://pkg.test/pkg-${version}.tar.gz" in output assert "- prebuilt wheel: False" in output + if suffix: + assert ( + "- sdist server url: https://legacy.test/simple/?private_token=***\n" + in output + ) + assert ( + "- wheel server url: https://wheels.test/simple/?private_token=***\n" + in output + ) + assert ( + "- download url: https://pkg.test/pkg-${version}.tar.gz?private_token=***\n" + in output + ) + for secret in ("SECRET_USER", "SECRET_PASSWORD", "QUERY_SECRET", "FRAGMENT_SECRET"): + assert secret not in output assert versions == {Version("1.0")} assert provider.cooldown is None find_and_invoke.assert_called_once_with( @@ -142,6 +212,6 @@ def test_resolve_fromager_preserves_legacy_config_output( req=req, include_sdists=False, include_wheels=True, - sdist_server_url="https://legacy.test/simple/", + sdist_server_url=index_url, ignore_platform=False, ) From 824225c6580ea5710038cc1210b346e206326563 Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Tue, 3 Nov 2026 16:30:56 -0500 Subject: [PATCH 31/35] fix(resolver): restore the empty resolution warning #1393 dropped the "resolver returned no results" warning when it restructured source resolution. Restore it and add a test. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Lalatendu Mohanty --- .../bootstrap_requirement_resolver.py | 5 +++ tests/test_bootstrap_requirement_resolver.py | 32 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/src/fromager/bootstrap_requirement_resolver.py b/src/fromager/bootstrap_requirement_resolver.py index 6e8407627..682123ca2 100644 --- a/src/fromager/bootstrap_requirement_resolver.py +++ b/src/fromager/bootstrap_requirement_resolver.py @@ -228,6 +228,11 @@ def _resolve_and_extend( max_age_cutoff=max_age_cutoff, age_fallback=age_fallback, ) + if not results: + logger.warning( + "resolver returned no results (%s version mode)", + "multiple" if self.multiple_versions else "single", + ) if results: key = (canonicalize_name(req.name), pre_built) diff --git a/tests/test_bootstrap_requirement_resolver.py b/tests/test_bootstrap_requirement_resolver.py index a1bc00251..c8b7fb9a6 100644 --- a/tests/test_bootstrap_requirement_resolver.py +++ b/tests/test_bootstrap_requirement_resolver.py @@ -915,3 +915,35 @@ def test_single_version_passes_all_age_fallback( mock_find.assert_called_once() call_kwargs = mock_find.call_args.kwargs assert call_kwargs["age_fallback"] == resolver.AgeFallback.ALL + + +@pytest.mark.parametrize( + ("multiple_versions", "mode"), [(False, "single"), (True, "multiple")] +) +def test_resolve_warns_when_resolver_returns_no_results( + tmp_context: WorkContext, + caplog: pytest.LogCaptureFixture, + multiple_versions: bool, + mode: str, +) -> None: + """Log the version mode when source resolution finds nothing.""" + brr = BootstrapRequirementResolver(tmp_context, multiple_versions=multiple_versions) + + with ( + patch.object(brr, "_resolve_from_graph", return_value=None), + patch("fromager.bootstrap_requirement_resolver.sources.get_source_provider"), + patch( + "fromager.bootstrap_requirement_resolver.resolver" + ".find_all_matching_from_provider", + return_value=[], + ), + ): + results = brr.resolve( + Requirement("testpkg"), + RequirementType.INSTALL, + parent_req=None, + pre_built=False, + ) + + assert results == [] + assert f"resolver returned no results ({mode} version mode)" in caplog.text From 70a0a7fa59cdd391db62e2bfd9e52a61f8e0b00a Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Tue, 3 Nov 2026 17:29:06 -0500 Subject: [PATCH 32/35] docs(resolver): correct source reuse and replay descriptions Describe clean Git checkout reuse and provider validation for every build-order entry. Use the exported get_resolver_provider hook name consistently. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Lalatendu Mohanty --- docs/concepts/resolver-architecture.rst | 6 +++--- docs/proposals/new-resolver-config.md | 12 ++++++------ docs/reference/files.md | 7 ++++--- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index 58a17506f..44825fd66 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -153,9 +153,9 @@ The default Git preparer rejects a checkout with local changes before applying patches, project overrides, and Rust vendoring. Git status does not prove that the checkout is the original clone or preserve it after preparation. Build metadata is written after preparation but is not used to choose the preparation path. -With ``--no-cleanup``, downloading to an existing checkout fails and preserves -its files for inspection. Enable cleanup or use a new work directory to -download and prepare a fresh checkout. Git status cannot detect changes to +An existing checkout that cannot be reused, as described below, is cloned +again with cleanup enabled. With ``--no-cleanup``, the download fails and +keeps its files for inspection. Git status cannot detect changes to patch or settings files outside the checkout. If preparation left the checkout clean, a manual second ``prepare-source`` call runs the preparation steps again. The separate ``step prepare-source`` invocation accepts ``--source-kind`` and diff --git a/docs/proposals/new-resolver-config.md b/docs/proposals/new-resolver-config.md index 880fe54c3..3895f8082 100644 --- a/docs/proposals/new-resolver-config.md +++ b/docs/proposals/new-resolver-config.md @@ -143,12 +143,12 @@ variants: `gitlab-tag-git` and `github-tag-git` profiles. Instead of cloning a git repository, they download a git tarball or an release artifact. -- The `hook-sdist` profile calls the `resolver_provider` and `download_source` - [hooks](../reference/hooks.rst). The `resolver_provider` hook receives the +- The `hook-sdist` profile calls the `get_resolver_provider` and `download_source` + [hooks](../reference/hooks.rst). The `get_resolver_provider` hook receives the `req` argument for access to the current requirement and `sdist_server_url=str(index_url)`. `index_url` defaults to PyPI. Hooks can return any source artifact (sdist, tarball, or git checkout). Without a - `resolver_provider` hook, Fromager finds versions on `index_url` in sdists + `get_resolver_provider` hook, Fromager finds versions on `index_url` in sdists and wheels for any platform, and the `download_source` hook gets the source. A package needs at least one of the two hooks. @@ -156,7 +156,7 @@ variants: artifact must be a pre-built wheel. Its required `index_url` replaces the legacy variant `wheel_server_url`; the hook receives it as `sdist_server_url` and may use, ignore, or replace it. Without a - `resolver_provider` hook, Fromager finds versions on `index_url` in + `get_resolver_provider` hook, Fromager finds versions on `index_url` in wheels for this platform. - The `not-available` profile raises an error. It can be used to block a @@ -165,11 +165,11 @@ variants: ### default behavior and hooks When a package setting file does not have a top-level `source` configuration, -then Fromager keep its old behavior. It first looks for `resolver_provider` +then Fromager keep its old behavior. It first looks for `get_resolver_provider` and `download_source` [hooks](../reference/hooks.rst), then looks for source distributions on PyPI. -When a package has a plugin with a `resolver_provider` or `download_source` +When a package has a plugin with a `get_resolver_provider` or `download_source` hook and `source` settings, then at least one `source` setting (top-level or variant) must use `provider: hook-sdist` or `provider: hook-prebuilt`. The rule ensures that the hooks are used. diff --git a/docs/reference/files.md b/docs/reference/files.md index a70026cbc..0485d2d3b 100644 --- a/docs/reference/files.md +++ b/docs/reference/files.md @@ -85,14 +85,15 @@ For example, a configured Git source is recorded as: ``` `download-sequence` calls the configured resolver's normal `download()` for -these entries. Before downloading, it checks the recorded provider name for -each source entry and each `prebuilt: true` wheel requested with `--include-wheels`. +these entries. Before downloading anything, it checks the recorded provider +name of every entry, including prebuilt entries it will skip. A missing or different provider is an error. Older build orders without `source_provider` still work with legacy settings; regenerate them before replaying a configured source. The check compares only provider names, so replay should still use the same package settings. Archive downloads can be reused from the download cache. Git downloads create a -checkout in the work directory and are not cached for a later build. The command +checkout in the work directory. A later build in the same work directory +reuses it if it is still a clean checkout of the same repository and ref. The command also uses `source_url_type` to skip entries that used a prebuilt fallback at runtime, even if source settings are configured. From 511b050c3bb13d3d107ba0eeae39b72d93525bd3 Mon Sep 17 00:00:00 2001 From: Lalatendu Mohanty Date: Tue, 3 Nov 2026 17:18:23 -0500 Subject: [PATCH 33/35] docs(resolver): make architecture guide easier to read Add a requirement-to-build overview and separate provider selection, source preparation, Git checkout reuse, and build-order replay into focused sections. Remove repeated explanations, clarify session-cache behavior, and tighten the wording throughout. Co-Authored-By: Codex Co-Authored-By: Claude Code Signed-off-by: Lalatendu Mohanty --- docs/concepts/resolver-architecture.rst | 464 +++++++++++++----------- 1 file changed, 258 insertions(+), 206 deletions(-) diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index 44825fd66..1d5923320 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -1,226 +1,274 @@ Resolver Architecture ===================== -Fromager resolves package versions from multiple sources using a -provider abstraction. This document describes the resolution -strategies, filtering, and key design decisions. +Fromager finds package versions on a Python index, in Git tags, or +through a plugin, and turns the selected source into something it can +build. .. seealso:: :doc:`architecture-overview` maps the major subsystems. + :doc:`bootstrapper-architecture` covers how resolution fits into the bootstrap pipeline. +From Requirement to Build +------------------------- + +A **provider** finds versions. A **source resolver** selects the +provider, downloads the artifact, and selects the sdist build method. +Prebuilt profiles download a wheel and skip source preparation and +building. + +For a source package, the work has four steps: + +1. **Resolve:** a provider finds matching versions and their download URLs. +2. **Download:** the source resolver returns a ``DownloadedSource`` with + a local path and an artifact kind. +3. **Prepare:** Fromager unpacks an archive or prepares a Git checkout, + then applies patches and other source changes. +4. **Build:** Fromager creates an sdist with the selected build method, + then builds the wheel. + Resolution Strategies --------------------- -During bootstrap, the resolver coordinator tries these sources in -priority order and accumulates results in a thread-safe session cache: +During bootstrap, ``BootstrapRequirementResolver`` keeps a thread-safe +session cache of resolved requirement rules and discovered versions. +For each requirement rule, it tries in order: -.. code-block:: text +1. the session cache, so repeated rules need no further lookup +2. the previous dependency graph, if one is available +3. the selected provider, which may query PyPI, GitHub, or GitLab + +New results are merged into the session cache and filtered by the +current requirement. - ┌───────────────────────────────────────────────────────┐ - │ Resolution Priority Order │ - │ │ - │ 1. Session cache (versions found earlier) │ - │ │ miss │ - │ ▼ │ - │ 2. Previous dependency graph (prior run) │ - │ │ miss │ - │ ▼ │ - │ 3. Network (PyPI / GitHub / GitLab) │ - │ (per-package configurable) │ - └───────────────────────────────────────────────────────┘ - - Git URL requirements (top-level only) bypass this - chain entirely: the repo is cloned and the version - is extracted from package metadata. - -Versions discovered from any source are merged into the session cache. -Subsequent requests for the same package skip the network entirely if -the cache already contains a matching version. - -The configured wheel cache server is not a version-resolution fallback. -After a version has been selected, the bootstrapper checks the cache -server separately for a previously built wheel. -For prebuilt packages, the local wheel server can supply a cached wheel -before a fresh resolution through the selected source provider. +Two cases need separate handling: + +- **Direct-reference URL requirements**, including Git URLs, are rejected. + Use a ``source:`` provider in package settings instead. +- **Prebuilt packages** may use the local wheel server before an + external provider. For source builds the wheel cache is checked only + after a version is selected; it never supplies fallback source + versions. Provider Hierarchy ------------------ -All resolution sources inherit from ``BaseProvider``, which implements -the resolvelib ``ExtrasProvider`` interface. The bootstrap engine and -CLI commands interact with providers through a common -``find_matches()`` method. +Providers inherit from ``BaseProvider``, which implements resolvelib's +``ExtrasProvider`` interface. The bootstrap engine and CLI commands call +their common ``find_matches()`` method. .. code-block:: text BaseProvider - ├── PyPIProvider (PEP 503 Simple API) - │ └── PyPICacheProvider (fromager's own wheel server) - ├── GenericProvider (callback-based, parses tags) + ├── PyPIProvider + │ └── PyPICacheProvider + ├── GenericProvider │ ├── GitHubTagProvider │ └── GitLabTagProvider - └── VersionMapProvider (pre-resolved version→URL map) - -.. list-table:: - :header-rows: 1 - :widths: 25 75 - - * - Provider - - What it queries - * - ``PyPIProvider`` - - Any PEP 503 Simple API index (pypi.org, custom mirrors). - Filters by platform tags, Python version, and yanked status. - * - ``PyPICacheProvider`` - - Subclass of ``PyPIProvider`` pointing at fromager's own wheel - server. No cooldown applied. Used by the bootstrapper's - previously-built-wheel lookup, not by source version resolution. - * - ``GenericProvider`` - - Callback-based provider that pairs a version source function - with a configurable match function (plain parse or regex). - Base class for the tag providers below. - * - ``GitHubTagProvider`` - - GitHub REST API for repository tags. Versions are parsed - from tag names using a configurable pattern. - * - ``GitLabTagProvider`` - - GitLab REST API for repository tags. Includes upload - timestamps for age filtering. - * - ``VersionMapProvider`` - - Wraps a pre-built ``VersionMap`` (version → URL mapping). - Used when versions are already known, e.g. from a prior - resolution or a settings-provided URL template. + └── VersionMapProvider + +- ``PyPIProvider`` queries a Python package index such as PyPI or a + mirror, filtering by platform, Python version, and yanked status. +- ``PyPICacheProvider`` queries Fromager's wheel server without a + cooldown. It serves wheel-cache lookups, not source version resolution. +- ``GenericProvider`` pairs a version-source callback with a + configurable matcher; it is the base of the tag providers. +- ``GitHubTagProvider`` and ``GitLabTagProvider`` query repository tags + and parse versions from tag names. GitLab also reports timestamps for + age filtering. +- ``VersionMapProvider`` uses an existing version-to-URL mapping, such + as results from an earlier resolution or a URL template. + +Selecting a Provider +-------------------- .. versionchanged:: 0.103.0 - Packages with ``source:`` configuration now select their provider through - the configured source resolver. - The ``hook-sdist`` and ``hook-prebuilt`` profiles create providers through - the package's ``get_resolver_provider`` hook when it exists. Their - ``download()`` methods invoke the package's ``download_source`` hook when - present and return a validated ``DownloadedSource``. - Configured source resolvers own their index URL and ignore caller URLs. - Hook profiles pass their ``index_url`` to the legacy - ``get_resolver_provider`` hook as ``sdist_server_url``. ``hook-sdist`` - defaults to PyPI, while ``hook-prebuilt`` requires an explicit index. - A hook profile without a ``get_resolver_provider`` hook finds versions on - its ``index_url`` and uses the ``download_source`` hook. - - ``find-updates`` uses the configured source resolver when a package has - ``source:`` settings. The source profile determines distribution types and - the index URL. Packages without ``source:`` settings retain the legacy - provider hook, distribution options, and index URL overrides. - Update reports ignore global and per-package release-age cooldowns so - recent releases remain visible with either configuration style. - -Per-package settings in YAML can select which provider to use and -configure its parameters (index URL, tag pattern, etc.). When a package -has a ``source:`` resolver configured, that resolver creates the provider. -The ``hook-sdist`` and ``hook-prebuilt`` profiles delegate provider creation -to the package's ``get_resolver_provider`` hook when it exists and validate -that it returns a :class:`~fromager.resolver.BaseProvider`. -The hook may ignore or replace the profile's index URL, but the profile's -value is stable across Fromager commands. -Without that hook, the profiles find versions on ``index_url``: -``hook-sdist`` looks at sdists and wheels for any platform, like -``pypi-git``, and ``hook-prebuilt`` looks at wheels for this platform, like -``pypi-prebuilt``. The package's ``download_source`` hook then gets the -artifact. A package with neither hook is an error. -Otherwise, override plugins can replace the provider for a specific package -via the ``get_resolver_provider`` hook, after which the legacy resolver -settings are used. + Packages can select a source resolver through ``source:`` settings, + which replaces the legacy resolver settings for those packages. + +A package's YAML settings select the provider and its parameters, such +as the index URL or tag pattern. With ``source:`` configured, the source +resolver creates the provider and owns the index URL; caller-supplied +index URLs are ignored. Without ``source:``, Fromager uses the legacy +resolver settings and the package's ``get_resolver_provider`` override +when present. + +Hook Profiles +~~~~~~~~~~~~~ + +The ``hook-sdist`` and ``hook-prebuilt`` profiles connect ``source:`` +settings to existing plugins. A package must provide at least one of +``get_resolver_provider`` or ``download_source``. + +- **With a resolver hook:** the hook creates the provider and must + return a :class:`~fromager.resolver.BaseProvider`. It receives + ``index_url`` as ``sdist_server_url`` and may ignore or replace it. +- **Without a resolver hook:** Fromager finds versions on ``index_url``. + ``hook-sdist`` considers sdists and wheels for any platform; + ``hook-prebuilt`` considers wheels for the current platform. The + download hook retrieves the artifact. + +``hook-sdist`` defaults to PyPI; ``hook-prebuilt`` requires an explicit +index URL. The profile passes the same configured URL to the hook across +Fromager commands. + +Update Reports +~~~~~~~~~~~~~~ + +``find-updates`` also uses the configured source resolver, including its +distribution types and index URL. Packages without ``source:`` retain +the legacy provider hook, distribution options, and index overrides. +Update reports ignore global and per-package cooldowns so recent +releases remain visible. + +Downloading and Preparing Sources +--------------------------------- + +Passing Resolution Results to Downloads +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ A configured resolver's ``download()`` receives only the version and the -download URL of the selected candidate. Resolution and download can run in -different commands, and these values are saved in ``graph.json`` and -``build-order.json``. The URL must therefore contain everything the download -needs, such as the git ref in ``git+https://host/repo.git@refs/tags/v1.0``. +download URL of the selected candidate. These values are saved in +``graph.json`` and ``build-order.json`` so a later command can download +the same source. The URL must contain everything the download needs, +including the Git ref: + +.. code-block:: text + + git+https://host.example/repo.git@refs/tags/v1.0 .. versionchanged:: 0.103.0 ``download()`` takes ``version`` and ``download_url`` instead of a ``Candidate``. -Hook downloads accept explicit artifact kinds and adapt legacy paths using -profile-specific defaults. If no download hook exists, the profile selects the -standard sdist or wheel downloader and its destination directory. Hook errors -propagate with context instead of triggering a fallback. In-process callers -pass the downloaded path and kind to preparation: ``sdist`` and ``tarball`` -are unpacked, while ``git_checkout`` uses the checkout directory. The checkout -uses the same double-nested work directory pattern as an unpacked archive; -its directory names use the canonical package name, even when the requirement -uses another spelling. -The default Git preparer rejects a checkout with local changes before applying -patches, project overrides, and Rust vendoring. Git status does not prove that the -checkout is the original clone or preserve it after preparation. Build metadata -is written after preparation but is not used to choose the preparation path. -An existing checkout that cannot be reused, as described below, is cloned -again with cleanup enabled. With ``--no-cleanup``, the download fails and -keeps its files for inspection. Git status cannot detect changes to -patch or settings files outside the checkout. If preparation left the checkout -clean, a manual second ``prepare-source`` call runs the preparation steps again. -The separate ``step prepare-source`` invocation accepts ``--source-kind`` and -``--source-path`` because it cannot retrieve the earlier Python download result. -``step download-source-archive --output-format json`` exposes that result's -actual path and kind without changing its default path-only output. -Without ``--source-kind``, it uses the provider's only supported source kind: -``sdist``, ``tarball``, or ``git_checkout``. Legacy settings and providers with -multiple kinds keep the ``sdist`` default. Prebuilt wheels are not source kinds -and are excluded from inference. -``download-sequence`` calls the configured resolver's normal ``download()`` -for source entries. Both sequence commands check each entry's recorded -``source_provider`` against current settings, including prebuilt entries whose -URLs ``download-sequence`` will skip. A missing or different provider fails -before any downloads or builds start. Older build orders without -``source_provider`` still work with legacy settings; regenerate them before -replaying a configured source. -These checks compare provider names, so replay should still use the same -package settings. ``download-sequence`` still skips downloading a runtime -``prebuilt`` entry even when the package has source settings. -Built-in archive resolvers cache the resolved file for a later build. Git -checkout resolvers clone into the work directory instead. A later build in the -same work directory reuses that checkout if it is still a clean clone of the -same repository at the same commit, so it can run without access to the Git -host. It only checks local state, so a tag moved on the remote is not noticed. -Any other existing checkout is removed and cloned again with cleanup enabled, -and the download fails without cleanup. Hook -download behavior depends on the hook. Legacy packages keep the -``source_url_type`` selection behavior. Build-order entries record the -effective configured ``source_provider`` for provenance; the -``source_url_type`` field remains a legacy compatibility classification, not -the actual downloaded artifact kind. +Artifact Kinds +~~~~~~~~~~~~~~ + +``DownloadedSource`` tells preparation how to handle the local path: + +- ``sdist`` and ``tarball``: unpack the archive. +- ``git_checkout``: prepare the existing checkout in place. +- ``prebuilt_wheel``: use the wheel without source preparation. + +Hook downloads can return an explicit kind or a legacy ``Path``, treated +as an sdist for ``hook-sdist`` and a wheel for ``hook-prebuilt``. +Without a download hook, the profile uses the standard sdist or wheel +downloader. Hook errors stop the operation with context; they do not +trigger a fallback. + +See :ref:`download_source_hook` for the return contract and validation +rules. + +Git Checkouts: Reuse and Preparation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Built-in archive resolvers cache downloaded files. Git resolvers keep +their checkouts in the work directory, under canonical package names: + +.. code-block:: text + + work-dir/test-pkg-1.0/test-pkg-1.0/ + +At download time, an existing checkout is reused when it is a clean +clone of the same repository at the same commit, with submodules at +their recorded commits. A later build in the same work directory then +runs without contacting the Git host. The check is local only, so a tag +moved on the remote is not detected. + +A checkout that cannot be reused is removed and cloned again. With +``--no-cleanup``, the download fails instead and keeps the files for +inspection. + +At preparation time, the default Git preparer rejects local changes +before applying patches, project overrides, and Rust vendoring. A clean +Git status does not prove the checkout's origin, protect it from later +changes, or detect changed patch and settings files outside the +checkout. Build metadata is written after preparation and does not +select the preparation path. If preparation leaves the checkout clean, +a manual second ``prepare-source`` call runs preparation again. + +Running Download and Preparation Separately +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Separate CLI invocations cannot share the Python ``DownloadedSource`` +result. These options pass its information between steps: + +- ``step download-source-archive --output-format json`` prints the path + and kind; the default output is the path alone. +- ``step prepare-source --source-kind`` selects the artifact kind. + Without it, providers with one supported source kind use that kind. + Legacy settings and providers with multiple kinds default to ``sdist``; + prebuilt wheels are excluded from inference. +- ``step prepare-source --source-path`` selects the downloaded path. + Git checkouts must still use the standard double-nested work directory. + +Building the Sdist +------------------ .. versionchanged:: 0.103.0 - Sequence commands check recorded providers before replaying a build order. - -Configured prebuilt resolvers select the prebuilt bootstrap path. Fresh -external wheels use their ``download()`` contract and are staged in the -prebuilt wheel directory. For local mirror URLs, bootstrap, build, and -download-sequence reuse the exact file the mirror serves, so they do not -rerun the configured download hook or pick a different same-named copy. -Test-mode fallback downloads directly. -When a fresh configured wheel shares a filename with an older cached wheel, -the local mirror and later build replay use the fresh staged wheel. -Legacy build downloads still pass through the build wheel directory before -the mirror moves them to downloads. -See :ref:`download_source_hook` for the return contract and compatibility rules. + Git checkouts can be prepared directly. Configured source resolvers + also select the sdist build method. + +For source builds, the configured resolver selects one of two methods: + +- ``pep517`` calls the package's build backend. Git checkout resolvers + default to it. +- ``tarball`` repacks the prepared source without VCS metadata such as + ``.git``. PyPI sdist resolvers default to it. + +``hook-sdist`` requires an explicit ``build_sdist: pep517`` or +``build_sdist: tarball`` setting. A configured method takes precedence +over a ``build_sdist`` override and logs a warning when the override is +ignored; packages without ``source:`` keep using the override. + +Replaying a Build Order +----------------------- .. versionchanged:: 0.103.0 - Explicit ``DownloadKind.git_checkout`` results can be prepared without - unpacking an archive. Configured source resolvers also select the sdist - build method. Git checkout resolvers use PEP 517 by default; hook sdist - resolvers require an explicit ``build_sdist: pep517`` or - ``build_sdist: tarball``. PyPI sdist resolvers - repack a tarball. A configured ``tarball`` build leaves out VCS metadata - such as ``.git``. Both configured strategies take precedence over an installed - ``build_sdist`` override and log a warning when it is ignored. Legacy - settings continue to use the override. Configured prebuilt profiles use - their source provider after a local cache check, and local mirror URLs - bypass configured download hooks during replay. + Sequence commands check recorded providers before replaying a build + order. + +``build-sequence`` and ``download-sequence`` validate every entry's +recorded ``source_provider`` against current settings before starting +any builds or downloads, including prebuilt entries whose URLs will be +skipped. A missing or different provider is an error. The comparison +uses provider names only, so replay should use the same package +settings. Older build orders without ``source_provider`` still work +with legacy settings; regenerate them before using a configured source. + +``download-sequence`` calls the configured resolver's normal +``download()`` for source entries and skips source downloads for entries +recorded as runtime prebuilt fallbacks, even if source settings are +configured. Legacy packages retain their ``source_url_type`` selection +behavior. The two recorded fields have different purposes: + +- ``source_provider`` identifies the effective configured provider. +- ``source_url_type`` is a legacy classification, not the downloaded + artifact kind. + +Prebuilt Wheel Downloads +------------------------ + +Configured prebuilt profiles skip source builds. Fresh external wheels +use the resolver's ``download()`` and are staged in the prebuilt wheel +directory. Test-mode fallback downloads directly. + +For local mirror URLs, bootstrap, build, and download-sequence reuse the +exact file served by the mirror: they do not rerun a configured download +hook or select a different copy with the same filename. When a fresh +configured wheel replaces an older cached wheel with the same filename, +the mirror and later replay use the fresh staged wheel. + +Legacy build downloads pass through the build wheel directory before the +mirror moves them to downloads. Version Filtering Window -------------------------- +------------------------ -Two age-based filters can narrow the set of acceptable versions: +Two age filters can narrow the set of matching versions: .. code-block:: text @@ -230,42 +278,46 @@ Two age-based filters can narrow the set of acceptable versions: max_age cutoff cooldown cutoff (reject before this) (reject after this) -- **Cooldown** rejects releases that are too new. It ensures a - minimum time has passed since publication, guarding against - problematic releases. Configurable per package; bypassed for - exact version pins (``==``). - -- **Max age** rejects releases that are too old. Used primarily in +- **Cooldown** rejects releases that are too new. It sets a minimum time + since publication and can be configured per package. Exact top-level + version pins (``==``) bypass it; pinned dependencies still use cooldown. +- **Max age** rejects releases that are too old. Used primarily in ``multiple_versions`` mode to limit the range of versions built. -When both are active, only versions published within the window are -considered. If all source candidates are filtered out, the behavior -depends on the mode: in single-version mode a warning is logged and all -candidates are kept; in ``multiple_versions`` mode only the newest -candidate is retained. A source provider with no matching candidates -raises a resolution error; it is not replaced by a cache-server lookup. +Providers apply cooldown before the max-age filter. If cooldown removes +every matching candidate, resolution fails; the max-age fallback does not +bypass cooldown. + +If the max-age filter excludes every remaining source candidate: + +- In single-version mode, Fromager logs a warning and keeps all matching + candidates for version selection. +- In ``multiple_versions`` mode, it keeps only the newest matching + candidate. + +If the source provider has no matching candidates at all, resolution +fails; Fromager does not look for fallback source versions on the wheel +cache server. Flat Resolution by Design ------------------------- -Fromager deliberately does **not** use resolvelib's transitive -dependency resolution. Every provider's ``get_dependencies()`` method -returns an empty list. This means resolution is a simple "find the -best matching version" operation, not a full constraint solver with -backtracking. +Each resolver call selects matching versions for one requirement. +Every provider's ``get_dependencies()`` returns an empty list, so +resolvelib does not solve transitive dependencies or backtrack across +them. Transitive dependencies are handled by the bootstrap engine's own DFS loop: after a package is built, its install dependencies are extracted -from the wheel metadata and pushed onto the work stack as new -``RESOLVE`` phase items. This separation keeps the resolver stateless -and makes each resolution call independent and thread-safe. +from the wheel metadata and pushed onto the work stack as new ``RESOLVE`` +phase items. Providers handle version lookup; the bootstrap engine +handles graph traversal. Per-Package Configuration ------------------------- -Package settings files (YAML) can configure the resolver for each -package independently: resolver type (PyPI, GitHub tags, GitLab tags, -hook-based, or blocked), index URL, cooldown, tag pattern, and -override plugins. +Package settings files (YAML) configure the resolver for each package +independently: resolver type (PyPI, GitHub tags, GitLab tags, hook-based, +or blocked), index URL, cooldown, tag pattern, and override plugins. See :doc:`package-settings` for how settings are loaded and merged. From f3ae3a3557db1dedbb7bc642f3c14ea1fd2e70ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Prpi=C4=8D?= Date: Wed, 4 Nov 2026 14:02:38 -0500 Subject: [PATCH 34/35] fix(pkgmetadata): accept LicenseRef anywhere in SPDX expressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_parse_spdx()` skipped validation only when the whole expression started with `LicenseRef-`. A valid PEP 639 expression such as `MIT OR LicenseRef-Proprietary` was rejected, while `LicenseRef-Foo OR Apache 2` and malformed ids like `LicenseRef-a_b` were accepted. It now parses without validation and rejects every unknown key that is not a well-formed `LicenseRef-` id, raising the same `ExpressionError` as before. This is needed to populate SBOM license fields (#1378). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Martin Prpič --- src/fromager/pkgmetadata/pep639.py | 25 ++++++++++++++++++++----- tests/test_pkgmetadata.py | 10 +++++++++- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/fromager/pkgmetadata/pep639.py b/src/fromager/pkgmetadata/pep639.py index d01192094..cf5b526ad 100644 --- a/src/fromager/pkgmetadata/pep639.py +++ b/src/fromager/pkgmetadata/pep639.py @@ -6,7 +6,9 @@ - https://spdx.org/licenses/ """ -from license_expression import LicenseExpression, get_spdx_licensing +import re + +from license_expression import ExpressionError, LicenseExpression, get_spdx_licensing from packaging.metadata import Metadata @@ -60,12 +62,25 @@ def license_from_metadata_values( _SPDX = get_spdx_licensing() +# SPDX idstring: letters, digits, "." and "-" +_LICENSE_REF_RE = re.compile(r"LicenseRef-[A-Za-z0-9.\-]+") + def _parse_spdx(text: str, *, simplify: bool = False) -> LicenseExpression: - """Parse, validate, and simplify a SPDX license expression""" - # LicenseRef are references to non-SPDX licenses - validate = not text.startswith("LicenseRef-") - expr = _SPDX.parse(text, validate=validate) + """Parse, validate, and simplify a SPDX license expression + + ``LicenseRef-`` keys reference non-SPDX licenses (PEP 639) and may + appear anywhere in the expression, so they are exempt from the SPDX + license list check. Every other unknown key is an error. + """ + expr = _SPDX.parse(text, validate=False) + unknown = [ + key + for key in _SPDX.unknown_license_keys(expr) + if not _LICENSE_REF_RE.fullmatch(key) + ] + if unknown: + raise ExpressionError(f"Unknown license key(s): {', '.join(unknown)}") if simplify: expr = expr.simplify() return expr diff --git a/tests/test_pkgmetadata.py b/tests/test_pkgmetadata.py index 1bb408500..62b8beadb 100644 --- a/tests/test_pkgmetadata.py +++ b/tests/test_pkgmetadata.py @@ -1,4 +1,5 @@ import pytest +from license_expression import ExpressionError from packaging.metadata import Metadata, RawMetadata from fromager.pkgmetadata import ( @@ -82,6 +83,9 @@ def test_project_urls_from_metadata( ["metadata", "expected"], [ [md("MIT AND MIT or MIT"), "MIT"], + # LicenseRef is valid anywhere in a PEP 639 expression + [md("MIT OR LicenseRef-Proprietary"), "LicenseRef-Proprietary OR MIT"], + [md("LicenseRef-Proprietary"), "LicenseRef-Proprietary"], [md(license="MIT License"), "MIT"], [md(license="MIT and PSF-2.0"), "MIT AND PSF-2.0"], [md(license="BSD 3-Clause License"), "BSD-3-Clause"], @@ -107,6 +111,10 @@ def test_license_from_metadata(metadata: Metadata, expected: str) -> None: [ (None, None, None), (None, "invalid", None), + # a leading LicenseRef must not disable validation of other keys + ("LicenseRef-Foo OR Apache 2", None, None), + # SPDX idstrings only allow letters, digits, "." and "-" + ("LicenseRef-a_b", None, None), (None, None, ["License :: OSI Approved :: BSD License"]), # classifiers with no entry that maps to SPDX (see issue #1372) (None, None, ["Programming Language :: Python :: 3"]), @@ -127,7 +135,7 @@ def test_license_from_metadata_errors( license: str | None, classifiers: list[str] | None, ) -> None: - with pytest.raises((ValueError, ExceptionGroup)): + with pytest.raises((ValueError, ExceptionGroup, ExpressionError)): license_from_metadata_values( license_expression=license_expression, license_text=license, From 2d34b1a279d1570cb6123c69b7e1a618003ea822 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Prpi=C4=8D?= Date: Wed, 4 Nov 2026 15:07:24 -0500 Subject: [PATCH 35/35] fix(pkgmetadata): validate SPDX expressions with packaging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validate with `packaging.licenses.canonicalize_license_expression()` so `_parse_spdx()` follows PEP 639 exactly, and use license-expression only to build the expression object. `Apache-2.0 WITH LicenseRef-x` is now rejected and `licenseref-foo` accepted, matching `packaging`. The exception type stays `ExpressionError`; its message now comes from `packaging`. The invalid-expression cases move to their own test that checks which key was rejected, and new cases cover an exception next to a `LicenseRef` and an unknown `WITH` exception. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Martin Prpič --- src/fromager/pkgmetadata/pep639.py | 25 ++++++++------------- tests/test_pkgmetadata.py | 36 +++++++++++++++++++++++++----- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/src/fromager/pkgmetadata/pep639.py b/src/fromager/pkgmetadata/pep639.py index cf5b526ad..2817f4a11 100644 --- a/src/fromager/pkgmetadata/pep639.py +++ b/src/fromager/pkgmetadata/pep639.py @@ -6,9 +6,8 @@ - https://spdx.org/licenses/ """ -import re - from license_expression import ExpressionError, LicenseExpression, get_spdx_licensing +from packaging.licenses import InvalidLicenseExpression, canonicalize_license_expression from packaging.metadata import Metadata @@ -62,25 +61,19 @@ def license_from_metadata_values( _SPDX = get_spdx_licensing() -# SPDX idstring: letters, digits, "." and "-" -_LICENSE_REF_RE = re.compile(r"LicenseRef-[A-Za-z0-9.\-]+") - def _parse_spdx(text: str, *, simplify: bool = False) -> LicenseExpression: """Parse, validate, and simplify a SPDX license expression - ``LicenseRef-`` keys reference non-SPDX licenses (PEP 639) and may - appear anywhere in the expression, so they are exempt from the SPDX - license list check. Every other unknown key is an error. + `canonicalize_license_expression()` validates, so the rules match + PEP 639 exactly (e.g. ``LicenseRef-`` anywhere, but only SPDX + exceptions after ``WITH``); license-expression only builds the object. """ - expr = _SPDX.parse(text, validate=False) - unknown = [ - key - for key in _SPDX.unknown_license_keys(expr) - if not _LICENSE_REF_RE.fullmatch(key) - ] - if unknown: - raise ExpressionError(f"Unknown license key(s): {', '.join(unknown)}") + try: + canonical = canonicalize_license_expression(text) + except InvalidLicenseExpression as err: + raise ExpressionError(str(err)) from err + expr = _SPDX.parse(canonical, validate=False) if simplify: expr = expr.simplify() return expr diff --git a/tests/test_pkgmetadata.py b/tests/test_pkgmetadata.py index 62b8beadb..ffc9384c2 100644 --- a/tests/test_pkgmetadata.py +++ b/tests/test_pkgmetadata.py @@ -86,6 +86,11 @@ def test_project_urls_from_metadata( # LicenseRef is valid anywhere in a PEP 639 expression [md("MIT OR LicenseRef-Proprietary"), "LicenseRef-Proprietary OR MIT"], [md("LicenseRef-Proprietary"), "LicenseRef-Proprietary"], + # exception keys must not be reported as unknown next to a LicenseRef + [ + md("MIT WITH LLVM-exception OR LicenseRef-z"), + "LicenseRef-z OR MIT WITH LLVM-exception", + ], [md(license="MIT License"), "MIT"], [md(license="MIT and PSF-2.0"), "MIT AND PSF-2.0"], [md(license="BSD 3-Clause License"), "BSD-3-Clause"], @@ -111,10 +116,6 @@ def test_license_from_metadata(metadata: Metadata, expected: str) -> None: [ (None, None, None), (None, "invalid", None), - # a leading LicenseRef must not disable validation of other keys - ("LicenseRef-Foo OR Apache 2", None, None), - # SPDX idstrings only allow letters, digits, "." and "-" - ("LicenseRef-a_b", None, None), (None, None, ["License :: OSI Approved :: BSD License"]), # classifiers with no entry that maps to SPDX (see issue #1372) (None, None, ["Programming Language :: Python :: 3"]), @@ -135,9 +136,34 @@ def test_license_from_metadata_errors( license: str | None, classifiers: list[str] | None, ) -> None: - with pytest.raises((ValueError, ExceptionGroup, ExpressionError)): + with pytest.raises((ValueError, ExceptionGroup)): license_from_metadata_values( license_expression=license_expression, license_text=license, classifiers=classifiers, ) + + +@pytest.mark.parametrize( + ["license_expression", "match"], + [ + # a leading LicenseRef must not disable validation of other keys + ("LicenseRef-Foo OR Apache 2", "Apache 2"), + # SPDX idstrings only allow letters, digits, "." and "-" + ("LicenseRef-a_b", "(?i)LicenseRef-a_b"), + ("MIT WITH NotAnException", "(?i)NotAnException"), + # PEP 639 only allows SPDX exceptions after WITH + ("Apache-2.0 WITH LicenseRef-x", "(?i)LicenseRef-x"), + ], +) +def test_license_from_metadata_invalid_expression( + license_expression: str, match: str +) -> None: + with pytest.raises(ExpressionError, match=match): + license_from_metadata_values(license_expression=license_expression) + + +def test_license_from_metadata_canonicalizes_licenseref_prefix() -> None: + # PEP 639 matches the LicenseRef- prefix case-insensitively + expr = license_from_metadata_values(license_expression="licenseref-foo") + assert str(expr) == "LicenseRef-foo"