From 80f0e058b119f3f1d66885976e2fce7b05538e53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Sat, 31 Oct 2026 20:56:49 +0100 Subject: [PATCH] build: a change to tools or tests runs the test projects it reaches When every changed path is prose, .gitignore, or inside a solution project under tools/ or test/, ci-lanes answers suite=reach and the projects the change reaches: a project's own directory, what its items and imports name outside it, and what its references reach, plus test/Prose.slnf for prose. The legs write them as test/Reach.slnf and build and test that. Anything else, or anything unreadable, runs the whole solution. A test that reads another project by path declares it with : the conventions the root, the oracle tests test/. TestReachTests holds the declarations against MSBuild's evaluation and the test sources' literals. .gitignore gets the class "ignore list", Linux alone. The gate refuses a reached suite with no projects, the pack or macOS. Decision 127 amends 113. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/dotnet.yml | 15 +- .gitignore | 2 + AGENTS.md | 1 + ...tests-runs-the-test-projects-it-reaches.md | 84 +++++++ design/adr/README.md | 3 +- .../GateTests.cs | 58 ++++- .../SelectionTests.cs | 194 +++++++++++++-- .../CiLaneTests.cs | 13 +- ...CodoMetis.TypeKit.Conventions.Tests.csproj | 8 + .../ReleaseWiringTests.cs | 17 +- .../TestReachTests.cs | 230 ++++++++++++++++++ .../CodoMetis.TypeKit.Oracles.Tests.csproj | 8 + tools/CiLanes/Program.cs | 221 +++++++++++++++-- 13 files changed, 809 insertions(+), 45 deletions(-) create mode 100644 design/adr/0127-a-change-to-tools-or-tests-runs-the-test-projects-it-reaches.md create mode 100644 test/CodoMetis.TypeKit.Conventions.Tests/TestReachTests.cs diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml index ea2bc80..01d0536 100644 --- a/.github/workflows/dotnet.yml +++ b/.github/workflows/dotnet.yml @@ -65,6 +65,7 @@ jobs: legs: ${{ steps.select.outputs.legs }} pack: ${{ steps.select.outputs.pack }} suite: ${{ steps.select.outputs.suite }} + projects: ${{ steps.select.outputs.projects }} steps: # The whole history: the diff is against the base branch, which a shallow clone has not fetched. @@ -93,13 +94,15 @@ jobs: # The legs the changes job selected: Linux in every run, Windows when the change can affect it, macOS only in a # run that selects nothing. A prose-only change builds and runs test/Prose.slnf alone, the test projects that - # read prose (ProseReaderTests holds the set); anything else, the whole solution. + # read prose (ProseReaderTests holds the set). A change to tools, tests, prose or the ignore list builds and runs + # the projects it reaches, written as test/Reach.slnf (design/adr/0127, TestReachTests holds what reaches what); + # anything else, the whole solution. test: name: Test (${{ matrix.os }}) runs-on: ${{ matrix.os }} needs: changes env: - SOLUTION: ${{ needs.changes.outputs.suite == 'prose' && 'test/Prose.slnf' || 'CodoMetis.TypeKit.slnx' }} + SOLUTION: ${{ needs.changes.outputs.suite == 'prose' && 'test/Prose.slnf' || needs.changes.outputs.suite == 'reach' && 'test/Reach.slnf' || 'CodoMetis.TypeKit.slnx' }} strategy: fail-fast: false matrix: @@ -117,6 +120,14 @@ jobs: with: global-json-file: global.json + # The projects the change reaches, as a solution filter beside test/Prose.slnf. The gate refuses a reached + # suite whose projects are not one or more project files, so a filter of nothing cannot pass. + - name: Write the reached suite + if: needs.changes.outputs.suite == 'reach' + env: + PROJECTS: ${{ needs.changes.outputs.projects }} + run: printf '{"solution":{"path":"../CodoMetis.TypeKit.slnx","projects":%s}}\n' "$PROJECTS" > test/Reach.slnf + - name: Restore run: dotnet restore "$SOLUTION" diff --git a/.gitignore b/.gitignore index b72a605..8ea7a1a 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ dist/ # Test output TestResults/ +# The suite CI writes for a change to tools or tests (design/adr/0127) +/test/Reach.slnf *.trx *.coverage coverage*.xml diff --git a/AGENTS.md b/AGENTS.md index 4eda931..b72c1bf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -64,6 +64,7 @@ dotnet tools/ConsumerSmokeTest/bin/Release/net10.0/consumer-smoke-test.dll # t dotnet tools/BuildVerdict/bin/Release/net10.0/build-verdict.dll test [--project P] [--filter-class C] [--filter-method M] [-- flags] # an agent's build and test: a verdict, not the transcript dotnet tools/PrHygiene/bin/Release/net10.0/pr-hygiene.dll event.json # a PR's labels, milestone, branch and issue link, as CI's PR hygiene check reads them from the payload dotnet tools/CiLanes/bin/Release/net10.0/ci-lanes.dll classify ... # the CI lanes a changed path selects (Linux, Windows, the pack; macOS only on the release's runs), as CI's changes job decides them +dotnet tools/CiLanes/bin/Release/net10.0/ci-lanes.dll suite ... # the tests a changed path runs alone: the whole solution, the prose suite, or the projects it reaches ``` The oracle kit's local tool, which needs no model and never runs in CI diff --git a/design/adr/0127-a-change-to-tools-or-tests-runs-the-test-projects-it-reaches.md b/design/adr/0127-a-change-to-tools-or-tests-runs-the-test-projects-it-reaches.md new file mode 100644 index 0000000..3531a5a --- /dev/null +++ b/design/adr/0127-a-change-to-tools-or-tests-runs-the-test-projects-it-reaches.md @@ -0,0 +1,84 @@ +# 0127. A change to tools or tests runs the test projects it reaches, and the ignore list has a class + +Date: 2026-10-31 +Status: accepted + +Context: 2026-10-31, issue #268, amending the selection of [decision 113](0113-ci-runs-the-lanes-a-change-can-affect.md). +Under 113 a lane builds and tests the whole solution unless every changed path is prose. #266 changed one tool, +`tools/BashGuard`, its tests, docs and one line of `.gitignore`. Its first run, 41215127177, took 12m55s on Linux and +13m32s on Windows, and packed and smoke-tested too, because `.gitignore` was unlisted. The tests that could notice +the change were `CodoMetis.TypeKit.BashGuard.Tests` (112 tests, 5 s locally) and the conventions. A docs-only PR +(#267) runs `test/Prose.slnf` alone, in 1m49s. + +## Decision + +**A third suite, `reach`.** When every changed path is prose, the ignore list, or a path inside a project of the +solution under `tools/` or `test/`, and at least one is not prose, the legs build and test the projects the change +reaches, not the solution. Any other path runs the whole solution as before: `src/**`, `tools/Shared/**`, +`tools/Directory.Build.*`, `tools/CiLanes/**` and `tools/ConsumerSmokeTest/**` (the smoke test class, which keeps +the pack), `spikes/**`, `test/Directory.Build.props`, `test/oracles/**`, a solution filter, a project the solution +does not list, the build files and every unlisted path. The lanes are 113's: a reached suite runs where the paths' +classes select, so a tool or test change still runs on Windows, with the conventions and the changed tool's tests. + +**What a change reaches.** A changed path reaches a solution project under `tools/` or `test/` when the project reads +it. A project reads its own directory; every path an item or an import of its project file names outside that +directory by a relative path (a linked source such as `../CodoMetis.TypeKit.BuildVerdict.Tests/ToolSandbox.cs`, a +`ProjectReference`); and, through each `ProjectReference`, whatever the referenced project reads, so a change to +`test/CodoMetis.TypeKit.Generators.Probes` reaches the four test projects that reference it. A prose path adds the +projects of `test/Prose.slnf`. The suite is every project reached, test projects and changed tools alike, so the +changed tool is built even when no test references it (`tools/Oracles`). + +**A read by path is declared.** A test that reads another project at run time, by path rather than by reference, +declares it in its project file with an item ci-lanes reads like any other: ``. Two +projects read the repository that way. The conventions read every file (its index, every project's sources and +evaluation) and declare `../../`, the root, so every reached suite runs them. The oracle tests find the test that +holds each claim among every test project's sources and declare `../`, `test/`. No other test project reads another +by path: the tools' tests read their own tool, which they reference, and `tools/Shared`, which runs the whole +solution. + +**`TestReachTests` holds the declarations** (conventions). It evaluates every project under `tools/` and `test/` with +MSBuild (items and imports, as `CiLaneTests` does for the packages) and fails a file a test project builds from, +outside its own directory, whose change alone would not run it: the evaluation, not ci-lanes' reading of the project +file, so an include ci-lanes cannot read (one through a property) is found. It reads every test project's string +literals and fails a project named by path (`"tools/PrHygiene/Program.cs"`, or `"PrHygiene"` beside `"tools"`) or a +whole directory scanned (`Path.Combine(root, "test")` in a file that enumerates) that the reader does not reach, +unless the file is waived with the reason it reads nothing. It fails a narrowed suite without the conventions. + +**The ignore list is its own class**, `ignore list`, Linux alone, which narrows to the projects that declare they +read the root, the conventions. Measured: no test reads `.gitignore`. The conventions list the index +(`git ls-files`), which the ignore list does not change, and a fresh checkout has nothing to ignore; `dotnet pack` +does not read it. `.gitattributes` stays unlisted: it sets the line endings of every file a Windows checkout holds +and a package packs. + +**The selection answers `projects`.** `ci-lanes select` writes `suite=prose|reach|full` and `projects`, a JSON array +of project files, empty unless the suite is `reach`. The test legs write it as `test/Reach.slnf`, a filter of the +solution beside `test/Prose.slnf` (ignored by git), and restore, build and test that. `ci-lanes suite ...` +answers the suite each path alone runs, for a contributor and for `TestReachTests`. The gate passes a `reach` only +beside no pack and no macOS leg, with one or more distinct project files under `tools/` or `test/`, since a filter +of nothing builds and tests nothing and passes; it fails `projects` beside the other suites, and a missing or +malformed `projects`. A cancelled or skipped leg of a reached suite fails it as any selected job does. + +**Anything ci-lanes cannot read runs the whole solution:** the solution file, a project file, a referenced project +that does not exist, `test/Prose.slnf`, or a path no project reaches. + +## Consequences + +- A change shaped like #266 runs the prose suite, `tools/BashGuard` and its tests on Linux and Windows, with no pack. +- A change to a test project runs it, the conventions and the oracle tests; a change to a tool, the tool, its tests, + what references them, and the conventions. +- A reach the declarations miss is a test that stops running for the change that breaks it, the failure this + repository calls silent. `TestReachTests` sees the evaluation and the literals; a read built from neither (a path + read from a file, or computed) is found only by review, and the whole solution still runs on the release's pull + request into `main`, the nightly and the release. +- Windows runs a narrowed suite as Linux does: a Windows-only defect in a test project the change does not reach + surfaces on those same runs. +- A path class is changed here, by a new record, and in `tools/CiLanes` together, as under 113. + +## Evidence + +Measured locally on 2026-10-31 against the checkout: `ci-lanes suite` over #266's eleven paths answers the five +projects of `test/Prose.slnf`, `tools/BashGuard` and `CodoMetis.TypeKit.BashGuard.Tests`, and `test/Reach.slnf` +written from them builds (17 projects with the shipping projects they reference). Each new `SelectionTests` and +`GateTests` row failed against 113's selection first; `TestReachTests` failed with the oracle tests' declaration +removed, the conventions' removed, a read of `tools/PrHygiene` seeded into the BashGuard tests, and ci-lanes made to +skip linked sources and references. The CI timings before and after are recorded on #268. diff --git a/design/adr/README.md b/design/adr/README.md index d5f04e3..826760c 100644 --- a/design/adr/README.md +++ b/design/adr/README.md @@ -119,7 +119,7 @@ per area ([design](../plan.md)) link to the records they rest on. | [0110](0110-catchasync-takes-a-valuetask-at-a-lower-priority.md) | `CatchAsync` has `ValueTask` forms marked `[OverloadResolutionPriority(-1)]`, so an `async` lambda binds to the `Task` form as before (measured); `Catch` refuses a `ValueTask` at run time still; `TraverseAsync` on `Option` and `Patch` awaits the one operation, and calls nothing and checks no token without a value | 2026-10-28 | accepted | | [0111](0111-a-mapping-can-withhold-the-fault-name.md) | A mapping withholds a member's name with `FaultResponse.WithoutFaultName()`: its problem has no `fault`, no `rule` and no member description as its detail; `IFaultFeature.NameWithheld` tells the host; a response whose every fault withholds its name is described as naming none; the same-name warning counts only members that write their name | 2026-10-28 | accepted | | [0112](0112-the-prober-gives-way-to-valueranges-oracle-kit.md) | The prober, its corpus, its seeds, the CI replay and the release gate are retired for CodoMetis.ValueRanges' oracle kit under `test/oracles/`: claims quoted verbatim and held by named tests, every public member claimed or waived, oracles as ordinary test classes each with a seed CI checks applies, candidates, and a hand-written axes file; `tools/Oracles seeds` builds one project and runs one class per seed | 2026-10-28 | accepted | -| [0113](0113-ci-runs-the-lanes-a-change-can-affect.md) | CI runs the lanes a change can affect: each changed path selects Linux, Windows and the pack by its class in `tools/CiLanes` (Windows for the analyzers, the generators, tests and tools; the pack for the packages), a path no class names selects all three, a prose-only change runs only the test projects that read prose (`test/Prose.slnf`), and only a pull request into `main`, a push to `main`, the nightly and the release run macOS; a gate job, `CI`, over every job is the required check; a convention test refuses non-portable file names; `release.yml` calls the build workflow before it verifies | 2026-10-29 | accepted | +| [0113](0113-ci-runs-the-lanes-a-change-can-affect.md) | CI runs the lanes a change can affect: each changed path selects Linux, Windows and the pack by its class in `tools/CiLanes` (Windows for the analyzers, the generators, tests and tools; the pack for the packages), a path no class names selects all three, a prose-only change runs only the test projects that read prose (`test/Prose.slnf`), and only a pull request into `main`, a push to `main`, the nightly and the release run macOS; a gate job, `CI`, over every job is the required check; a convention test refuses non-portable file names; `release.yml` calls the build workflow before it verifies | 2026-10-29 | accepted; amended by [0127](0127-a-change-to-tools-or-tests-runs-the-test-projects-it-reaches.md) | | [0114](0114-a-value-object-s-temporary-number-is-its-wrapped-type-s.md) | A value object over a number gets the temporary values a property of its wrapped type gets, from a factory in its converter's mapping hints, which EF consults after a generator on the property and after HiLo: a default `ulong` key no longer throws at `Add`, and a `char` key configured on add gets a temporary `char` on SQL Server; under `--nativeaot` the factory is lost and a default `ulong` key still throws | 2026-10-29 | accepted | | [0115](0115-value-objects-can-be-inlined-everywhere.md) | `AddTypeKit(typeKit => typeKit.InlineValueObjects = true)` writes every value object in place, by its wrapped type's schema, in every position and version, with no component of its own; off by default, the document unchanged; closed sets and wrapped enums keep their components; an inlined value object claims no name (decision 55) and shares no schema between its JSON positions and its parameters (decision 106) | 2026-10-29 | accepted | | [0116](0116-the-docs-site-is-docfx-built-from-main.md) | The docs site is DocFX, published to GitHub Pages from `main` alone: the guides, the READMEs, the changelog and the API pages of the run-time packages, from the files as they are; a link to a file that is not a page becomes a GitHub address at the built commit; a pull request touching a page builds it with warnings as errors; `site/**` is prose; DocFX 2.81.0 lists the members of C# 14 extension blocks as extension methods | 2026-10-29 | accepted | @@ -133,3 +133,4 @@ per area ([design](../plan.md)) link to the records they rest on. | [0124](0124-an-operator-in-a-query-is-the-wrapped-type-s-arithmetic.md) | In an EF Core query, every generated arithmetic operator is rewritten to the wrapped type's operator over `ProviderValue`, which translates only through TypeKit's converter, its value-object result `Materialize`d and re-typed, never cast; `byte`, `short` and `char` operators are refused; refines decision 122, on decision 123's mark | 2026-10-30 | accepted | | [0125](0125-a-package-documents-the-third-party-code-it-contains.md) | A package carries a `THIRD-PARTY-NOTICES.txt` only when it contains third-party code (a bundled assembly, copied source, a binary release), built from the shipping closure with the license texts; a package that bundles nothing documents its own license, copyright and dependency list, and its dependencies' licenses are held to an allow-list by a conventions test, not listed (#181); no SBOM is shipped, since GitHub's export is an inventory without license data | 2026-10-30 | open | | [0126](0126-parallel-work-runs-in-worktrunk-worktrees-beside-the-repository.md) | Parallel work runs in worktrunk worktrees at `../CodoMetis.TypeKit-worktrees/`, set per machine in worktrunk's user config; worktrunk's default branch is the open `release/`, set per clone, so `wt switch -c`, the plugin's agent worktrees, `wt remove` and `wt step prune` base on it; the Bash guard refuses `wt merge` and `wt step squash`, `rebase` and `push`, since work lands by a PR and a merge commit (decision 94); nothing is copied into a new worktree, which builds the solution cold in 27 s; `.claude/worktrees/` joins `.gitignore`; [design/worktrees.md](../worktrees.md) is the runbook | 2026-10-31 | accepted | +| [0127](0127-a-change-to-tools-or-tests-runs-the-test-projects-it-reaches.md) | A change whose paths are prose, the ignore list or inside solution projects under `tools/` and `test/` runs the projects it reaches (`suite=reach`, written as `test/Reach.slnf`): a project reaches its directory, what its items and imports name and what its references reach, and a run-time read by path is declared with `` (the conventions the root, the oracle tests `test/`), which `TestReachTests` holds against MSBuild's evaluation and the sources' literals; anything else, or anything unreadable, runs the whole solution; `.gitignore` is Linux alone; the gate refuses a reached suite with no projects, the pack or macOS; amends decision 113 | 2026-10-31 | accepted | diff --git a/test/CodoMetis.TypeKit.CiLanes.Tests/GateTests.cs b/test/CodoMetis.TypeKit.CiLanes.Tests/GateTests.cs index d788697..af76d18 100644 --- a/test/CodoMetis.TypeKit.CiLanes.Tests/GateTests.cs +++ b/test/CodoMetis.TypeKit.CiLanes.Tests/GateTests.cs @@ -22,6 +22,7 @@ private sealed record Run( string? Legs = LinuxOnly, string? Pack = "false", string? Suite = "prose", + string? Projects = "[]", string Test = "success", string PackJob = "skipped", string Consumer = "skipped"); @@ -32,12 +33,17 @@ private sealed record Run( private static readonly Run Full = Package with { Legs = EveryLeg }; + private const string ReachedProjects = """["test/CodoMetis.TypeKit.Conventions.Tests/CodoMetis.TypeKit.Conventions.Tests.csproj","tools/PrHygiene/PrHygiene.csproj"]"""; + + private static readonly Run Reach = Prose with { Legs = LinuxAndWindows, Suite = "reach", Projects = ReachedProjects }; + private (int Code, string Output) Judge(Run run, Action>? edit = null) { var outputs = new Dictionary(); if (run.Legs is not null) outputs["legs"] = run.Legs; if (run.Pack is not null) outputs["pack"] = run.Pack; if (run.Suite is not null) outputs["suite"] = run.Suite; + if (run.Projects is not null) outputs["projects"] = run.Projects; var needs = new Dictionary { @@ -98,12 +104,60 @@ public void The_prose_suite_beside_the_pack_fails() => [Fact] public void A_run_that_is_not_selected_passes_with_every_leg() => Passes(Full); + [Fact] + public void A_reached_suite_passes_on_Linux_and_Windows() => Passes(Reach); + + [Fact] + public void A_reached_suite_passes_on_Linux_alone() => Passes(Reach with { Legs = LinuxOnly }); + + /// The narrowed suite is held like any other: a leg that was skipped or cancelled fails the run (#268). + [Theory] + [InlineData("cancelled")] + [InlineData("skipped")] + [InlineData("failure")] + public void A_reached_suite_whose_legs_did_not_succeed_fails(string result) => + Fails(Reach with { Test = result }, $"The job 'test' was selected and ended '{result}'."); + + [Fact] + public void A_reached_suite_beside_the_pack_fails() => + Fails(Reach with { Pack = "true", PackJob = "success", Consumer = "success" }, "The changes job chose the reached suite for a run with the pack or the macOS leg"); + + [Fact] + public void A_reached_suite_beside_every_leg_fails() => + Fails(Reach with { Legs = EveryLeg }, "The changes job chose the reached suite for a run with the pack or the macOS leg"); + + /// An empty filter would build and test nothing, and pass. + [Theory] + [InlineData("[]")] + [InlineData("""["CodoMetis.TypeKit.slnx"]""")] + [InlineData("""["src/CodoMetis.TypeKit/CodoMetis.TypeKit.csproj"]""")] + [InlineData("""["test/../src/CodoMetis.TypeKit/CodoMetis.TypeKit.csproj"]""")] + [InlineData("""["test/A.Tests/A.Tests.csproj","test/A.Tests/A.Tests.csproj"]""")] + public void A_reached_suite_without_its_projects_fails(string projects) => + Fails(Reach with { Projects = projects }, "The changes job chose the reached suite with the projects"); + + [Theory] + [InlineData("full")] + [InlineData("prose")] + public void Projects_beside_a_suite_that_has_its_own_fail(string suite) => + Fails((suite == "full" ? Package : Prose) with { Projects = ReachedProjects }, $"The changes job named projects for the {suite} suite"); + + [Theory] + [InlineData("test/A.Tests/A.Tests.csproj")] + [InlineData("""{"projects":[]}""")] + [InlineData("""[1]""")] + public void A_projects_answer_the_selection_does_not_write_fails(string projects) => + Fails(Reach with { Projects = projects }, "The changes job's projects output is"); + + [Fact] + public void A_missing_projects_answer_fails() => Fails(Prose with { Projects = null }, "The changes job's projects output is missing"); + [Theory] [InlineData("failure")] [InlineData("cancelled")] [InlineData("skipped")] public void A_selection_that_did_not_succeed_fails(string result) => - Fails(Prose with { Changes = result, Legs = null, Pack = null, Suite = null }, $"The changes job ended '{result}'"); + Fails(Prose with { Changes = result, Legs = null, Pack = null, Suite = null, Projects = null }, $"The changes job ended '{result}'"); [Theory] [InlineData("cancelled")] @@ -157,7 +211,7 @@ public void The_summary_tables_each_job() var file = Path.Combine(_tool.Work, "needs.json"); File.WriteAllText(file, JsonSerializer.Serialize(new Dictionary { - ["changes"] = new { result = "success", outputs = new { legs = LinuxOnly, pack = "false", suite = "prose" } }, + ["changes"] = new { result = "success", outputs = new { legs = LinuxOnly, pack = "false", suite = "prose", projects = "[]" } }, ["test"] = new { result = "success" }, ["pack"] = new { result = "skipped" }, ["consumer"] = new { result = "skipped" }, diff --git a/test/CodoMetis.TypeKit.CiLanes.Tests/SelectionTests.cs b/test/CodoMetis.TypeKit.CiLanes.Tests/SelectionTests.cs index 8ecf0ca..091bb62 100644 --- a/test/CodoMetis.TypeKit.CiLanes.Tests/SelectionTests.cs +++ b/test/CodoMetis.TypeKit.CiLanes.Tests/SelectionTests.cs @@ -11,10 +11,54 @@ public sealed class SelectionTests : IDisposable private const string LinuxAndWindows = """legs=["ubuntu-latest","windows-latest"]"""; private const string EveryLeg = """legs=["ubuntu-latest","windows-latest","macos-latest"]"""; + private const string Conventions = "test/CodoMetis.TypeKit.Conventions.Tests/CodoMetis.TypeKit.Conventions.Tests.csproj"; + private const string Oracles = "test/CodoMetis.TypeKit.Oracles.Tests/CodoMetis.TypeKit.Oracles.Tests.csproj"; + private const string Samples = "test/CodoMetis.TypeKit.Samples/CodoMetis.TypeKit.Samples.csproj"; + private const string CoreTests = "test/CodoMetis.TypeKit.Tests/CodoMetis.TypeKit.Tests.csproj"; + private const string HygieneTests = "test/CodoMetis.TypeKit.PrHygiene.Tests/CodoMetis.TypeKit.PrHygiene.Tests.csproj"; + private const string GuardTests = "test/CodoMetis.TypeKit.BashGuard.Tests/CodoMetis.TypeKit.BashGuard.Tests.csproj"; + private const string WorkflowTests = "test/CodoMetis.TypeKit.Workflow.Tests/CodoMetis.TypeKit.Workflow.Tests.csproj"; + private const string Hygiene = "tools/PrHygiene/PrHygiene.csproj"; + private readonly ToolSandbox _tool = new("CiLanes", "ci-lanes"); + /// + /// The base is a small solution shaped like the repository's: a package, two tools and their tests, one test + /// project that compiles in a file of another, one that references a test project, and the two that read the + /// repository by path and declare it, the conventions (the whole tree) and the oracles (test/). + /// public SelectionTests() { + Write("CodoMetis.TypeKit.slnx", $""" + + + + + + + + + + + + + + """); + Write("test/Prose.slnf", $$"""{ "solution": { "path": "../CodoMetis.TypeKit.slnx", "projects": [ "{{Conventions}}", "{{Oracles}}", "{{Samples}}" ] } }"""); + Write("src/CodoMetis.TypeKit/CodoMetis.TypeKit.csproj", Project()); + Write(Hygiene, Project()); + Write("tools/BashGuard/BashGuard.csproj", Project()); + Write(CoreTests, Project("""""")); + Write(HygieneTests, Project("""""", + """""")); + Write(GuardTests, Project("""""")); + Write(WorkflowTests, Project("""""")); + Write(Conventions, Project("""""")); + Write(Oracles, Project("""""")); + Write(Samples, Project("""""")); + Write("tools/PrHygiene/Program.cs", "// a tool\n"); + Write("test/CodoMetis.TypeKit.BashGuard.Tests/ToolSandbox.cs", "// shared by the tools' tests\n"); + Write("test/CodoMetis.TypeKit.Tests/OptionTests.cs", "// a test\n"); Write("src/CodoMetis.TypeKit/Option.cs", "// a package's source\n"); Write("src/CodoMetis.TypeKit.Analyzers/CodeFixes/AddCreate.cs", "// a code fix's source\n"); Write("docs/guide.md", "# A guide\n"); @@ -54,6 +98,8 @@ public SelectionTests() { "test/CodoMetis.TypeKit.Tests/OptionTests.cs", "code no package reads", "linux, windows" }, { "test/Directory.Build.props", "code no package reads", "linux, windows" }, { "spikes/FabricSpike/Program.cs", "code no package reads", "linux, windows" }, + { ".gitignore", "ignore list", "linux" }, + { ".gitattributes", "unlisted", "linux, windows, pack" }, { "global.json", "unlisted", "linux, windows, pack" }, { "Directory.Build.props", "unlisted", "linux, windows, pack" }, { "Directory.Packages.props", "unlisted", "linux, windows, pack" }, @@ -78,17 +124,17 @@ public void A_prose_change_runs_the_Linux_leg_alone() { Commit("docs/guide.md", "# A guide, reworded\n"); - Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxOnly, "pack=false", "suite=prose"]); + Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxOnly, "pack=false", "suite=prose", "projects=[]"]); } - /// The prose suite runs only for a change that is prose throughout. + /// The prose suite runs only for a change that is prose throughout; beside a test change it joins the projects reached. [Fact] - public void Prose_beside_any_other_change_runs_the_whole_suite() + public void Prose_beside_a_test_change_adds_the_prose_suite_to_what_the_test_reaches() { Commit("docs/guide.md", "# A guide, reworded\n"); - Commit("test/CodoMetis.TypeKit.Tests/OptionTests.cs", "// a test\n"); + Commit("test/CodoMetis.TypeKit.Tests/OptionTests.cs", "// a test, changed\n"); - Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxAndWindows, "pack=false", "suite=full"]); + Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxAndWindows, "pack=false", "suite=reach", Projects(Conventions, Oracles, Samples, CoreTests)]); } [Fact] @@ -96,7 +142,7 @@ public void A_changelog_change_runs_the_whole_suite() { Commit("CHANGELOG.md", "## 9.9.0\n"); - Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxOnly, "pack=true", "suite=full"]); + Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxOnly, "pack=true", "suite=full", "projects=[]"]); } [Fact] @@ -105,7 +151,7 @@ public void A_package_change_runs_Linux_and_the_pack() Commit("docs/guide.md", "# A guide, reworded\n"); Commit("src/CodoMetis.TypeKit/Option.cs", "// changed\n"); - Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxOnly, "pack=true", "suite=full"]); + Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxOnly, "pack=true", "suite=full", "projects=[]"]); } /// The one src/ defect a Windows leg alone caught in 150 runs was a code fix's line endings (f9f64e5). @@ -115,15 +161,119 @@ public void An_analyzer_change_adds_the_Windows_leg() Commit("src/CodoMetis.TypeKit/Option.cs", "// changed\n"); Commit("src/CodoMetis.TypeKit.Analyzers/CodeFixes/AddCreate.cs", "// changed\n"); - Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxAndWindows, "pack=true", "suite=full"]); + Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxAndWindows, "pack=true", "suite=full", "projects=[]"]); + } + + /// + /// A test project's change runs that project, the conventions and the oracles, which read every test project's + /// sources (decision 127), on Linux and Windows, without the pack. + /// + [Fact] + public void A_test_project_change_runs_that_project_and_the_repository_readers() + { + Commit("test/CodoMetis.TypeKit.Tests/OptionTests.cs", "// a test, changed\n"); + + Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxAndWindows, "pack=false", "suite=reach", Projects(Conventions, Oracles, CoreTests)]); + } + + /// + /// A tool's change runs the tool's build, its tests, a test project that reaches it through another + /// (ProjectReference, transitively), and the conventions; not the oracles, which read test/ alone (#268). + /// + [Fact] + public void A_tool_change_runs_the_test_projects_that_reach_it() + { + Commit("tools/PrHygiene/Program.cs", "// a tool, changed\n"); + + Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxAndWindows, "pack=false", "suite=reach", Projects(Conventions, HygieneTests, WorkflowTests, Hygiene)]); + } + + /// A file another test project compiles in reaches that project, and what references it. + [Fact] + public void A_linked_file_reaches_the_project_that_compiles_it_in() + { + Commit("test/CodoMetis.TypeKit.BashGuard.Tests/ToolSandbox.cs", "// changed\n"); + + Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxAndWindows, "pack=false", "suite=reach", Projects(GuardTests, Conventions, Oracles, HygieneTests, WorkflowTests)]); } + /// The shape of #266: one tool, its tests, the docs and the ignore list. [Fact] - public void A_test_change_runs_Linux_and_Windows_without_the_pack() + public void A_tool_its_tests_and_docs_run_a_narrowed_suite_without_the_pack() { - Commit("test/CodoMetis.TypeKit.Tests/OptionTests.cs", "// a test\n"); + Commit("tools/PrHygiene/Program.cs", "// a tool, changed\n"); + Commit("test/CodoMetis.TypeKit.PrHygiene.Tests/HygieneTests.cs", "// its tests\n"); + Commit("docs/guide.md", "# A guide, reworded\n"); + Commit(".gitignore", "bin/\n"); - Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxAndWindows, "pack=false", "suite=full"]); + Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxAndWindows, "pack=false", "suite=reach", Projects(Conventions, Oracles, Samples, HygieneTests, WorkflowTests, Hygiene)]); + } + + /// A fresh checkout ignores nothing, so the ignore list is read by git alone and the conventions that list the index. + [Fact] + public void An_ignore_list_change_runs_the_conventions_on_Linux() + { + Commit(".gitignore", "bin/\n"); + + Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxOnly, "pack=false", "suite=reach", Projects(Conventions)]); + } + + /// + /// tools/Shared is compiled into every tool, the smoke test among them: the whole solution, and the pack. (A file + /// the fingerprint does not hash, since an edited source would make the sandbox's tool refuse to run stale.) + /// + [Fact] + public void A_shared_tooling_change_runs_the_whole_suite() + { + Commit("tools/PrHygiene/Program.cs", "// a tool, changed\n"); + Commit("tools/Shared/notes.txt", "changed\n"); + + Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxAndWindows, "pack=true", "suite=full", "projects=[]"]); + } + + [Fact] + public void A_tool_change_beside_a_package_change_runs_the_whole_suite() + { + Commit("tools/PrHygiene/Program.cs", "// a tool, changed\n"); + Commit("src/CodoMetis.TypeKit/Option.cs", "// changed\n"); + + Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxAndWindows, "pack=true", "suite=full", "projects=[]"]); + } + + /// A path under test/ or tools/ that no solution project holds reaches projects nobody can name: the whole suite. + [Theory] + [InlineData("test/Directory.Build.props")] + [InlineData("test/oracles/claims.json")] + [InlineData("test/CodoMetis.TypeKit.Unlisted.Tests/Tests.cs")] + [InlineData("tools/AgentTools.slnf")] + public void A_path_outside_every_solution_project_runs_the_whole_suite(string path) + { + Commit("tools/PrHygiene/Program.cs", "// a tool, changed\n"); + Commit(path, "// changed\n"); + + Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxAndWindows, "pack=false", "suite=full", "projects=[]"]); + } + + /// A project file that cannot be read hides what that project reaches, so nothing is narrowed. + [Fact] + public void A_project_file_that_cannot_be_read_runs_the_whole_suite() + { + Commit(WorkflowTests, "A selection never runs macOS: a path no class names gets every lane but that one. @@ -132,7 +282,7 @@ public void An_unlisted_change_runs_every_lane_but_macOS() { Commit("global.json", "{}\n"); - Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxAndWindows, "pack=true", "suite=full"]); + Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxAndWindows, "pack=true", "suite=full", "projects=[]"]); } /// Without --no-renames, git lists a rename's new path alone, and a move out of src/ reads as prose (#70). @@ -142,7 +292,7 @@ public void A_move_out_of_a_package_counts_its_old_path() Git("mv", "src/CodoMetis.TypeKit/Option.cs", "docs/Option.cs.md"); Git("commit", "-q", "-m", "move"); - Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxOnly, "pack=true", "suite=full"]); + Answers(PullRequestInto("release/9.9.0")).ShouldBe([LinuxOnly, "pack=true", "suite=full", "projects=[]"]); } [Fact] @@ -150,7 +300,7 @@ public void The_release_pull_request_into_main_runs_every_lane() { Commit("docs/guide.md", "# A guide, reworded\n"); - Answers(PullRequestInto("main")).ShouldBe([EveryLeg, "pack=true", "suite=full"]); + Answers(PullRequestInto("main")).ShouldBe([EveryLeg, "pack=true", "suite=full", "projects=[]"]); } [Fact] @@ -159,7 +309,7 @@ public void A_push_to_a_release_branch_is_selected_by_what_it_replaced() var before = Git("rev-parse", "HEAD").Trim(); Commit("docs/guide.md", "# A guide, reworded\n"); - Answers("--event", "push", "--ref", "refs/heads/release/9.9.0", "--before", before).ShouldBe([LinuxOnly, "pack=false", "suite=prose"]); + Answers("--event", "push", "--ref", "refs/heads/release/9.9.0", "--before", before).ShouldBe([LinuxOnly, "pack=false", "suite=prose", "projects=[]"]); } [Theory] @@ -170,7 +320,7 @@ public void A_push_without_a_commit_to_diff_against_runs_every_lane(string befor { Commit("docs/guide.md", "# A guide, reworded\n"); - Answers("--event", "push", "--ref", "refs/heads/release/9.9.0", "--before", before).ShouldBe([EveryLeg, "pack=true", "suite=full"]); + Answers("--event", "push", "--ref", "refs/heads/release/9.9.0", "--before", before).ShouldBe([EveryLeg, "pack=true", "suite=full", "projects=[]"]); } [Theory] @@ -182,12 +332,12 @@ public void Every_other_run_gets_every_lane(string @event, string @ref) { Commit("docs/guide.md", "# A guide, reworded\n"); - Answers("--event", @event, "--ref", @ref, "--before", Git("rev-parse", "HEAD~1").Trim()).ShouldBe([EveryLeg, "pack=true", "suite=full"]); + Answers("--event", @event, "--ref", @ref, "--before", Git("rev-parse", "HEAD~1").Trim()).ShouldBe([EveryLeg, "pack=true", "suite=full", "projects=[]"]); } [Fact] public void A_diff_that_names_no_path_runs_every_lane() => - Answers(PullRequestInto("release/9.9.0")).ShouldBe([EveryLeg, "pack=true", "suite=full"]); + Answers(PullRequestInto("release/9.9.0")).ShouldBe([EveryLeg, "pack=true", "suite=full", "projects=[]"]); /// A diff that cannot be taken fails the job, never answers "nothing changed" (#70). [Fact] @@ -219,11 +369,19 @@ public void The_summary_names_each_path_and_its_class() [InlineData("select", "--event", "push")] [InlineData("select", "--event", "push", "--ref", "refs/heads/main", "--colour", "red")] [InlineData("classify")] + [InlineData("suite")] [InlineData("gate")] [InlineData("lanes")] public void A_usage_error_is_exit_2(params string[] arguments) => _tool.Run(arguments: arguments).Code.ShouldBe(2); + private static string Project(params string[] items) => + $"\n \n{string.Concat(items.Select(item => $" {item}\n"))} \n\n"; + + // The projects answer: a JSON array, in ordinal order. + private static string Projects(params string[] projects) => + $"projects={System.Text.Json.JsonSerializer.Serialize(projects.Order(StringComparer.Ordinal))}"; + private static string[] PullRequestInto(string target) => ["--event", "pull_request", "--ref", "refs/pull/1/merge", "--base", target]; private string[] Answers(params string[] options) diff --git a/test/CodoMetis.TypeKit.Conventions.Tests/CiLaneTests.cs b/test/CodoMetis.TypeKit.Conventions.Tests/CiLaneTests.cs index 0fe055d..8d7931e 100644 --- a/test/CodoMetis.TypeKit.Conventions.Tests/CiLaneTests.cs +++ b/test/CodoMetis.TypeKit.Conventions.Tests/CiLaneTests.cs @@ -75,10 +75,13 @@ private static bool InRepository(string path) => path.StartsWith(Repository.Root + Path.DirectorySeparatorChar, StringComparison.Ordinal) && !Path.GetRelativePath(Repository.Root, path).Split(Path.DirectorySeparatorChar).Any(part => part is "bin" or "obj"); + /// The built ci-lanes.dll this project's build ordered before it. + internal static string CiLanes { get; } = Path.Combine( + typeof(CiLaneTests).Assembly.GetCustomAttributes().Single(a => a.Key == "CiLanesOutput").Value!, "ci-lanes.dll"); + private static List<(string Path, string Class, string[] Lanes)> Classify(IReadOnlyList paths) { - var output = typeof(CiLaneTests).Assembly.GetCustomAttributes().Single(a => a.Key == "CiLanesOutput").Value!; - var (code, text) = Run(Path.Combine(output, "ci-lanes.dll"), ["classify", .. paths]); + var (code, text) = Run(CiLanes, ["classify", .. paths]); code.ShouldBe(0, text); return @@ -90,7 +93,7 @@ .. text.Split('\n', StringSplitOptions.RemoveEmptyEntries) } // The project and every file it imports, from the preprocessed project: each import is named there by its full path, on a line of its own. - private static async Task> Imports(string project) + internal static async Task> Imports(string project) { var file = Path.Combine(Path.GetTempPath(), $"cilanes-{Guid.NewGuid():N}.xml"); try @@ -106,7 +109,7 @@ private static async Task> Imports(string project) } // Every item that names a file: what is compiled, embedded, packed or analysed, and the projects referenced. - private static async Task> Items(string project) + internal static async Task> Items(string project) { string[] kinds = ["Compile", "None", "Content", "EmbeddedResource", "AdditionalFiles", "ProjectReference"]; var (code, text) = await RunAsync(["msbuild", project, .. kinds.Select(kind => $"-getItem:{kind}"), "-nodeReuse:false"]); @@ -124,7 +127,7 @@ .. evaluation.RootElement.GetProperty("Items").EnumerateObject() private static Task<(int Code, string Output)> RunAsync(string[] arguments) => Task.Run(() => Run(null, arguments)); // dotnet with the arguments, or the dll with them when one is given; stdout, and stderr after it on a failure. - private static (int Code, string Output) Run(string? dll, string[] arguments) + internal static (int Code, string Output) Run(string? dll, string[] arguments) { using var process = Process.Start(new ProcessStartInfo("dotnet", dll is null ? arguments : [dll, .. arguments]) { diff --git a/test/CodoMetis.TypeKit.Conventions.Tests/CodoMetis.TypeKit.Conventions.Tests.csproj b/test/CodoMetis.TypeKit.Conventions.Tests/CodoMetis.TypeKit.Conventions.Tests.csproj index e05e52e..3be52fb 100644 --- a/test/CodoMetis.TypeKit.Conventions.Tests/CodoMetis.TypeKit.Conventions.Tests.csproj +++ b/test/CodoMetis.TypeKit.Conventions.Tests/CodoMetis.TypeKit.Conventions.Tests.csproj @@ -22,6 +22,14 @@ + + + + + + + + + diff --git a/tools/CiLanes/Program.cs b/tools/CiLanes/Program.cs index 754a220..d7ede03 100644 --- a/tools/CiLanes/Program.cs +++ b/tools/CiLanes/Program.cs @@ -1,5 +1,6 @@ -// Which lanes of .github/workflows/dotnet.yml a run needs, and whether the run did what was selected -// (design/adr/0113-ci-runs-the-lanes-a-change-can-affect.md). Four lanes: the Linux test leg, in every run; +// Which lanes of .github/workflows/dotnet.yml a run needs, which tests its legs run, and whether the run did what was +// selected (design/adr/0113-ci-runs-the-lanes-a-change-can-affect.md, design/adr/0127-a-change-to-tools-or-tests-runs-the-test-projects-it-reaches.md). +// Four lanes: the Linux test leg, in every run; // `windows` and `macos`, those test legs; `pack`, the Pack and Consumer jobs. Each changed path selects lanes by // the first class of Table whose pattern matches it, and a path no class names selects every lane a selection // can: all but macOS, which only the runs that are not selected get. @@ -8,11 +9,15 @@ // ci-lanes select --event --ref [--base ] [--before ] [--output ] [--summary ] // A pull request into release/** is selected by `git diff origin/...HEAD`, a push to release/** by // `git diff HEAD`; every other run gets every lane, macOS included. Appends `legs=`, -// `pack=true|false` and `suite=prose|full` to --output (the step's GITHUB_OUTPUT), or prints them, and each path's -// class to --summary. The suite is prose when every changed path is: the Linux leg then builds and runs only the -// test projects of test/Prose.slnf, the ones that read prose. +// `pack=true|false`, `suite=prose|reach|full` and `projects=` to --output (the step's GITHUB_OUTPUT), +// or prints them, and each path's class to --summary. The suite is prose when every changed path is: the Linux +// leg then builds and runs only the test projects of test/Prose.slnf, the ones that read prose. It is reach when +// every other path is in a solution project under tools/ or test/, or is the ignore list: the legs then build +// and run the projects those paths reach (Reach below), which `projects` names, and the whole solution otherwise. // ci-lanes classify ... // One line per path: the path, its class and its lanes, tab-separated. +// ci-lanes suite ... +// One line per path: the path, the suite a change to it alone runs, and that suite's projects, tab-separated. // ci-lanes gate [--summary ] // Reads the workflow's toJSON(needs) and passes only when the changes job succeeded with answers this tool // writes, every job it selected succeeded and every other was skipped. @@ -23,6 +28,8 @@ using System.Text; using System.Text.Json; using System.Text.RegularExpressions; +using System.Xml; +using System.Xml.Linq; Console.Out.NewLine = "\n"; Console.Error.NewLine = "\n"; @@ -41,6 +48,7 @@ ["select", .. var rest] when Options.Parse(rest, ["event", "ref", "base", "before", "output", "summary"]) is { } options && options.ContainsKey("event") && options.ContainsKey("ref") => Selection.Run(root!, options), ["classify", _, ..] => Classify(args[1..]), + ["suite", _, ..] => Suite(root!, args[1..]), ["gate", var needs, .. var rest] when Options.Parse(rest, ["summary"]) is { } options => Gate.Run(needs, options), _ => Usage(), }; @@ -55,11 +63,22 @@ static int Classify(string[] paths) return 0; } +static int Suite(string root, string[] paths) +{ + foreach (var path in paths) + { + var suite = Reach.Suite(root, [(path, Table.Of(path))]); + Console.WriteLine($"{path}\t{suite.Kind}\t{string.Join(", ", suite.Projects)}"); + } + return 0; +} + static int Usage() { Console.Error.WriteLine("Usage: dotnet tools/CiLanes/bin/Release/net10.0/ci-lanes.dll"); Console.Error.WriteLine(" select --event --ref [--base ] [--before ] [--output ] [--summary ]"); Console.Error.WriteLine(" classify ..."); + Console.Error.WriteLine(" suite ..."); Console.Error.WriteLine(" gate [--summary ]"); return 2; } @@ -102,6 +121,17 @@ public sealed record Class(string Name, Lane Lanes, string[] Patterns, string Wh public static readonly Class Prose = new("prose", Lane.Linux, ["docs/**", "design/**", "site/**", ".github/ISSUE_TEMPLATE/**", "**/*.md"], "read by the prose suite, built into nothing"); + // Built and tested, never packed. A path of it in a solution project under tools/ or test/ narrows the suite to the + // projects it reaches; any other (spikes/, test/Directory.Build.props) runs the whole solution. + public static readonly Class Code = new("code no package reads", Lane.Linux | Lane.Windows, ["test/**", "spikes/**", "tools/**"], + "built and tested, never packed, and handling paths, line endings and child processes itself"); + + // A fresh checkout has nothing it ignores, and the tests list the index (git ls-files), never the working tree: read + // by git alone, and by the projects that declare they read the root. `.gitattributes` is not this class: it sets the + // line endings of every file a Windows checkout holds and a package packs. + public static readonly Class IgnoreList = new("ignore list", Lane.Linux, [".gitignore"], + "read by git alone; a fresh checkout ignores nothing, and the tests read the index"); + // The first class whose pattern matches a path decides its lanes. `**` matches any number of directories, // `*` and `?` stay within one. Decision 113 holds this table; CiLaneTests holds it to the files MSBuild // evaluates for the packages and the smoke test. @@ -115,8 +145,8 @@ public sealed record Class(string Name, Lane Lanes, string[] Patterns, string Wh new("smoke test", Lane.Selectable, ["tools/ConsumerSmokeTest/**", "tools/Shared/**", "tools/Directory.Build.*", "tools/CiLanes/**"], "the smoke test, what it compiles in, or this selection"), Prose, - new("code no package reads", Lane.Linux | Lane.Windows, ["test/**", "spikes/**", "tools/**"], - "built and tested, never packed, and handling paths, line endings and child processes itself"), + Code, + IgnoreList, ]; public static readonly Class Unlisted = new("unlisted", Lane.Selectable, [], "no class names it"); @@ -207,19 +237,24 @@ public static int Run(string root, Dictionary options) var legs = JsonSerializer.Serialize(Table.Legs(lanes)); var pack = lanes.HasFlag(Lane.Pack) ? "true" : "false"; - var suite = classified.Count > 0 && classified.All(row => row.Class == Table.Prose) ? "prose" : "full"; + var suite = Reach.Suite(root, classified); + var projects = JsonSerializer.Serialize(suite.Projects); Console.WriteLine($"{char.ToUpperInvariant(why[0])}{why[1..]}."); foreach (var (path, @class) in classified) Console.WriteLine($" {path}: {@class.Name} ({Table.Describe(@class.Lanes)})"); - Console.WriteLine($"Lanes: {Table.Describe(lanes)}. Test legs: {string.Join(", ", Table.Legs(lanes))}. Pack and Consumer: {(pack == "true" ? "run" : "skipped")}. Suite: {suite}."); + Console.WriteLine($"Lanes: {Table.Describe(lanes)}. Test legs: {string.Join(", ", Table.Legs(lanes))}. Pack and Consumer: {(pack == "true" ? "run" : "skipped")}. Suite: {suite.Kind}{(suite.Why is null ? "" : $" ({suite.Why})")}."); + foreach (var project in suite.Projects) Console.WriteLine($" {project}"); - var answers = $"legs={legs}\npack={pack}\nsuite={suite}\n"; + var answers = $"legs={legs}\npack={pack}\nsuite={suite.Kind}\nprojects={projects}\n"; if (options.TryGetValue("output", out var file)) File.AppendAllText(file, answers); else Console.Write(answers); if (options.TryGetValue("summary", out var summary)) { - var text = new StringBuilder($"### Lanes: {Table.Describe(lanes)}\n\n{char.ToUpperInvariant(why[0])}{why[1..]}.\n\n"); + var text = new StringBuilder($"### Lanes: {Table.Describe(lanes)}, suite: {suite.Kind}\n\n{char.ToUpperInvariant(why[0])}{why[1..]}.\n\n"); + if (suite.Why is not null) text.Append($"The suite is the whole solution: {suite.Why}.\n\n"); + if (suite.Projects.Length > 0) + text.Append("The legs build and test the projects the change reaches:\n\n").AppendJoin("", suite.Projects.Select(p => $"- `{p}`\n")).Append('\n'); if (classified.Count > 0) { text.Append("| Path | Class | Lanes |\n|---|---|---|\n"); @@ -245,7 +280,145 @@ static int Git(string root, out string output, params string[] arguments) } } -static class Gate +// The test suite a selected run builds and tests (design/adr/0127). A path reaches a solution project under tools/ or +// test/ when the project reads it: the project's own directory, every path an item or import of its project file names +// outside that directory (a linked source, a declared ``), and, through each +// ProjectReference, whatever the referenced project reads. A test that reads a project by path at run time declares +// it with ReadsByPath; TestReachTests (conventions) holds every test project's evaluated inputs and path literals to +// what this reads. Anything that cannot be read, or a path no project holds, runs the whole solution. +static class Reach +{ + public sealed record Answer(string Kind, string[] Projects, string? Why = null); + + const string ProseFilter = "test/Prose.slnf"; + + public static Answer Suite(string root, IReadOnlyList<(string Path, Table.Class Class)> classified) + { + if (classified.Count == 0) return new("full", [], "no changed path was classified"); + if (classified.All(row => row.Class == Table.Prose)) return new("prose", []); + + var (projects, failure) = Read(root); + if (projects is null) return new("full", [], failure); + + var suite = new SortedSet(StringComparer.Ordinal); + if (classified.Any(row => row.Class == Table.Prose)) + { + var (prose, unread) = ProseProjects(root); + if (prose is null) return new("full", [], unread); + suite.UnionWith(prose); + } + + foreach (var (path, @class) in classified) + { + if (@class == Table.Prose) continue; + if (@class == Table.IgnoreList || (@class == Table.Code && Owner(projects, path) is not null)) + { + var reached = projects.Where(project => project.Value.Any(read => Under(path, read))).Select(project => project.Key).ToList(); + if (reached.Count == 0) return new("full", [], $"{path} reaches no project"); + suite.UnionWith(reached); + } + else + return new("full", [], @class == Table.Code ? $"{path} is in no solution project under tools/ or test/" : $"{path} is in the class {@class.Name}"); + } + return new("reach", [.. suite]); + } + + static string? Owner(Dictionary projects, string path) => + projects.Keys.Where(project => Under(path, DirectoryOf(project))).MaxBy(project => project.Length); + + // The solution's projects under tools/ and test/, each with the paths it reads; null, and why, when a file cannot be read. + static (Dictionary?, string?) Read(string root) + { + List solution; + try + { + solution = [.. XDocument.Load(Path.Combine(root, "CodoMetis.TypeKit.slnx")).Descendants("Project") + .Select(project => Normalize((string?)project.Attribute("Path") ?? "")) + .Where(path => path.StartsWith("tools/", StringComparison.Ordinal) || path.StartsWith("test/", StringComparison.Ordinal))]; + } + catch (Exception e) when (e is IOException or UnauthorizedAccessException or XmlException) + { + return (null, $"CodoMetis.TypeKit.slnx cannot be read ({e.Message})"); + } + + var reads = new Dictionary(StringComparer.Ordinal); + var projects = new Dictionary(StringComparer.Ordinal); + foreach (var project in solution) + { + var read = Reads(root, project, reads, []); + if (read is null) return (null, $"{project}, or a project it references, cannot be read"); + projects[project] = read; + } + return (projects, null); + } + + // What a project reads: its directory, the paths its project file names outside it, and what each project it references reads. + static string[]? Reads(string root, string project, Dictionary known, HashSet visiting) + { + if (known.TryGetValue(project, out var cached)) return cached; + if (!visiting.Add(project)) return []; + + XDocument document; + try { document = XDocument.Load(Path.Combine(root, project)); } + catch (Exception e) when (e is IOException or UnauthorizedAccessException or XmlException) { return known[project] = null; } + + var directory = DirectoryOf(project); + var read = new SortedSet(StringComparer.Ordinal) { directory }; + foreach (var element in document.Descendants()) + { + var attribute = element.Name.LocalName == "Import" ? "Project" : "Include"; + foreach (var include in ((string?)element.Attribute(attribute) ?? "").Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + if (include.Contains("$(") || include.Contains("@(") || include.Contains("%(") || include.IndexOfAny(['/', '\\']) < 0) continue; + if (Resolve(directory, include) is not { } path) continue; + read.Add(path); + + if (element.Name.LocalName != "ProjectReference") continue; + if (Reads(root, path, known, visiting) is not { } referenced) return known[project] = null; + read.UnionWith(referenced); + } + } + visiting.Remove(project); + return known[project] = [.. read]; + } + + static (string[]?, string?) ProseProjects(string root) + { + try + { + using var filter = JsonDocument.Parse(File.ReadAllText(Path.Combine(root, ProseFilter))); + return ([.. filter.RootElement.GetProperty("solution").GetProperty("projects").EnumerateArray().Select(p => Normalize(p.GetString()!))], null); + } + catch (Exception e) when (e is IOException or UnauthorizedAccessException or JsonException or KeyNotFoundException or InvalidOperationException) + { + return (null, $"{ProseFilter} cannot be read ({e.Message})"); + } + } + + // A path an include names, relative to the root, up to its first wildcard segment; null when it leaves the repository. + static string? Resolve(string directory, string include) + { + var parts = directory.Length == 0 ? new List() : [.. directory.Split('/')]; + foreach (var segment in include.Replace('\\', '/').Split('/')) + { + if (segment.IndexOfAny(['*', '?']) >= 0) break; + if (segment is "" or ".") continue; + if (segment != "..") parts.Add(segment); + else if (parts.Count == 0) return null; + else parts.RemoveAt(parts.Count - 1); + } + return string.Join('/', parts); + } + + static string Normalize(string path) => path.Replace('\\', '/'); + + static string DirectoryOf(string project) => project.LastIndexOf('/') is var slash and >= 0 ? project[..slash] : ""; + + static bool Under(string path, string read) => + read.Length == 0 || path == read || path.StartsWith(read + "/", StringComparison.Ordinal); +} + +static partial class Gate { // Every job of dotnet.yml the gate needs, and the lane that selects it: null for the changes job, which // selects, and Linux for the test matrix, which runs in every selection. @@ -294,14 +467,23 @@ public static int Run(string file, Dictionary options) var legs = Legs(changes.Outputs.GetValueOrDefault("legs")); var pack = changes.Outputs.GetValueOrDefault("pack"); var suite = changes.Outputs.GetValueOrDefault("suite"); + var projects = Projects(changes.Outputs.GetValueOrDefault("projects")); if (legs is null) problems.Add($"The changes job's legs output is {Quote(changes.Outputs.GetValueOrDefault("legs"))}, not one of the three matrices ci-lanes writes."); if (pack is not ("true" or "false")) problems.Add($"The changes job's pack output is {Quote(pack)}, not 'true' or 'false'."); - if (suite is not ("prose" or "full")) - problems.Add($"The changes job's suite output is {Quote(suite)}, not 'prose' or 'full'."); + if (projects is null) + problems.Add($"The changes job's projects output is {Quote(changes.Outputs.GetValueOrDefault("projects"))}, not a JSON array of project files."); + if (suite is not ("prose" or "reach" or "full")) + problems.Add($"The changes job's suite output is {Quote(suite)}, not 'prose', 'reach' or 'full'."); else if (suite == "prose" && (legs != Lane.Linux || pack != "false")) problems.Add("The changes job chose the prose suite for a run with more than the Linux leg: only a prose-only change runs it."); + else if (suite == "reach" && (legs is not (Lane.Linux or (Lane.Linux | Lane.Windows)) || pack != "false")) + problems.Add("The changes job chose the reached suite for a run with the pack or the macOS leg: only a change to tools, tests, prose or the ignore list runs it."); + if (projects is not null && suite == "reach" && (projects.Length == 0 || projects.Distinct(StringComparer.Ordinal).Count() != projects.Length || !projects.All(ProjectFile().IsMatch))) + problems.Add($"The changes job chose the reached suite with the projects {changes.Outputs["projects"]}, not one or more distinct project files under tools/ or test/: a filter of nothing builds and tests nothing, and passes."); + if (projects is { Length: > 0 } && suite is "prose" or "full") + problems.Add($"The changes job named projects for the {suite} suite, which tests a filter of its own."); if (legs is not null && pack is ("true" or "false")) lanes = legs.Value | (pack == "true" ? Lane.Pack : 0); } @@ -356,6 +538,17 @@ public static int Run(string file, Dictionary options) return null; } + // The projects output: a JSON array of strings; null for anything else. + static string[]? Projects(string? projects) + { + try { return projects is null ? null : JsonSerializer.Deserialize(projects); } + catch (JsonException) { return null; } + } + + // tools//.csproj or test//.csproj: one directory deep, no `..`. + [GeneratedRegex(@"^(?:tools|test)/(?!\.\.?/)[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+\.csproj$")] + private static partial Regex ProjectFile(); + static string Text(JsonElement element, string name) => element.ValueKind == JsonValueKind.Object && element.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String ? value.GetString()! : "";