diff --git a/.github/ci-evidence.json b/.github/ci-evidence.json index ef0cc19..653e3eb 100644 --- a/.github/ci-evidence.json +++ b/.github/ci-evidence.json @@ -5,6 +5,7 @@ "inputs": [ "dev.json", "scripts/dev/check.sh", + "scripts/dev/changed_checks.py", "lib/ci_evidence.py", "templates/check.sh", "lib/generated_docs.py", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aeadbc7..9b32c08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,9 @@ name: CI # The check list is dev.json, run by scripts/dev/check.sh (the kit's own vendored runner). The two # jobs keep their names, which branch protection requires: `lint` runs the cheap steps, `tests` the -# kit's test run. +# kit's test run. A pull request runs, of each job's steps, those `check.sh --changed` selects +# against the base branch (all of them when a path is unmapped or a selection input changed); a push +# to main and a dispatch run every step, and only they record the evidence a release takes. on: push: @@ -20,17 +22,26 @@ jobs: name: lint runs-on: ubuntu-latest steps: + # The whole history, so that --changed finds the merge base with the base branch. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + fetch-depth: 0 persist-credentials: false # Whitespace; no machine-local path in tracked text; CLAUDE.md is AGENTS.md and the root # copy starts with the template; every script parses; no block of code is duplicated over the # thresholds of jpdev.json; the JSON documents parse; shellcheck (the program the runner image # carries). - name: The lint steps of dev.json + env: + EVENT: ${{ github.event_name }} + BASE_REF: ${{ github.base_ref }} run: | set -o pipefail - scripts/dev/check.sh --all --only diff-check,local-paths,agent-docs,bash-n,duplication,json,shellcheck | tee "${{ runner.temp }}/checks.log" + if [ "$EVENT" = pull_request ]; then + scripts/dev/check.sh --all --changed "origin/$BASE_REF" --skip tests | tee "${{ runner.temp }}/checks.log" + else + scripts/dev/check.sh --all --skip tests | tee "${{ runner.temp }}/checks.log" + fi - name: Record exact CI inputs and coverage if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' @@ -50,11 +61,19 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + fetch-depth: 0 persist-credentials: false - name: The kit's tests (jpdev check and check.sh on the fixture, the caps, the record gate) + env: + EVENT: ${{ github.event_name }} + BASE_REF: ${{ github.base_ref }} run: | set -o pipefail - scripts/dev/check.sh --only tests | tee "${{ runner.temp }}/checks.log" + if [ "$EVENT" = pull_request ]; then + scripts/dev/check.sh --changed "origin/$BASE_REF" --skip diff-check,local-paths,agent-docs,bash-n,duplication,json,shellcheck | tee "${{ runner.temp }}/checks.log" + else + scripts/dev/check.sh --only tests | tee "${{ runner.temp }}/checks.log" + fi - name: Record exact CI inputs and coverage if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b7adda6..d976da2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -168,6 +168,10 @@ jobs: want="# from judgment-pack-devkit ${RELEASE_TAG} (lib/handover_check.sh)" [ "${line}" = "${want}" ] || { echo "scripts/dev/handover_check.sh line 2 is '${line}'; the tag ${RELEASE_TAG} needs '${want}'" >&2; exit 1; } cmp -s <(tail -n +2 lib/handover_check.sh) <(tail -n +3 scripts/dev/handover_check.sh) || { echo "scripts/dev/handover_check.sh is not a versioned copy of lib/handover_check.sh" >&2; exit 1; } + line="$(sed -n 2p scripts/dev/changed_checks.py)" + want="# from judgment-pack-devkit ${RELEASE_TAG} (lib/changed_checks.py)" + [ "${line}" = "${want}" ] || { echo "scripts/dev/changed_checks.py line 2 is '${line}'; the tag ${RELEASE_TAG} needs '${want}'" >&2; exit 1; } + cmp -s <(head -n 1 lib/changed_checks.py; tail -n +2 lib/changed_checks.py) <(sed 2d scripts/dev/changed_checks.py) || { echo "scripts/dev/changed_checks.py is not a versioned copy of lib/changed_checks.py" >&2; exit 1; } echo "version=${version}" >> "${GITHUB_OUTPUT}" # git archive writes the named paths of the commit, with the commit's time and each file's @@ -198,7 +202,7 @@ jobs: kit="${unpack}/judgment-pack-devkit-${VERSION}" [ "$("${kit}/bin/jpdev" --version)" = "jpdev ${VERSION}" ] || { echo "the packaged jpdev does not report ${VERSION}" >&2; exit 1; } "${kit}/bin/jpdev" env > /dev/null - for path in bin/jpdev lib/wrap.sh lib/handover_check.sh templates/check.sh templates/dev.schema.json scripts/dev/handover_check.sh scripts/sync-agent-docs.sh examples/judgment-pack-agent.dev.json LICENSE NOTICE README.md; do + for path in bin/jpdev lib/wrap.sh lib/handover_check.sh templates/check.sh templates/dev.schema.json scripts/dev/handover_check.sh scripts/dev/changed_checks.py scripts/sync-agent-docs.sh examples/judgment-pack-agent.dev.json LICENSE NOTICE README.md; do test -e "${kit}/${path}" || { echo "the tarball does not hold ${path}" >&2; exit 1; } done diff --git a/AGENTS.md b/AGENTS.md index bad39ac..2cac66e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -99,7 +99,8 @@ summary block, and anything you stopped on or chose not to do. - `scripts/dev/check.sh` is a copy of `templates/check.sh`; after a change to the template, copy it again. Line 2 of the template names the kit version (`# from judgment-pack-devkit vX.Y.Z (templates/check.sh)`) and moves with `JPDEV_VERSION` in `lib/common.sh` at a release; a test - holds the two together. + holds the two together. `scripts/dev/changed_checks.py` is `lib/changed_checks.py` with the + same version on line 2; after a change to the library file, copy it again. - This file is `templates/AGENTS.md` plus this section. Change a shared rule in the template and here together, then run `scripts/sync-agent-docs.sh .` for `CLAUDE.md`. - Machine settings (paths, caps, name lists) never live in this repository. They are read from the diff --git a/CLAUDE.md b/CLAUDE.md index bad39ac..2cac66e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -99,7 +99,8 @@ summary block, and anything you stopped on or chose not to do. - `scripts/dev/check.sh` is a copy of `templates/check.sh`; after a change to the template, copy it again. Line 2 of the template names the kit version (`# from judgment-pack-devkit vX.Y.Z (templates/check.sh)`) and moves with `JPDEV_VERSION` in `lib/common.sh` at a release; a test - holds the two together. + holds the two together. `scripts/dev/changed_checks.py` is `lib/changed_checks.py` with the + same version on line 2; after a change to the library file, copy it again. - This file is `templates/AGENTS.md` plus this section. Change a shared rule in the template and here together, then run `scripts/sync-agent-docs.sh .` for `CLAUDE.md`. - Machine settings (paths, caps, name lists) never live in this repository. They are read from the diff --git a/README.md b/README.md index abd20a8..40f3d8e 100644 --- a/README.md +++ b/README.md @@ -101,8 +101,10 @@ the same name overrides it. `config/env.example` documents each setting with its working tree (deletions, both sides of a rename, and submodules whatever their `ignore` setting), plus untracked files that are not ignored. `BASE` defaults to the default branch: `origin/HEAD`, else `origin/main`, else `origin/master`. The whole list runs when a changed path matches no rule, - when `jpdev.json`, the manifest or `scripts/dev/check.sh` changed, when nothing changed, or when - there is no mapping. A block before the summary names each check selected with the globs that + when `jpdev.json`, the manifest, `scripts/dev/check.sh` or `scripts/dev/changed_checks.py` + changed, when nothing changed, or when there is no mapping. The vendored + `scripts/dev/check.sh --changed [BASE]` makes the same selection with the vendored + `scripts/dev/changed_checks.py`, where the kit is not installed and in CI on pull requests. A block before the summary names each check selected with the globs that selected it and each check not selected, or why the whole list runs. An invalid mapping, a `BASE` that names no commit, a missing default branch or merge base, and `--changed` with `--only` are refused with exit 2 before any check runs. @@ -182,6 +184,10 @@ the same name overrides it. `config/env.example` documents each setting with its command and is ignored; the product term “adversarial” is not in the base list. `codex run` runs the same check first and does not start the run on a refusal. - `jpdev env`: the effective settings and where each came from. +- `jpdev --version` prints the release version (`jpdev X.Y.Z`); `jpdev version` adds, when the kit is + a git checkout, the commit it is at (`jpdev X.Y.Z (vX.Y.Z-4-g1f02688)`), which tells a kit at a + release from one past it. An option `jpdev check` does not know, and a refused `--changed` + selection, name the same. ## The duplication check @@ -236,8 +242,8 @@ duplication: 1 duplicate pair (at least 70 tokens over at least 6 lines, 214 fil The kit runs the check on itself with `python3 lib/duplication.py`, over its Python and shell files at 100 tokens and 10 lines, and leaves out `scripts/dev/**`: those files are the vendored -copies of `templates/check.sh`, `lib/generated_docs.py` and `lib/handover_check.sh`, kept the -same on purpose. +copies of `templates/check.sh`, `lib/generated_docs.py`, `lib/handover_check.sh` and +`lib/changed_checks.py`, kept the same on purpose. The exit status is 0 when nothing is found or the check is off, 1 when a pair or an ignore marker problem is found, and 2 when the declaration is refused or the files in scope do not fit in @@ -270,8 +276,9 @@ Python steps, and for other steps that invoke `python3`, name the interpreter us - `templates/dev.json`, `templates/dev.schema.json`: the manifest by example and its JSON Schema. - `templates/jpdev.json`, `templates/jpdev.schema.json`: the generated-document declaration, the changed-path mapping and the duplication thresholds by example, and their JSON Schema. -- `templates/check.sh` and `lib/handover_check.sh`: the public checks a repository vendors as - `scripts/dev/check.sh` and `scripts/dev/handover_check.sh`. +- `templates/check.sh`, `lib/handover_check.sh` and `lib/changed_checks.py`: the public checks a + repository vendors as `scripts/dev/check.sh`, `scripts/dev/handover_check.sh` and + `scripts/dev/changed_checks.py`. - `templates/codex-brief.md`, `templates/codex-review-brief.md`: the build and review brief skeletons `jpdev codex brief-template` prints. - `templates/AGENTS.md`, `templates/CLAUDE.md`: the agent instructions (one text, generated from diff --git a/bin/jpdev b/bin/jpdev index 92a904b..7cec67a 100755 --- a/bin/jpdev +++ b/bin/jpdev @@ -63,9 +63,27 @@ usage: jpdev COMMAND [ARGS] neutral phrasing; require the handover heading for a build brief env print the effective settings and where each came from --version print the version + version print the version and, when the kit is a git checkout, the commit it is at + (git describe), which tells a kit at a release from one past it USAGE } +# kit_version prints the version, with the kit checkout's git describe when the kit is its own clone. +# A repository named by the environment (GIT_DIR under a hook, for one) is never taken for the kit's. +kit_version() { + local top described="" + top=$(kit_git rev-parse --show-toplevel 2>/dev/null) || top="" + if [ -n "$top" ] && [ "$top" -ef "$kit" ]; then + described=$(kit_git describe --tags --always 2>/dev/null) || described="" + fi + printf 'jpdev %s%s\n' "$JPDEV_VERSION" "${described:+ ($described)}" +} + +kit_git() { + env -u GIT_DIR -u GIT_WORK_TREE -u GIT_COMMON_DIR -u GIT_INDEX_FILE -u GIT_OBJECT_DIRECTORY \ + -u GIT_ALTERNATE_OBJECT_DIRECTORIES -u GIT_NAMESPACE git -C "$kit" "$@" +} + need_value() { [ $# -ge 2 ] || jpdev_die "$1 needs a value"; } repo_root() { @@ -85,7 +103,7 @@ cmd_check() { if [ $# -gt 0 ] && [[ $1 != -* ]]; then base_args=(--base "$1"); shift; fi ;; -h | --help) usage; exit 0 ;; - *) jpdev_die "check: unknown argument $1" ;; + *) jpdev_die "check: unknown argument $1; this kit is $(kit_version), and \`jpdev check --help\` lists what it takes" ;; esac done [ "$changed" -eq 0 ] || [ "$only" -eq 0 ] || jpdev_die "check: --changed and --only both choose the steps; give one" @@ -95,7 +113,8 @@ cmd_check() { if [ "$changed" -eq 1 ]; then # The first line is the selection (whole, or only NAME,NAME); the rest is the report. local selection first - selection=$(python3 -I "$kit/lib/changed_checks.py" --root "$root" --manifest "$manifest" "${base_args[@]}") || exit 2 + selection=$(python3 -I "$kit/lib/changed_checks.py" --root "$root" --manifest "$manifest" "${base_args[@]}") || + jpdev_die "check --changed: refused by this kit, $(kit_version)" first=${selection%%$'\n'*} printf '%s\n' "${selection#*$'\n'}" case $first in @@ -666,7 +685,7 @@ codex_run() { rm -f -- "$dir/msg.txt" "$dir/pr.md" prompt="The rules are this repository's AGENTS.md; follow them." if [ "$small" -eq 1 ]; then - prompt+=$'\nThis is a small fix: run only the check-list steps the change touches with `jpdev check --only `; CI runs the whole list.' + prompt+=$'\nThis is a small fix: run `jpdev check --changed`, which runs only the check-list steps jpdev.json maps the changed paths to (the whole list when a path is unmapped); do not choose the steps yourself. CI checks the pull request the same way and runs the whole list on the default branch.' fi prompt+=$'\n'"$(cat "$brief")" local args=(exec -m "$model" -c "model_reasoning_effort=$effort" --sandbox "$sandbox") @@ -1049,7 +1068,8 @@ case ${1:-} in carry) shift; cmd_carry "$@" ;; codex) shift; cmd_codex "$@" ;; env) shift; cmd_env "$@" ;; - --version | version) echo "jpdev $JPDEV_VERSION" ;; + --version) echo "jpdev $JPDEV_VERSION" ;; + version) kit_version ;; -h | --help | help | "") usage; [ -n "${1:-}" ] || exit 2 ;; *) echo "jpdev: unknown command '$1'" >&2; usage >&2; exit 2 ;; esac diff --git a/docs/adopting.md b/docs/adopting.md index 5bed692..2d4e0cb 100644 --- a/docs/adopting.md +++ b/docs/adopting.md @@ -56,6 +56,7 @@ mkdir -p scripts/dev devkit=path/to/judgment-pack-devkit cp "$devkit/templates/check.sh" scripts/dev/check.sh cp "$devkit/scripts/dev/handover_check.sh" scripts/dev/handover_check.sh +cp "$devkit/scripts/dev/changed_checks.py" scripts/dev/changed_checks.py ``` Line 2 of the template, `# from judgment-pack-devkit vX.Y.Z (templates/check.sh)`, records the kit @@ -63,10 +64,16 @@ release the copy came from. A vendored copy keeps that line, so a later reader c it is behind. The kit's `scripts/dev/handover_check.sh` similarly vendors `lib/handover_check.sh` with its release on line 2. Vendor `scripts/dev/check.sh` and `scripts/dev/handover_check.sh` together, and copy both whole files again when updating them. `jpdev setup` names this pair when -either file is absent. - -`scripts/dev/check.sh [--setup] [--only A,B] [--skip A,B] [--all]` runs the manifest with no -caps and prints the summary block. Before a setup or check run, it creates a missing declared +either file is absent. `scripts/dev/changed_checks.py`, the selection code of `--changed`, is +`lib/changed_checks.py` with the release on line 2; vendor it beside `check.sh` and update the two +together (`check.sh --changed` refuses when it is absent, and the CI template uses `--changed` on +pull requests). A runner given an option it does not know names the release on its line 2, and +`jpdev version` prints the installed kit's version with the commit a kit checkout is at. + +`scripts/dev/check.sh [--setup] [--only A,B | --changed [BASE]] [--skip A,B] [--all]` runs the +manifest with no caps and prints the summary block. `--changed` selects the checks as +`jpdev check --changed` does (see [changed-path check selection](#changed-path-check-selection)), +with the same rules and the same report, through the vendored `scripts/dev/changed_checks.py`. Before a setup or check run, it creates a missing declared Python environment with `python3.X` on PATH, then the newest `~/.pyenv/versions/3.X.*/bin/python3`; it refuses before any step when neither exists. Every step runs with that environment's `bin` first on PATH, so the manifest's Python setup step installs its @@ -100,8 +107,21 @@ regenerates `CLAUDE.md`. ## 4. Add the CI workflow Copy `templates/ci-check.yml` to `.github/workflows/ci.yml`. It reads the toolchain -versions from `dev.json`, sets the toolchains up, and runs `scripts/dev/check.sh --setup` and -`scripts/dev/check.sh --all`. Its handover job rejects removals from boundary paths unless the pull +versions from `dev.json`, sets the toolchains up, and runs `scripts/dev/check.sh --setup` and the +checks: on a pull request `scripts/dev/check.sh --all --changed origin/`, on a push to +the default branch and on a dispatch `scripts/dev/check.sh --all`. The pull request runs the checks +`jpdev.json` maps its changed paths to, and the whole list when a path is unmapped, when the +mapping, the manifest or the runner changed, or when there is no mapping, so a repository without +a `changed` mapping runs the whole list on every pull request as before. The default branch always +runs the whole list, and only that run is the evidence a release takes (see +[trusted CI evidence](ci-evidence.md)); a check a pull request did not select did not pass there, +and the default-branch run is where a mapping that missed an effect shows. This is the +[policy](policy.md#testing)'s split: CI owns every deterministic check, the scoped run is allowed on +pull requests, and the full list is required on the default branch and before a release. The check +job checks out the whole history so that the merge base with the base branch is found. When the +base cannot be resolved (a missing or empty base branch name, or no merge base in the history), +the pull-request job fails with the refusal before any check runs rather than falling back to the +whole list; a push to the default branch runs the whole list whatever the mapping says. Its handover job rejects removals from boundary paths unless the pull request body contains `Boundary change: yes`, then runs `handover.schema_probe` when present through the vendored `scripts/dev/handover_check.sh`. Copy `templates/update-branches.yml` to `.github/workflows/update-branches.yml` to update open branches @@ -219,7 +239,7 @@ message that names the member or rule at fault, before any check runs. `templates/jpdev.schema.json` documents the same shape and `templates/jpdev.json` holds an example. The whole list runs when a changed path matches no rule, when `jpdev.json`, the manifest (as named -or as a symlink resolves) or `scripts/dev/check.sh` changed, when nothing changed, and when there is +or as a symlink resolves), `scripts/dev/check.sh` or `scripts/dev/changed_checks.py` changed, when nothing changed, and when there is no mapping (no `jpdev.json`, or one without `changed`). Map only paths whose effect on the checks is known, and put the checks that read every file (whitespace, path and name checks) in every rule. Selection sees only what git reports in this clone: an edit hidden by assume-unchanged or diff --git a/docs/ci-evidence.md b/docs/ci-evidence.md index c9ef61f..bdeba53 100644 --- a/docs/ci-evidence.md +++ b/docs/ci-evidence.md @@ -103,7 +103,9 @@ PR or another repository is ineligible. Unavailable evidence never becomes succe No workflow automatically dispatches/retries, and no automatic full CI is disabled. The unchanged `lint`/`tests` and `contract`/`check` job names retain branch-protection -statuses. Main/PR CI still runs full deterministic checks. Release now verifies +statuses. Main CI still runs the full deterministic checks; PR CI runs the checks its +changed-path mapping selects (`check.sh --changed`, the whole list when a path is unmapped) and +records no evidence. Release now verifies that the required full deterministic work was completed at the exact source and compatible inputs, rather than executing those equivalent checks again. Review, exact-head CI and publication approval requirements remain. diff --git a/docs/policy.md b/docs/policy.md index bb5452a..6b6222c 100644 --- a/docs/policy.md +++ b/docs/policy.md @@ -8,7 +8,9 @@ are the ones a brief does not repeat. ## Who owns what - **CI owns everything deterministic.** Formatting, builds, unit tests, schema checks, - certification suites, whitespace and path checks run in CI on every pull request. A session + certification suites, whitespace and path checks run in CI: on a pull request those its + changed-path mapping selects (the whole list without one), and the whole list on the default + branch. A session does not re-run them to reassure itself and does not watch CI run. - **Duplication is where two copies drift:** a repository opts in to the `duplication` check with its own token and line thresholds in `jpdev.json`, and marks an intentional copy with an explicit @@ -112,7 +114,8 @@ pull request before review, MEDIUMs named by the review of record are fixed in t MEDIUM found only on re-read goes to the pre-release batch, and LOW rides. `jpdev codex run --small` is for a bounded fix on an existing branch. It tells the builder to run -only the touched check-list steps; CI runs the whole list. The wrapper refuses the result after the +`jpdev check --changed`, so the mapping and not the builder chooses the steps; CI checks the pull +request the same way and runs the whole list on the default branch. The wrapper refuses the result after the run if it changed a manifest `handover.shared_paths` or `handover.boundary_paths` entry, or if its added and removed lines exceed `JPDEV_SMALL_MAX_LINES` (80 by default). The size/path guard is unchanged by the batch workflow. A relevant CI failure gets a focused fix and rerun; reuse the builder unless the safeguard-stop rule below requires a fresh session. diff --git a/docs/releasing.md b/docs/releasing.md index a4dceff..ec9aa7a 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -47,7 +47,9 @@ the second. copies the template to `scripts/dev/check.sh`; sets line 2 of `scripts/dev/handover_check.sh` to `# from judgment-pack-devkit v (lib/handover_check.sh)` and copies the rest from - `lib/handover_check.sh` (tests hold both vendored scripts to their sources and version); + `lib/handover_check.sh`; sets line 2 of `scripts/dev/changed_checks.py` to + `# from judgment-pack-devkit v (lib/changed_checks.py)`, the other lines being those + of `lib/changed_checks.py` (tests hold the vendored scripts to their sources and version); - writes `docs/releases/.md`: what the release holds and what it does not. It is the text the release page carries, and the first line ends with the tag's name, for example `# jpdev v0.2.0`. @@ -94,7 +96,7 @@ Every job works on the commit the run was started for, by its digest, not on the validation keep every existing check and required status. The gate document records API limits and measured baseline durations; candidate savings are not yet measured. 3. **Packages the kit.** It refuses unless `bin/jpdev --version` prints `jpdev `, line 2 of - each vendored script names ``, and both scripts match their sources. Then + each vendored script names ``, and each matches its source. Then `git archive` of the commit into `judgment-pack-devkit-.tgz`, `checksums.txt` written over it and checked, and the unpacked tarball run once (`jpdev --version`, `jpdev env`). 4. **Attests and drafts.** Only if the tag still names the commit and no release exists under the diff --git a/lib/changed_checks.py b/lib/changed_checks.py index a00030d..bb292ee 100644 --- a/lib/changed_checks.py +++ b/lib/changed_checks.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 -"""changed_checks.py --root ROOT --manifest MANIFEST [--base BASE] +"""changed_checks.py --root ROOT --manifest MANIFEST [--base BASE] [--label TEXT] Selects the check-list steps that the changed paths of a clone can affect, for -`jpdev check --changed [BASE]`. The changed paths are those that differ from the merge base of +`jpdev check --changed [BASE]` and `scripts/dev/check.sh --changed [BASE]`. A repository vendors +this file as scripts/dev/changed_checks.py beside scripts/dev/check.sh. The changed paths are those that differ from the merge base of BASE and HEAD in HEAD, in the index or in the working tree (deletions, both sides of a rename, and submodules whatever their ignore setting), plus untracked files that are not ignored. BASE defaults to the default branch: origin/HEAD, else origin/main, else origin/master. Only what git @@ -13,11 +14,12 @@ names they select. A glob is matched against the whole repository-relative path; `*` and `?` stay within one path segment and a `**` segment matches any number of segments. The whole list runs when a changed path matches no rule, when a selection input changed (the mapping file, -the manifest, scripts/dev/check.sh), when nothing changed, or when there is no mapping. +the manifest, scripts/dev/check.sh, scripts/dev/changed_checks.py), when nothing changed, or when +there is no mapping. The first line of the output is the selection, `whole` or `only NAME,NAME`; the lines after it -are the report jpdev check prints. Exit 0, or 2 with one line naming the refusal. Standard -library only. +are the report the runner prints, headed and refused with --label (default `jpdev check +--changed`). Exit 0, or 2 with one line naming the refusal. Standard library only. """ import argparse import json @@ -174,7 +176,7 @@ def changed_paths(root, merge_base): def selection_inputs(root, manifest): """The repository paths whose change changes the selection itself.""" - inputs = [CONFIG, 'scripts/dev/check.sh'] + inputs = [CONFIG, 'scripts/dev/check.sh', 'scripts/dev/changed_checks.py'] # The manifest as named (a symlink retargeted changes the link) and as resolved. for path in (os.path.abspath(manifest), os.path.realpath(manifest)): for top in (str(root), os.path.realpath(str(root))): @@ -187,7 +189,7 @@ def selection_inputs(root, manifest): return inputs -def select(root, manifest, base=None): +def select(root, manifest, base=None, runner='jpdev check --changed'): """Return (selected names or None for the whole list, report lines).""" status, top = git(root, 'rev-parse', '--show-toplevel') if status: @@ -195,11 +197,11 @@ def select(root, manifest, base=None): root = Path(top.decode('utf-8', 'surrogateescape').strip()) steps = step_names(manifest) rules, no_mapping = load_rules(root, steps, os.path.basename(manifest)) - label, merge_base = resolve_base(root, base) + base_label, merge_base = resolve_base(root, base) paths = changed_paths(root, merge_base) short = git(root, 'rev-parse', '--short', merge_base)[1].decode().strip() or merge_base[:12] noun = 'path' if len(paths) == 1 else 'paths' - report = ['== jpdev check --changed: %d changed %s since %s (merge base %s) ==' % (len(paths), noun, label, short)] + report = ['== %s: %d changed %s since %s (merge base %s) ==' % (runner, len(paths), noun, base_label, short)] why = [] changed_inputs = [p for p in selection_inputs(root, manifest) if p in paths] if changed_inputs: @@ -251,11 +253,12 @@ def main(): parser.add_argument('--root', required=True) parser.add_argument('--manifest', required=True) parser.add_argument('--base') + parser.add_argument('--label', default='jpdev check --changed') args = parser.parse_args() try: - selected, report = select(args.root, args.manifest, args.base) + selected, report = select(args.root, args.manifest, args.base, args.label) except Refused as error: - print('jpdev check --changed: refused: %s' % error, file=sys.stderr) + print('%s: refused: %s' % (args.label, error), file=sys.stderr) return 2 print('whole' if selected is None else 'only ' + ','.join(selected)) print('\n'.join(report)) diff --git a/scripts/dev/changed_checks.py b/scripts/dev/changed_checks.py new file mode 100644 index 0000000..559fc8a --- /dev/null +++ b/scripts/dev/changed_checks.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +# from judgment-pack-devkit v0.7.0 (lib/changed_checks.py) +"""changed_checks.py --root ROOT --manifest MANIFEST [--base BASE] [--label TEXT] + +Selects the check-list steps that the changed paths of a clone can affect, for +`jpdev check --changed [BASE]` and `scripts/dev/check.sh --changed [BASE]`. A repository vendors +this file as scripts/dev/changed_checks.py beside scripts/dev/check.sh. The changed paths are those that differ from the merge base of +BASE and HEAD in HEAD, in the index or in the working tree (deletions, both sides of a rename, and +submodules whatever their ignore setting), plus untracked files that are not ignored. BASE +defaults to the default branch: origin/HEAD, else origin/main, else origin/master. Only what git +reports is seen: an edit hidden by assume-unchanged or skip-worktree, an ignored file and a path +outside the clone are not. + +The mapping is the `changed` member of jpdev.json at the root: rules of path globs and the check +names they select. A glob is matched against the whole repository-relative path; `*` and +`?` stay within one path segment and a `**` segment matches any number of segments. The whole +list runs when a changed path matches no rule, when a selection input changed (the mapping file, +the manifest, scripts/dev/check.sh, scripts/dev/changed_checks.py), when nothing changed, or when +there is no mapping. + +The first line of the output is the selection, `whole` or `only NAME,NAME`; the lines after it +are the report the runner prints, headed and refused with --label (default `jpdev check +--changed`). Exit 0, or 2 with one line naming the refusal. Standard library only. +""" +import argparse +import json +import os +from pathlib import Path +import re +import subprocess +import sys + +CONFIG = 'jpdev.json' +# Every diff names paths from the top, counts both sides of a rename, runs no external diff, and +# reports a submodule whatever its ignore setting in .gitmodules or the git configuration. +DIFF = ('diff', '--name-only', '-z', '--no-renames', '--no-relative', '--no-ext-diff', '--ignore-submodules=none') +# `generators` is read by lib/generated_docs.py and `duplication` by lib/duplication.py; this reads `changed`. +MEMBERS = {'version', 'generators', 'changed', 'duplication'} +RULE_MEMBERS = {'paths', 'checks'} +UNSUPPORTED = '\\[]{}' +SHOWN = 20 + + +class Refused(Exception): + pass + + +def glob_regex(glob): + """Compile a repository-relative path glob: * and ? within a segment, ** as a whole segment.""" + if not isinstance(glob, str) or not glob: + raise Refused('a glob must be a non-empty string') + bad = [c for c in UNSUPPORTED if c in glob] + if bad: + raise Refused('glob %r: %s is not supported (only *, ? and a ** segment are)' % (glob, ' '.join(bad))) + if glob.startswith('/'): + raise Refused('glob %r: must be relative to the repository root' % glob) + parts = glob.split('/') + out = [] + for index, part in enumerate(parts): + last = index == len(parts) - 1 + if part in ('', '.', '..'): + raise Refused('glob %r: has an empty, . or .. segment' % glob) + if part == '**': + out.append('.*' if last else '(?:[^/]+/)*') + continue + if '**' in part: + raise Refused('glob %r: ** must be a whole segment' % glob) + segment = ''.join('[^/]*' if c == '*' else '[^/]' if c == '?' else re.escape(c) for c in part) + out.append(segment if last else segment + '/') + return re.compile(''.join(out), re.DOTALL) + + +def step_names(manifest): + """The names check.sh runs, in its order: the checks, then parity when the manifest has one.""" + try: + with open(manifest, encoding='utf-8') as handle: + value = json.load(handle) + except (OSError, ValueError) as error: + raise Refused('cannot read the manifest %s: %s' % (manifest, error)) + names = [step.get('name') for step in value.get('checks', []) if isinstance(step, dict)] + if isinstance(value.get('parity'), dict): + names.append('parity') + return [name for name in names if isinstance(name, str)] + + +def load_rules(root, steps, manifest_label, config=CONFIG): + """Return (rules, None), or (None, why the whole list runs) when there is no mapping.""" + path = root / config + if not path.is_file(): + return None, 'no %s' % config + try: + value = json.loads(path.read_text(encoding='utf-8')) + except (OSError, ValueError) as error: + raise Refused('%s: not readable JSON: %s' % (config, error)) + if not isinstance(value, dict): + raise Refused('%s: not a JSON object' % config) + unknown = sorted(set(value) - MEMBERS) + if unknown: + raise Refused('%s: unknown member %s' % (config, ', '.join(repr(k) for k in unknown))) + if type(value.get('version')) is not int or value['version'] != 1: + raise Refused('%s: version must be 1' % config) + if 'changed' not in value: + return None, 'no change mapping in %s' % config + rules = value['changed'] + if not isinstance(rules, list) or not rules: + raise Refused('%s changed: must be a non-empty array of rules' % config) + compiled = [] + for index, rule in enumerate(rules): + where = '%s changed[%d]' % (config, index) + if not isinstance(rule, dict) or set(rule) != RULE_MEMBERS: + raise Refused('%s: a rule has exactly the members paths and checks' % where) + for key in ('paths', 'checks'): + items = rule[key] + if not isinstance(items, list) or not items or not all(isinstance(i, str) and i for i in items): + raise Refused('%s.%s: must be a non-empty array of non-empty strings' % (where, key)) + if len(set(items)) != len(items): + raise Refused('%s.%s: names a value twice' % (where, key)) + globs = [] + for glob in rule['paths']: + try: + globs.append((glob, glob_regex(glob))) + except Refused as error: + raise Refused('%s.paths: %s' % (where, error)) + for name in rule['checks']: + if name not in steps: + raise Refused('%s.checks: no check named %r in %s' % (where, name, manifest_label)) + compiled.append((globs, rule['checks'])) + return compiled, None + + +def git(root, *args): + result = subprocess.run(['git', '-C', str(root)] + list(args), stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + return result.returncode, result.stdout + + +def resolve_base(root, base): + """Return (label, merge base commit) for BASE, or for the default branch when BASE is None.""" + if base is None: + status, out = git(root, 'symbolic-ref', '-q', 'refs/remotes/origin/HEAD') + refs = [out.decode('utf-8', 'replace').strip()] if status == 0 else [] + refs += ['refs/remotes/origin/main', 'refs/remotes/origin/master'] + found = [ref for ref in refs if git(root, 'rev-parse', '--verify', '-q', ref + '^{commit}')[0] == 0] + if not found: + raise Refused('no default branch found (origin/HEAD, origin/main or origin/master); name BASE') + ref = found[0] + label = ref[len('refs/remotes/'):] if ref.startswith('refs/remotes/') else ref + else: + if not base or base.startswith('-'): + raise Refused('BASE %r is not a revision' % base) + ref = label = base + status, out = git(root, 'rev-parse', '--verify', '-q', ref + '^{commit}') + if status: + raise Refused('cannot resolve BASE %r to a commit' % label) + commit = out.decode().strip() + status, out = git(root, 'merge-base', commit, 'HEAD') + if status: + raise Refused('no merge base of %s and HEAD (a shallow clone needs the history)' % label) + return label, out.decode().strip() + + +def changed_paths(root, merge_base): + """The paths HEAD, the index or the working tree changed since the merge base, and untracked.""" + raw = set() + for side, args in (('the working tree', (merge_base,)), ('the index', ('--cached', merge_base)), + ('HEAD', (merge_base, 'HEAD'))): + status, diff = git(root, *(DIFF + args + ('--',))) + if status: + raise Refused('git diff of %s against the merge base %s failed' % (side, merge_base[:12])) + raw |= set(diff.split(b'\0')) + status, untracked = git(root, 'ls-files', '--others', '--exclude-standard', '--full-name', '-z') + if status: + raise Refused('git ls-files of the untracked files failed') + raw |= set(untracked.split(b'\0')) + return sorted(p.decode('utf-8', 'replace') for p in raw - {b''}) + + +def selection_inputs(root, manifest): + """The repository paths whose change changes the selection itself.""" + inputs = [CONFIG, 'scripts/dev/check.sh', 'scripts/dev/changed_checks.py'] + # The manifest as named (a symlink retargeted changes the link) and as resolved. + for path in (os.path.abspath(manifest), os.path.realpath(manifest)): + for top in (str(root), os.path.realpath(str(root))): + relative = os.path.relpath(path, top) + if relative == os.pardir or relative.startswith(os.pardir + os.sep) or os.path.isabs(relative): + continue + relative = relative.replace(os.sep, '/') + if relative not in inputs: + inputs.append(relative) + return inputs + + +def select(root, manifest, base=None, runner='jpdev check --changed'): + """Return (selected names or None for the whole list, report lines).""" + status, top = git(root, 'rev-parse', '--show-toplevel') + if status: + raise Refused('%s is not inside a git clone' % root) + root = Path(top.decode('utf-8', 'surrogateescape').strip()) + steps = step_names(manifest) + rules, no_mapping = load_rules(root, steps, os.path.basename(manifest)) + base_label, merge_base = resolve_base(root, base) + paths = changed_paths(root, merge_base) + short = git(root, 'rev-parse', '--short', merge_base)[1].decode().strip() or merge_base[:12] + noun = 'path' if len(paths) == 1 else 'paths' + report = ['== %s: %d changed %s since %s (merge base %s) ==' % (runner, len(paths), noun, base_label, short)] + why = [] + changed_inputs = [p for p in selection_inputs(root, manifest) if p in paths] + if changed_inputs: + why = ['selection input changed: %s: whole list' % p for p in changed_inputs] + elif no_mapping: + why = ['%s: whole list' % no_mapping] + elif not paths: + why = ['no changed path: whole list'] + reasons = {} + if not why: + unmapped = [] + for path in paths: + hit = False + for rule, (globs, checks) in enumerate(rules): + matched = [(rule, index) for index, (_, pattern) in enumerate(globs) if pattern.fullmatch(path)] + if not matched: + continue + hit = True + for name in checks: + reasons.setdefault(name, set()).update(matched) + if not hit: + unmapped.append(path) + why = ['unmapped path %s: whole list' % path for path in unmapped[:SHOWN]] + if len(unmapped) > SHOWN: + why.append('... %d more unmapped paths' % (len(unmapped) - SHOWN)) + if not why and not reasons: + why = ['no check selected: whole list'] + if why: + report.extend(why) + report.append('== whole list: %d checks ==' % len(steps)) + return None, report + selected = [name for name in steps if name in reasons] + width = max(len(name) for name in steps) + for name in steps: + if name in reasons: + globs = [] + for rule, index in sorted(reasons[name]): + if rules[rule][0][index][0] not in globs: + globs.append(rules[rule][0][index][0]) + report.append('selected %-*s %s' % (width, name, ', '.join(globs))) + else: + report.append('not selected %-*s no changed path maps to it' % (width, name)) + report.append('== %d of %d checks selected ==' % (len(selected), len(steps))) + return selected, report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument('--root', required=True) + parser.add_argument('--manifest', required=True) + parser.add_argument('--base') + parser.add_argument('--label', default='jpdev check --changed') + args = parser.parse_args() + try: + selected, report = select(args.root, args.manifest, args.base, args.label) + except Refused as error: + print('%s: refused: %s' % (args.label, error), file=sys.stderr) + return 2 + print('whole' if selected is None else 'only ' + ','.join(selected)) + print('\n'.join(report)) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/dev/check.sh b/scripts/dev/check.sh index 7baa18a..8250cdb 100755 --- a/scripts/dev/check.sh +++ b/scripts/dev/check.sh @@ -1,22 +1,48 @@ #!/usr/bin/env bash # from judgment-pack-devkit v0.7.0 (templates/check.sh) -# scripts/dev/check.sh [--setup] [--only a,b] [--skip a,b] [--all] [--manifest FILE] +# scripts/dev/check.sh [--setup] [--only a,b | --changed [BASE]] [--skip a,b] [--all] [--manifest FILE] # Runs the checks of dev.json in order and prints one summary block; exit status = failures. +# --changed runs only the checks jpdev.json maps the paths changed since the merge base of BASE +# (default: the default branch) and HEAD to, with changed_checks.py vendored beside this script; +# the whole list runs when a changed path is unmapped, when jpdev.json, the manifest, this script +# or changed_checks.py changed, when nothing changed, or with no mapping. # Stops at the first failure unless --all. --setup runs the manifest's setup steps instead. # A step whose requires_env variable is empty is skipped, and said to be; when CI is set that is a # failure unless the step is marked optional: true. Standard library python3 only. jpdev sets JPDEV_WRAP to add its caps around each step. root=${JPDEV_ROOT:-$(cd "$(dirname "$0")" && git rev-parse --show-toplevel)} || exit 2 -exec python3 -I - "$root" "$@" <<'PY' +exec python3 -I - "$root" "$0" "$@" <<'PY' import json, os, re, shutil, signal, subprocess, sys, time from pathlib import Path -root, args = sys.argv[1], sys.argv[2:] -opt = {'only': set(), 'skip': set(), 'all': False, 'setup': False, 'manifest': root + '/dev.json'} +root, script, args = sys.argv[1], sys.argv[2], sys.argv[3:] +opt = {'only': set(), 'skip': set(), 'all': False, 'setup': False, 'manifest': root + '/dev.json', 'changed': None} +def refuse(message): print('check.sh: ' + message, file=sys.stderr); sys.exit(2) while args: a = args.pop(0) if a in ('--only', '--skip'): opt[a[2:]] = set(args.pop(0).split(',')) elif a == '--manifest': opt['manifest'] = args.pop(0) elif a in ('--all', '--setup'): opt[a[2:]] = True - else: sys.exit('check.sh: unknown argument ' + a) + elif a == '--changed': opt['changed'] = [args.pop(0)] if args and not args[0].startswith('-') else [] + else: + try: + with open(script, encoding='utf-8') as handle: origin = handle.readlines()[1][2:].split(' (')[0].strip() + except (OSError, IndexError, UnicodeDecodeError): origin = 'an unknown kit version' + refuse('unknown argument %s (this runner is %s)' % (a, origin)) +if opt['changed'] is not None: + if opt['only']: refuse('--changed and --only both choose the steps; give one') + if opt['setup']: refuse('--changed selects checks, not setup steps') + here = os.path.dirname(os.path.abspath(script)) + found = [p for p in (os.path.join(here, 'changed_checks.py'), os.path.join(here, os.pardir, 'lib', 'changed_checks.py')) + if os.path.isfile(p)] + if not found: refuse('--changed needs changed_checks.py beside this script (vendor the kit\'s scripts/dev/changed_checks.py)') + # The first line is the selection (whole, or only NAME,NAME); the rest is the report. A refusal is its own. + picked = subprocess.run([sys.executable, '-I', found[0], '--root', root, '--manifest', opt['manifest'], + '--label', 'check.sh --changed'] + ['--base=' + b for b in opt['changed']], + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, text=True) + if picked.returncode: sys.exit(2) + first, _, report = picked.stdout.partition('\n') + if first.startswith('only ') and first[5:]: opt['only'] = set(first[5:].split(',')) + elif first != 'whole': refuse('--changed: the selection could not be read') + print(report, end='', flush=True) m = json.load(open(opt['manifest'], encoding='utf-8')) logdir = os.environ.get('JPDEV_LOGDIR') log_probe_error = '' diff --git a/templates/check.sh b/templates/check.sh index 7baa18a..8250cdb 100755 --- a/templates/check.sh +++ b/templates/check.sh @@ -1,22 +1,48 @@ #!/usr/bin/env bash # from judgment-pack-devkit v0.7.0 (templates/check.sh) -# scripts/dev/check.sh [--setup] [--only a,b] [--skip a,b] [--all] [--manifest FILE] +# scripts/dev/check.sh [--setup] [--only a,b | --changed [BASE]] [--skip a,b] [--all] [--manifest FILE] # Runs the checks of dev.json in order and prints one summary block; exit status = failures. +# --changed runs only the checks jpdev.json maps the paths changed since the merge base of BASE +# (default: the default branch) and HEAD to, with changed_checks.py vendored beside this script; +# the whole list runs when a changed path is unmapped, when jpdev.json, the manifest, this script +# or changed_checks.py changed, when nothing changed, or with no mapping. # Stops at the first failure unless --all. --setup runs the manifest's setup steps instead. # A step whose requires_env variable is empty is skipped, and said to be; when CI is set that is a # failure unless the step is marked optional: true. Standard library python3 only. jpdev sets JPDEV_WRAP to add its caps around each step. root=${JPDEV_ROOT:-$(cd "$(dirname "$0")" && git rev-parse --show-toplevel)} || exit 2 -exec python3 -I - "$root" "$@" <<'PY' +exec python3 -I - "$root" "$0" "$@" <<'PY' import json, os, re, shutil, signal, subprocess, sys, time from pathlib import Path -root, args = sys.argv[1], sys.argv[2:] -opt = {'only': set(), 'skip': set(), 'all': False, 'setup': False, 'manifest': root + '/dev.json'} +root, script, args = sys.argv[1], sys.argv[2], sys.argv[3:] +opt = {'only': set(), 'skip': set(), 'all': False, 'setup': False, 'manifest': root + '/dev.json', 'changed': None} +def refuse(message): print('check.sh: ' + message, file=sys.stderr); sys.exit(2) while args: a = args.pop(0) if a in ('--only', '--skip'): opt[a[2:]] = set(args.pop(0).split(',')) elif a == '--manifest': opt['manifest'] = args.pop(0) elif a in ('--all', '--setup'): opt[a[2:]] = True - else: sys.exit('check.sh: unknown argument ' + a) + elif a == '--changed': opt['changed'] = [args.pop(0)] if args and not args[0].startswith('-') else [] + else: + try: + with open(script, encoding='utf-8') as handle: origin = handle.readlines()[1][2:].split(' (')[0].strip() + except (OSError, IndexError, UnicodeDecodeError): origin = 'an unknown kit version' + refuse('unknown argument %s (this runner is %s)' % (a, origin)) +if opt['changed'] is not None: + if opt['only']: refuse('--changed and --only both choose the steps; give one') + if opt['setup']: refuse('--changed selects checks, not setup steps') + here = os.path.dirname(os.path.abspath(script)) + found = [p for p in (os.path.join(here, 'changed_checks.py'), os.path.join(here, os.pardir, 'lib', 'changed_checks.py')) + if os.path.isfile(p)] + if not found: refuse('--changed needs changed_checks.py beside this script (vendor the kit\'s scripts/dev/changed_checks.py)') + # The first line is the selection (whole, or only NAME,NAME); the rest is the report. A refusal is its own. + picked = subprocess.run([sys.executable, '-I', found[0], '--root', root, '--manifest', opt['manifest'], + '--label', 'check.sh --changed'] + ['--base=' + b for b in opt['changed']], + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, text=True) + if picked.returncode: sys.exit(2) + first, _, report = picked.stdout.partition('\n') + if first.startswith('only ') and first[5:]: opt['only'] = set(first[5:].split(',')) + elif first != 'whole': refuse('--changed: the selection could not be read') + print(report, end='', flush=True) m = json.load(open(opt['manifest'], encoding='utf-8')) logdir = os.environ.get('JPDEV_LOGDIR') log_probe_error = '' diff --git a/templates/ci-check.yml b/templates/ci-check.yml index 1780c4a..4c557b3 100644 --- a/templates/ci-check.yml +++ b/templates/ci-check.yml @@ -1,5 +1,8 @@ # Vendor as .github/workflows/ci.yml. Runs the repository's own check list (dev.json) # with scripts/dev/check.sh, with the toolchains set up from the manifest and no caps. +# A pull request runs `check.sh --changed` against its base branch: the checks jpdev.json maps the +# changed paths to, and the whole list when a path is unmapped, when the mapping, the manifest or the +# runner changed, or with no mapping. A push to the default branch and a dispatch run the whole list. name: dev-check run-name: dev-check ${{ inputs.sha || github.sha }} @@ -32,8 +35,10 @@ jobs: name: check runs-on: ubuntu-latest steps: + # The whole history, so that --changed finds the merge base with the base branch. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + fetch-depth: 0 persist-credentials: false ref: ${{ inputs.sha }} - name: Read the toolchains from dev.json @@ -87,7 +92,15 @@ jobs: - name: Setup steps run: scripts/dev/check.sh --setup - name: Checks - run: scripts/dev/check.sh --all + env: + EVENT: ${{ github.event_name }} + BASE_REF: ${{ github.base_ref }} + run: | + if [ "$EVENT" = pull_request ]; then + scripts/dev/check.sh --all --changed "origin/$BASE_REF" + else + scripts/dev/check.sh --all + fi handover: name: handover diff --git a/tests/changed_checks_test.py b/tests/changed_checks_test.py index 7e5b5b8..7c82622 100644 --- a/tests/changed_checks_test.py +++ b/tests/changed_checks_test.py @@ -1,4 +1,5 @@ -"""Handwritten fixture repositories exercise `jpdev check --changed` through bin/jpdev.""" +"""Handwritten fixture repositories exercise `jpdev check --changed` through bin/jpdev, and the +same fixtures `scripts/dev/check.sh --changed` through the vendored runner and selection code.""" import copy import importlib.util import json @@ -80,6 +81,11 @@ def git(cwd, *args): class ChangedCheckTests(unittest.TestCase): """A clone of a fixture origin, on a topic branch, with the mapping of tests/fixture.""" + LABEL = 'jpdev check --changed' + + def seed(self, seed): + """Add files to the fixture origin before its first commit.""" + def setUp(self): temporary = tempfile.TemporaryDirectory(prefix='changed-checks-') self.addCleanup(temporary.cleanup) @@ -97,6 +103,7 @@ def setUp(self): 'cmd/tool/main.go': 'package main\n', 'tests/unit_test.py': 'pass\n', 'scripts/tool.sh': 'true\n'} for name, text in files.items(): self.write(name, text, seed) + self.seed(seed) git(seed, 'add', '-A') git(seed, 'commit', '-q', '-m', 'seed') git(self.work, 'init', '-q', '--bare', '-b', 'main', str(self.work / 'origin.git')) @@ -138,12 +145,12 @@ def test_docs_only_change_runs_the_docs_steps_only(self): status, out, err = self.check('--changed') self.assertEqual(status, 0, out + err) self.assertEqual(self.ran(out), {'docs-lint', 'whitespace'}, out) - self.assertIn('== jpdev check --changed: 2 changed paths since origin/main (merge base ', out) + self.assertIn('== %s: 2 changed paths since origin/main (merge base ' % self.LABEL, out) self.assertRegex(out, r'(?m)^selected +docs-lint +docs/\*\*, \*\.md$') self.assertRegex(out, r'(?m)^not selected +go-test +no changed path maps to it$') self.assertIn('== 2 of 5 checks selected ==', out) self.assertIn('== 2 passed, 0 failed, 0 skipped, 0 not run ==', out) - self.assertLess(out.index('== jpdev check --changed'), out.index('== check summary')) + self.assertLess(out.index('== ' + self.LABEL), out.index('== check summary')) def test_mapped_go_path_runs_the_go_steps(self): self.write('cmd/tool/main.go', 'package main\n\nfunc main() {}\n') @@ -208,6 +215,29 @@ def test_a_change_to_the_vendored_runner_runs_the_whole_list(self): self.assertIn('selection input changed: scripts/dev/check.sh: whole list', out) self.assertEqual(self.ran(out), ALL, out) + def test_a_change_to_the_vendored_selection_code_runs_the_whole_list(self): + selection = self.clone / 'scripts' / 'dev' / 'changed_checks.py' + selection.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(str(ROOT / 'scripts' / 'dev' / 'changed_checks.py'), str(selection)) + self.mapping(lambda value: value['changed'].append({'paths': ['scripts/dev/**'], 'checks': ['whitespace']})) + base = self.commit_base('vendored selection code') + with selection.open('a') as handle: + handle.write('# a line after the selection code\n') + status, out, err = self.check('--changed', base) + self.assertEqual(status, 0, out + err) + self.assertIn('selection input changed: scripts/dev/changed_checks.py: whole list', out) + self.assertEqual(self.ran(out), ALL, out) + + def test_failures_name_the_kit_version(self): + version = subprocess.run([str(JPDEV), '--version'], stdout=subprocess.PIPE, universal_newlines=True, + check=True).stdout.strip() + status, out, err = self.check('--changed', 'no-such-ref') + self.assertEqual(status, 2, out + err) + self.assertIn('jpdev: check --changed: refused by this kit, %s' % version, err) + status, out, err = self.check('--chagned') + self.assertEqual(status, 2, out + err) + self.assertIn('check: unknown argument --chagned; this kit is %s' % version, err) + def test_a_symlinked_manifest_retargeted_runs_the_whole_list(self): manifest = json.loads((FIXTURE / 'changed-dev.json').read_text()) self.write('manifests/a.json', json.dumps(manifest)) @@ -292,7 +322,7 @@ def test_no_mapping_runs_the_whole_list(self): def test_no_changed_path_runs_the_whole_list(self): status, out, err = self.check('--changed') self.assertEqual(status, 0, out + err) - self.assertIn('== jpdev check --changed: 0 changed paths since origin/main', out) + self.assertIn('== %s: 0 changed paths since origin/main' % self.LABEL, out) self.assertIn('no changed path: whole list', out) self.assertEqual(self.ran(out), ALL, out) @@ -341,26 +371,57 @@ def test_invalid_mapping_is_refused_with_a_named_message(self): self.mapping(change) status, out, err = self.check('--changed') self.assertEqual(status, 2, out + err) - self.assertIn('jpdev check --changed: refused: ' + message + '\n', err) + self.assertIn(self.LABEL + ': refused: ' + message + '\n', err) self.assertEqual(out, '') self.assertNotIn('check: running', err) self.write('jpdev.json', '{"version": 1, "changed": [') status, out, err = self.check('--changed') self.assertEqual(status, 2, out + err) - self.assertIn('jpdev check --changed: refused: jpdev.json: not readable JSON', err) + self.assertIn(self.LABEL + ': refused: jpdev.json: not readable JSON', err) def test_an_unresolved_base_is_refused(self): + self.write('docs/guide.md', 'guide, edited\n') status, out, err = self.check('--changed', 'no-such-ref') self.assertEqual(status, 2, out + err) self.assertIn("refused: cannot resolve BASE 'no-such-ref' to a commit", err) self.assertEqual(out, '') + # A pull-request job whose base branch name came through empty asks for `origin/`. + status, out, err = self.check('--changed', 'origin/') + self.assertEqual(status, 2, out + err) + self.assertIn("refused: cannot resolve BASE 'origin/' to a commit", err) + self.assertEqual(out, '') git(self.clone, 'remote', 'remove', 'origin') status, out, err = self.check('--changed') self.assertEqual(status, 2, out + err) self.assertIn('refused: no default branch found (origin/HEAD, origin/main or origin/master); name BASE', err) status, out, err = self.check('--changed', 'main') self.assertEqual(status, 0, out + err) - self.assertIn('0 changed paths since main', out) + self.assertIn('1 changed path since main', out) + + def test_a_shallow_clone_without_the_merge_base_is_refused(self): + seed = self.work / 'seed' + git(seed, 'switch', '-q', '-c', 'side') + self.write('docs/side.md', 'side\n', seed) + git(seed, 'add', '-A') + git(seed, 'commit', '-q', '-m', 'side') + git(seed, 'switch', '-q', 'main') + self.write('docs/main.md', 'main\n', seed) + git(seed, 'add', '-A') + git(seed, 'commit', '-q', '-m', 'main moves on') + git(seed, 'push', '-q', str(self.work / 'origin.git'), 'main', 'side') + # Each branch tip alone: the commit they share is not in the clone. + self.clone = self.work / 'shallow' + git(self.work, 'clone', '-q', '--depth', '1', '--no-single-branch', '--branch', 'side', + (self.work / 'origin.git').as_uri(), str(self.clone)) + self.write('docs/guide.md', 'guide, edited\n') + for args in (('--changed',), ('--changed', 'origin/main')): + with self.subTest(args=args): + status, out, err = self.check(*args) + self.assertEqual(status, 2, out + err) + self.assertIn(self.LABEL + ': refused: no merge base of origin/main and HEAD ' + '(a shallow clone needs the history)\n', err) + self.assertEqual(out, '') + self.assertNotIn('check: running', err) def test_changed_and_only_are_not_given_together(self): status, out, err = self.check('--changed', '--only', 'docs-lint') @@ -368,6 +429,75 @@ def test_changed_and_only_are_not_given_together(self): self.assertIn('--changed and --only both choose the steps; give one', err) +class VendoredChangedCheckTests(ChangedCheckTests): + """Every fixture of ChangedCheckTests again, through the clone's vendored scripts/dev/check.sh and + scripts/dev/changed_checks.py instead of bin/jpdev: the same selection, refusals and report.""" + + LABEL = 'check.sh --changed' + + def seed(self, seed): + runner = seed / 'scripts' / 'dev' / 'check.sh' + runner.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(str(ROOT / 'scripts' / 'dev' / 'check.sh'), str(runner)) + shutil.copy(str(ROOT / 'scripts' / 'dev' / 'changed_checks.py'), str(runner.parent / 'changed_checks.py')) + + def check(self, *args, runner=None, env=None): + result = subprocess.run([str(runner or self.clone / 'scripts' / 'dev' / 'check.sh')] + list(args), + cwd=str(self.clone), env=env or self.env, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, + timeout=120) + return result.returncode, result.stdout, result.stderr + + def report(self, out, label): + """The selection block, from its heading to its last line, with the runner's name left out.""" + lines = out.split('== check summary')[0].splitlines() + return [line.replace(label, 'RUNNER') for line in lines if not line.startswith('PASS ')] + + def test_the_kit_and_the_vendored_runner_select_alike(self): + cases = [('docs', {'docs/guide.md': 'guide, edited\n'}), + ('go and tests', {'cmd/tool/main.go': 'package main\n\nfunc main() {}\n', 'tests/unit_test.py': 'assert 1\n'}), + ('unmapped', {'scripts/new.sh': 'true\n'})] + for label, files in cases: + with self.subTest(case=label): + for name, text in files.items(): + self.write(name, text) + status, vendored, err = self.check('--changed') + self.assertEqual(status, 0, vendored + err) + status, kit, err = ChangedCheckTests.check(self, '--changed') + self.assertEqual(status, 0, kit + err) + self.assertEqual(self.report(vendored, self.LABEL), self.report(kit, ChangedCheckTests.LABEL)) + self.assertEqual(self.ran(vendored), self.ran(kit)) + git(self.clone, 'checkout', '-q', '--', '.') + git(self.clone, 'clean', '-qfd') + + def test_the_template_in_the_kit_uses_the_kit_selection_code(self): + self.write('docs/guide.md', 'guide, edited\n') + env = dict(self.env, JPDEV_ROOT=str(self.clone)) + status, out, err = self.check('--changed', runner=ROOT / 'templates' / 'check.sh', env=env) + self.assertEqual(status, 0, out + err) + self.assertEqual(self.ran(out), {'docs-lint', 'whitespace'}, out) + + def test_failures_name_the_kit_version(self): + line2 = (ROOT / 'templates' / 'check.sh').read_text().splitlines()[1] + self.assertRegex(line2, r'^# from judgment-pack-devkit v[0-9]+\.[0-9]+\.[0-9]+ \(templates/check\.sh\)$') + status, out, err = self.check('--chagned') + self.assertEqual(status, 2, out + err) + self.assertIn('check.sh: unknown argument --chagned (this runner is %s)\n' % line2[2:].split(' (')[0], err) + + def test_changed_needs_the_vendored_selection_code(self): + (self.clone / 'scripts' / 'dev' / 'changed_checks.py').unlink() + self.write('docs/guide.md', 'guide, edited\n') + status, out, err = self.check('--changed') + self.assertEqual(status, 2, out + err) + self.assertIn('check.sh: --changed needs changed_checks.py beside this script', err) + self.assertEqual(out, '') + + def test_changed_selects_checks_not_setup_steps(self): + status, out, err = self.check('--setup', '--changed') + self.assertEqual(status, 2, out + err) + self.assertIn('check.sh: --changed selects checks, not setup steps', err) + + SCHEMA_KEYWORDS = {'$schema', 'title', 'description', 'type', 'const', 'enum', 'required', 'anyOf', 'properties', 'additionalProperties', 'items', 'minItems', 'uniqueItems', 'minLength', 'minimum', 'maximum', 'multipleOf', 'pattern'} diff --git a/tests/run.sh b/tests/run.sh index 3f8cc2f..f9cdc32 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -423,6 +423,39 @@ handover_want="# from judgment-pack-devkit v$("$jpdev" --version | awk '{ print if [ "$handover_line2" = "$handover_want" ]; then ok "scripts/dev/handover_check.sh line 2 names the kit version and source"; else bad "scripts/dev/handover_check.sh line 2 names the kit version and source" "line 2: $handover_line2 wanted: $handover_want"; fi if cmp -s <(tail -n +2 "$kit/lib/handover_check.sh") <(tail -n +3 "$kit/scripts/dev/handover_check.sh" 2>/dev/null); then ok "scripts/dev/handover_check.sh is the library script after its version line"; else bad "scripts/dev/handover_check.sh is the library script after its version line"; fi +changed_line2=$(sed -n 2p "$kit/scripts/dev/changed_checks.py" 2>/dev/null) +changed_want="# from judgment-pack-devkit v$("$jpdev" --version | awk '{ print $2 }') (lib/changed_checks.py)" +if [ "$changed_line2" = "$changed_want" ]; then ok "scripts/dev/changed_checks.py line 2 names the kit version and source"; else bad "scripts/dev/changed_checks.py line 2 names the kit version and source" "line 2: $changed_line2 +wanted: $changed_want"; fi +if cmp -s "$kit/lib/changed_checks.py" <(sed 2d "$kit/scripts/dev/changed_checks.py" 2>/dev/null); then ok "scripts/dev/changed_checks.py is the library selection code but for its version line"; else bad "scripts/dev/changed_checks.py is the library selection code but for its version line"; fi + +# The release evidence hashes the runner and its selection code, which CI runs on pull requests. +out=$(python3 -I -c 'import json, sys; inputs = json.load(open(sys.argv[1]))["inputs"]; missing = [p for p in ("scripts/dev/check.sh", "scripts/dev/changed_checks.py") if p not in inputs]; print("missing: " + " ".join(missing) if missing else "covered")' "$kit/.github/ci-evidence.json" 2>&1) && s=0 || s=$? +expect ".github/ci-evidence.json inputs hold the vendored runner and its selection code" 0 '^covered$' "$out" "$s" + +# --- jpdev version tells a kit at a release from one past it; --version stays the release's line +released=$("$jpdev" --version) +if [[ $released =~ ^jpdev\ [0-9]+\.[0-9]+\.[0-9]+$ ]]; then ok "jpdev --version prints the release version alone"; else bad "jpdev --version prints the release version alone" "$released"; fi +out=$("$jpdev" version 2>&1) && s=0 || s=$? +if [ "$(git -C "$kit" rev-parse --show-toplevel 2>/dev/null)" = "$kit" ]; then + expect "jpdev version adds the kit checkout's git describe" 0 "^${released//./\\.} \\($(git -C "$kit" describe --tags --always)\\)\$" "$out" "$s" +else + expect "jpdev version is the release version where the kit is no checkout" 0 "^${released//./\\.}\$" "$out" "$s" +fi +# A kit unpacked inside another repository's clone is not that repository's commit. +git init -q -b main "$work/kitcopy" +git -C "$work/kitcopy" -c user.name=test -c user.email=test@example.com -c commit.gpgsign=false commit -q --allow-empty -m outer +mkdir -p "$work/kitcopy/vendor/kit" +cp -R "$kit/bin" "$kit/lib" "$work/kitcopy/vendor/kit/" +out=$("$work/kitcopy/vendor/kit/bin/jpdev" version 2>&1) && s=0 || s=$? +expect "jpdev version describes no repository the kit only sits inside" 0 "^${released//./\\.}\$" "$out" "$s" +out=$(GIT_DIR=$work/kitcopy/.git "$work/kitcopy/vendor/kit/bin/jpdev" version 2>&1) && s=0 || s=$? +expect "jpdev version describes no repository GIT_DIR names" 0 "^${released//./\\.}\$" "$out" "$s" +if [ "$(git -C "$kit" rev-parse --show-toplevel 2>/dev/null)" = "$kit" ]; then + out=$(GIT_DIR=$work/kitcopy/.git "$jpdev" version 2>&1) && s=0 || s=$? + expect "jpdev version describes the kit checkout whatever GIT_DIR names" 0 "^${released//./\\.} \\($(git -C "$kit" describe --tags --always)\\)\$" "$out" "$s" +fi +rm -rf "$work/kitcopy" # --- the lock lock=$work/test.lock @@ -989,7 +1022,8 @@ cx_reset out=$(STUB_CODEX_EDIT=1 cx_run --no-commit --small) && s=0 || s=$? expect "codex run --small refuses a shared path after the run" 2 'refused after the run: file.txt is a shared or boundary path' "$out" "$s" prompt=$(python3 -I -c 'import sys; a = open(sys.argv[1], "rb").read().split(b"\0"); sys.stdout.write(a[-2].decode())' "$STUB_CODEX_ARGS") -expect "codex run --small prepends its scoped-check line" 0 '^This is a small fix: run only the check-list steps' "$(sed -n '2p' <<< "$prompt")" 0 +expect "codex run --small prepends its scoped-check line" 0 '^This is a small fix: run .jpdev check --changed., which runs only the check-list steps jpdev.json maps' "$(sed -n '2p' <<< "$prompt")" 0 +absent "codex run --small leaves the choice of steps to the mapping, not --only" '--only' "$(sed -n '2p' <<< "$prompt")" cx_reset out=$(JPDEV_SMALL_MAX_LINES=0 STUB_CODEX_EDIT=1 STUB_CODEX_EDIT_FILE=AGENTS.md cx_run --no-commit --small) && s=0 || s=$? expect "codex run --small refuses a change over the line bound" 2 'changed lines exceeds JPDEV_SMALL_MAX_LINES=0' "$out" "$s"