diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e0cfe0b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# The scripts run under bash and python3: a checkout never converts their line endings. +* text=auto eol=lf +*.tgz binary diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..fe37c30 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,20 @@ +## Summary + +Which issue or change this is. One per pull request. + +## What changes + +The files and behaviour, in words. + +## Public surface + +`none`, or what changed in a command's output or options, the manifest schema (`templates/dev.schema.json`), +a template, or a setting, and the tests and documentation changed with it. + +## Checks + +The summary block `scripts/dev/check.sh` printed (or `jpdev check`), and any check you ran another way. + +## Sign-off + +Every commit carries a Developer Certificate of Origin sign-off that matches its author. diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..ca79ca5 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9030e5..90ef768 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,9 +1,15 @@ name: CI +# The check list is dev.json, run by scripts/dev/check.sh (the kit's own vendored runner). The two +# jobs keep their names, which branch protection requires: `lint` runs the cheap steps, `tests` the +# kit's test run. + on: push: branches: [main] pull_request: + # Called by release.yml, so the release runs the tagged commit's own jobs. + workflow_call: permissions: contents: read @@ -16,32 +22,11 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - name: shellcheck - run: shellcheck -x -P ".:SCRIPTDIR" bin/* lib/*.sh templates/*.sh scripts/*.sh tests/*.sh tests/fixture/*.sh - - name: bash -n - run: | - set -eu - for file in bin/* lib/*.sh templates/*.sh scripts/*.sh tests/*.sh tests/fixture/*.sh; do - bash -n "$file" - done - - name: CLAUDE.md is generated from AGENTS.md - run: scripts/sync-agent-docs.sh --check - - name: The JSON and YAML documents parse - run: | - set -eu - python3 -c ' - import glob, json - for path in glob.glob("templates/*.json") + glob.glob("examples/*.json") + glob.glob("tests/fixture/*.json"): - json.load(open(path, encoding="utf-8")) - print("parsed " + path)' - - name: No machine-local paths in tracked text - run: | - set -eu - if git grep -nE '(^|[^A-Za-z0-9_])/(home|tmp)/[A-Za-z0-9_.-]+'; then - echo 'a machine-local path is in tracked text'; exit 1 - fi - - name: Whitespace - run: git diff --check "$(git hash-object -t tree /dev/null)" HEAD + # Whitespace; no machine-local path in tracked text; CLAUDE.md is AGENTS.md and the root + # copy starts with the template; every script parses; the JSON documents parse; shellcheck + # (the program the runner image carries). + - name: The lint steps of dev.json + run: scripts/dev/check.sh --all --only diff-check,local-paths,agent-docs,bash-n,json,shellcheck tests: name: tests @@ -51,4 +36,4 @@ jobs: with: persist-credentials: false - name: The kit's tests (jpdev check and check.sh on the fixture, the caps, the record gate) - run: tests/run.sh + run: scripts/dev/check.sh --only tests diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..25c6782 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,388 @@ +name: Release + +# Publishes a release of the development kit: jpdev as a tarball, its checksums and a +# build-provenance attestation, from a tag a person pushes on a commit already on main +# (docs/releasing.md). No registry is involved: a person installs the attested tarball. +# +# A person decides there is a release, writes its notes (docs/releases/.md) and sets the kit's +# version to the tag's. The workflow holds the tagged commit to CI, packages the kit from it, +# attests the files, and publishes no release until a maintainer approves the `release` +# environment. +# +# Not secret before that approval: the files (an artifact of the run), their attestation (recorded +# in a public log when made), and the draft (visible to those who can write to the repository). +# +# Every job works on the commit the run was started for, by digest. A tag is a name and can be +# moved until its release is published, so the tag is compared with that commit when the run +# begins, before the draft, and before publishing. + +on: + push: + tags: + # Narrowed here, held exactly to the grammar by the first step. + - "v[0-9]+.[0-9]+.[0-9]+" + +permissions: {} + +env: + RELEASE_TAG: ${{ github.ref_name }} + +concurrency: + group: release-${{ github.ref_name }} + cancel-in-progress: false + +jobs: + admit: + name: Admit the tag + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + # Reading an environment's protection rules. + actions: read + steps: + # vMAJOR.MINOR.PATCH in ASCII digits, no leading zeroes, nothing before or after: no + # prerelease and no build metadata. + - name: Validate release tag syntax + shell: bash + env: + LC_ALL: C + run: | + number='(0|[1-9][0-9]*)' + grammar="^v${number}\.${number}\.${number}$" + if [[ ! "${RELEASE_TAG}" =~ ${grammar} ]]; then + echo "Release tag is not vMAJOR.MINOR.PATCH with no leading zeroes: ${RELEASE_TAG}" >&2 + exit 1 + fi + + - name: Check out the commit the run was started for + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + ref: ${{ github.sha }} + + # A signed tag is an annotated tag; a lightweight tag is refused. + - name: Require an annotated tag + shell: bash + run: | + kind="$(git cat-file -t "refs/tags/${RELEASE_TAG}")" \ + || { echo "refs/tags/${RELEASE_TAG} could not be read; releases need an annotated tag (docs/releasing.md)" >&2; exit 1; } + [ "${kind}" = "tag" ] || { echo "refs/tags/${RELEASE_TAG} is a ${kind} object; releases need an annotated tag (git tag -s), not a lightweight one" >&2; exit 1; } + + - name: Require the tag to name that commit + shell: bash + run: | + # ls-remote patterns match the end of a name, so lines are selected by exact name. The + # peeled line is the commit an annotated tag points to; a lightweight tag has only its + # own line. + named="$(git ls-remote "https://github.com/${GITHUB_REPOSITORY}.git" "refs/tags/${RELEASE_TAG}" "refs/tags/${RELEASE_TAG}^{}" \ + | awk -v tag="refs/tags/${RELEASE_TAG}" '$2 == tag { plain = $1 } $2 == tag "^{}" { peeled = $1 } END { print (peeled != "" ? peeled : plain) }')" + if [ -z "${named}" ] || [ "${named}" != "${GITHUB_SHA}" ]; then + echo "refs/tags/${RELEASE_TAG} names ${named:-nothing}; the run was started for ${GITHUB_SHA}" >&2 + exit 1 + fi + + # Only merged work is released: a tag on a branch commit names a state main may never hold. + # Any commit on main is admitted, not only its tip. Main is named in full: a tag called + # origin/main would answer the short name before the branch. + - name: Require the tagged commit on main + shell: bash + run: | + git merge-base --is-ancestor "${GITHUB_SHA}" refs/remotes/origin/main || { echo "Tagged commit ${GITHUB_SHA} is not on main" >&2; exit 1; } + + # The notes are the text the release page carries. A notes file written for another tag is + # not this release's notes: the tag's name is the last word of its first line. + - name: Require release notes + shell: bash + run: | + notes="docs/releases/${RELEASE_TAG}.md" + test -s "${notes}" || { echo "${notes} is missing or empty at the tag" >&2; exit 1; } + first="$(head -n 1 "${notes}" | sed -E 's/ +$//')" + [ "${first##* }" = "${RELEASE_TAG}" ] || { echo "${notes} must end its first line with ${RELEASE_TAG} (found: ${first})" >&2; exit 1; } + + # GitHub creates a missing environment on first use, without protection. Read the rule + # before anything is written so the publish gate cannot be a bare name. + - name: Require a reviewer on the release environment + shell: bash + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + reviewers="$(gh api "repos/${GH_REPO}/environments/release" --jq '[.protection_rules[]? | select(.type == "required_reviewers")] | length')" \ + || { echo "The release environment could not be read; create it with a required reviewer (docs/releasing.md)" >&2; exit 1; } + [ "${reviewers}" -ge 1 ] || { echo "The release environment requires no reviewer" >&2; exit 1; } + + # The tagged commit's own ci.yml, called, so release.yml carries no second copy of the `lint` + # and `tests` jobs. A check main has gained since that commit is not run. + verify: + name: CI at the tag + needs: admit + permissions: + contents: read + uses: ./.github/workflows/ci.yml + + package: + name: Package + needs: + - admit + - verify + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + outputs: + version: ${{ steps.version.outputs.version }} + steps: + - name: Check out the commit the run was started for + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ github.sha }} + + # The kit's version is the tag without its v, or the job refuses: the tarball's name, the + # version `jpdev --version` prints and the version line of the runner template + # (templates/check.sh, line 2) must agree with the tag. + - name: Require the kit's version to equal the tag + id: version + shell: bash + run: | + version="${RELEASE_TAG#v}" + held="$(bin/jpdev --version)" + [ "${held}" = "jpdev ${version}" ] || { echo "bin/jpdev --version prints '${held}'; the tag ${RELEASE_TAG} needs 'jpdev ${version}' (JPDEV_VERSION in lib/common.sh)" >&2; exit 1; } + line="$(sed -n 2p templates/check.sh)" + want="# from judgment-pack-devkit ${RELEASE_TAG} (templates/check.sh)" + [ "${line}" = "${want}" ] || { echo "templates/check.sh line 2 is '${line}'; the tag ${RELEASE_TAG} needs '${want}'" >&2; exit 1; } + cmp -s templates/check.sh scripts/dev/check.sh || { echo "scripts/dev/check.sh is not a copy of templates/check.sh" >&2; exit 1; } + echo "version=${version}" >> "${GITHUB_OUTPUT}" + + # git archive writes the named paths of the commit, with the commit's time and each file's + # recorded mode; gzip -n leaves out its own name and time. The same commit gives the same + # bytes. + - name: Package the kit and write its checksums + shell: bash + env: + VERSION: ${{ steps.version.outputs.version }} + run: | + set -eu + mkdir dist + tarball="judgment-pack-devkit-${VERSION}.tgz" + git archive --format=tar --prefix="judgment-pack-devkit-${VERSION}/" "${GITHUB_SHA}" \ + bin lib templates config scripts docs README.md LICENSE NOTICE | gzip -n -9 > "dist/${tarball}" + (cd dist && sha256sum "${tarball}" > checksums.txt && sha256sum --check --strict checksums.txt) + cat dist/checksums.txt + + # The tarball runs as it is: unpacked, it reports the version the tag names. + - name: Run the packaged kit + shell: bash + env: + VERSION: ${{ steps.version.outputs.version }} + run: | + set -eu + unpack="$(mktemp -d)" + tar -xzf "dist/judgment-pack-devkit-${VERSION}.tgz" -C "${unpack}" + kit="${unpack}/judgment-pack-devkit-${VERSION}" + [ "$("${kit}/bin/jpdev" --version)" = "jpdev ${VERSION}" ] || { echo "the packaged jpdev does not report ${VERSION}" >&2; exit 1; } + "${kit}/bin/jpdev" env > /dev/null + for path in bin/jpdev lib/wrap.sh templates/check.sh templates/dev.schema.json scripts/sync-agent-docs.sh LICENSE NOTICE README.md; do + test -e "${kit}/${path}" || { echo "the tarball does not hold ${path}" >&2; exit 1; } + done + + - name: Carry the release notes with the files + shell: bash + run: cp "docs/releases/${RELEASE_TAG}.md" dist/release-notes.md + + - name: Upload the release's files + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: release-dist + path: | + dist/judgment-pack-devkit-*.tgz + dist/checksums.txt + dist/release-notes.md + if-no-files-found: error + # As long as an approval may be waited for: the publish job reads the artifact again + # after the gate. + retention-days: 30 + + attest-and-draft: + name: Attest and draft release + needs: package + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + id-token: write + attestations: write + artifact-metadata: write + env: + VERSION: ${{ needs.package.outputs.version }} + outputs: + # The draft this run made, by its id: the publish job reads and publishes that release, never + # whatever the tag names by then. + release-id: ${{ steps.draft.outputs.release-id }} + steps: + - name: Download the release's files + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: release-dist + path: dist + + - name: Require the tag to still name that commit + shell: bash + run: | + named="$(git ls-remote "https://github.com/${GITHUB_REPOSITORY}.git" "refs/tags/${RELEASE_TAG}" "refs/tags/${RELEASE_TAG}^{}" \ + | awk -v tag="refs/tags/${RELEASE_TAG}" '$2 == tag { plain = $1 } $2 == tag "^{}" { peeled = $1 } END { print (peeled != "" ? peeled : plain) }')" + if [ -z "${named}" ] || [ "${named}" != "${GITHUB_SHA}" ]; then + echo "refs/tags/${RELEASE_TAG} names ${named:-nothing}; the files were built from ${GITHUB_SHA}" >&2 + exit 1 + fi + + # An existing release, draft or published, is never replaced or completed from here. A person + # inspects what a failed run left (docs/releasing.md). + - name: Require that no release exists under this tag + shell: bash + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + # List every release, drafts included, and match the tag exactly. A failed lookup fails + # the step; it is never read as absence. + state="$(gh api "repos/${GH_REPO}/releases" --paginate --jq '.[] | select(.tag_name == env.RELEASE_TAG) | if .draft then "a draft" else "published" end')" + if [ -n "${state}" ]; then + echo "A release under ${RELEASE_TAG} already exists, and it is $(echo "${state}" | sort -u | paste -sd, -)" >&2 + exit 1 + fi + + # Both files the release attaches are subjects, each by its path; checksums.txt also names + # the tarball's digest. + - name: Attest the release's files + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-path: | + dist/judgment-pack-devkit-${{ env.VERSION }}.tgz + dist/checksums.txt + + - name: Create draft release + id: draft + shell: bash + env: + GH_TOKEN: ${{ github.token }} + # No checkout in this job, so gh cannot infer the repository. + GH_REPO: ${{ github.repository }} + run: | + gh release create "${RELEASE_TAG}" "dist/judgment-pack-devkit-${VERSION}.tgz" dist/checksums.txt \ + --draft \ + --notes-file dist/release-notes.md \ + --title "jpdev ${RELEASE_TAG}" \ + --verify-tag + # The one draft under the tag, which this job has just made (no release existed under it + # before). The listing can lag the creation by a few seconds: read it again until the + # draft appears, at most twelve times. + release_id="" + for _ in $(seq 1 12); do + release_id="$(gh api "repos/${GH_REPO}/releases" --paginate --jq '.[] | select(.tag_name == env.RELEASE_TAG and .draft) | .id')" + if [[ "${release_id}" =~ ^[0-9]+$ ]]; then break; fi + sleep 5 + done + [[ "${release_id}" =~ ^[0-9]+$ ]] || { echo "Expected one draft release under ${RELEASE_TAG}; found: ${release_id:-none}" >&2; exit 1; } + echo "release-id=${release_id}" >> "${GITHUB_OUTPUT}" + + publish: + name: Publish release + needs: + - package + - attest-and-draft + runs-on: ubuntu-latest + environment: release + timeout-minutes: 10 + permissions: + contents: write + env: + RELEASE_ID: ${{ needs.attest-and-draft.outputs.release-id }} + VERSION: ${{ needs.package.outputs.version }} + steps: + - name: Require the tag to still name that commit + shell: bash + run: | + named="$(git ls-remote "https://github.com/${GITHUB_REPOSITORY}.git" "refs/tags/${RELEASE_TAG}" "refs/tags/${RELEASE_TAG}^{}" \ + | awk -v tag="refs/tags/${RELEASE_TAG}" '$2 == tag { plain = $1 } $2 == tag "^{}" { peeled = $1 } END { print (peeled != "" ? peeled : plain) }')" + if [ -z "${named}" ] || [ "${named}" != "${GITHUB_SHA}" ]; then + echo "refs/tags/${RELEASE_TAG} names ${named:-nothing}; the draft was written from ${GITHUB_SHA}" >&2 + exit 1 + fi + + - name: Download the files this run wrote + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: release-dist + path: run + + # The run's own files, the bytes the draft must hold: checksums.txt in the one form this + # workflow writes it, naming the tarball alone. + - name: Hold the run's files to their checksums + shell: bash + working-directory: run + run: | + tarball="judgment-pack-devkit-${VERSION}.tgz" + if grep -q -v -E '^[0-9a-f]{64} [A-Za-z0-9._-]+$' checksums.txt; then + echo "checksums.txt holds a line that is not a digest, two spaces and a plain name" >&2 + exit 1 + fi + [ "$(awk '{ print $2 }' checksums.txt)" = "${tarball}" ] || { echo "checksums.txt does not name ${tarball} alone" >&2; exit 1; } + sha256sum --check --strict checksums.txt + + # A draft can be changed while it waits at the gate, by anyone who can write to the + # repository. The release this run drafted is read through the API by its id, never + # downloaded: its state, title and notes, then the digest GitHub reports for each asset, + # compared with the run's files. The publish call follows that last read at once, by the same + # id. The moment between the two is the residual window docs/releasing.md states; the next + # step closes it after the fact. + - name: Require the draft to hold what this run wrote, and publish it + shell: bash + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + [[ "${RELEASE_ID}" =~ ^[0-9]+$ ]] || { echo "No release id from the draft job: ${RELEASE_ID:-none}" >&2; exit 1; } + tarball="judgment-pack-devkit-${VERSION}.tgz" + written="$(cd run && for name in checksums.txt "${tarball}"; do printf '%s sha256:%s\n' "${name}" "$(sha256sum "${name}" | cut -d ' ' -f 1)"; done | LC_ALL=C sort)" + + state="$(gh api "repos/${GH_REPO}/releases/${RELEASE_ID}" --jq '[.draft, .tag_name, .name] | @tsv')" + [ "${state}" = "$(printf 'true\t%s\tjpdev %s' "${RELEASE_TAG}" "${RELEASE_TAG}")" ] || { echo "Release ${RELEASE_ID} is not this run's draft of ${RELEASE_TAG}: ${state}" >&2; exit 1; } + # Both are taken without the newlines they end in, which the two ways of reading them do + # not agree on. + drafted_notes="$(gh api "repos/${GH_REPO}/releases/${RELEASE_ID}" --jq .body)" + written_notes="$(cat run/release-notes.md)" + [ "${drafted_notes}" = "${written_notes}" ] || { echo "The draft's notes are not the notes the commit holds:" >&2; diff <(echo "${written_notes}") <(echo "${drafted_notes}") >&2; exit 1; } + + drafted="$(gh api "repos/${GH_REPO}/releases/${RELEASE_ID}/assets" --paginate --jq '.[] | "\(.name) \(.digest)"' | LC_ALL=C sort)" + if [ "${drafted}" != "${written}" ]; then + printf 'The draft does not hold what this run wrote.\nThis run wrote:\n%s\nThe draft holds:\n%s\n' "${written}" "${drafted}" >&2 + exit 1 + fi + gh api -X PATCH "repos/${GH_REPO}/releases/${RELEASE_ID}" -F draft=false -f make_latest=true --silent + + # Published, the release is immutable once the repository's release immutability is on + # (docs/releasing.md). Its assets are read once more: a file replaced in the window is + # published only as a release this job fails, by name, with both digest sets. + - name: Require the published release to hold what this run wrote + shell: bash + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + tarball="judgment-pack-devkit-${VERSION}.tgz" + written="$(cd run && for name in checksums.txt "${tarball}"; do printf '%s sha256:%s\n' "${name}" "$(sha256sum "${name}" | cut -d ' ' -f 1)"; done | LC_ALL=C sort)" + draft="$(gh api "repos/${GH_REPO}/releases/${RELEASE_ID}" --jq .draft)" \ + || { echo "Could not verify the published release ${RELEASE_TAG} (release ${RELEASE_ID}): reading it failed. Treat it as unverified (docs/releasing.md)" >&2; exit 1; } + published="$(gh api "repos/${GH_REPO}/releases/${RELEASE_ID}/assets" --paginate --jq '.[] | "\(.name) \(.digest)"')" \ + || { echo "Could not verify the published release ${RELEASE_TAG} (release ${RELEASE_ID}): reading its assets failed. Treat it as unverified (docs/releasing.md)" >&2; exit 1; } + published="$(printf '%s\n' "${published}" | LC_ALL=C sort)" + if [ "${draft}" != "false" ] || [ "${published}" != "${written}" ]; then + printf 'The published release does not hold what this run wrote: %s (release %s, draft: %s).\nThis run wrote:\n%s\nThe release holds:\n%s\n' \ + "${RELEASE_TAG}" "${RELEASE_ID}" "${draft}" "${written}" "${published}" >&2 + exit 1 + fi + echo "Published ${RELEASE_TAG} (release ${RELEASE_ID}):" + echo "${published}" diff --git a/.github/workflows/thank-merged-contributors.yml b/.github/workflows/thank-merged-contributors.yml new file mode 100644 index 0000000..36ff7b2 --- /dev/null +++ b/.github/workflows/thank-merged-contributors.yml @@ -0,0 +1,60 @@ +name: Thank merged contributors + +on: + pull_request_target: + types: [closed] + +permissions: + pull-requests: write + +jobs: + thank-contributor: + if: github.event.pull_request.merged == true && github.event.pull_request.user.type != 'Bot' + runs-on: ubuntu-latest + env: + COMMENT_BODY: | + + 🎉 Thank you for your contribution to Judgment Pack! + + If you’d like another task, choose a path that interests you: + + - [Go runtime](https://github.com/Judgment-Pack/judgment-pack-runtime/labels/good%20first%20issue) + - [Judgment desk](https://github.com/Judgment-Pack/judgment-pack-desk/labels/good%20first%20issue) + - [Gateway and provenance](https://github.com/Judgment-Pack/judgment-pack-gateway/labels/good%20first%20issue) + - [Python and evaluator experiments](https://github.com/Judgment-Pack/judgment-pack-evaluator-experiments/labels/good%20first%20issue) + - [Specification and documentation](https://github.com/Judgment-Pack/judgment-pack-spec/labels/good%20first%20issue) + - [Agent harness and adapters](https://github.com/Judgment-Pack/judgment-pack-agent/labels/good%20first%20issue) + steps: + - name: Check for an existing recommendation + id: existing + env: + GH_TOKEN: ${{ github.token }} + GH_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + case "${PR_NUMBER}" in + ""|*[!0-9]*) exit 1 ;; + esac + + existing="$( + gh api --paginate \ + "repos/${GH_REPOSITORY}/issues/${PR_NUMBER}/comments?per_page=100" \ + --jq '.[] | select(.body | contains("")) | .id' + )" + + if [ -n "${existing}" ]; then + printf 'already-posted=true\n' >> "${GITHUB_OUTPUT}" + else + printf 'already-posted=false\n' >> "${GITHUB_OUTPUT}" + fi + + - name: Thank the contributor and suggest another task + if: steps.existing.outputs.already-posted != 'true' + env: + GH_TOKEN: ${{ github.token }} + GH_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + gh api --method POST \ + "repos/${GH_REPOSITORY}/issues/${PR_NUMBER}/comments" \ + --raw-field body="${COMMENT_BODY}" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..06d83b6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +.DS_Store +.idea/ + +# Local AI-assistant configuration and state. +/.agent/ +/.agents/ +/.ai/ +/.aider/ +/.amazonq/ +/.claude/ +/.codeium/ +/.codex/ +/.continue/ +/.copilot/ +/.cursor/ +/.gemini/ +/.roo/ +/.tabnine/ +/.windsurf/ + +.vscode/ +*.log +__pycache__/ +*.py[cod] +.venv/ +/dist/ +/.review-scratch/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..5af7bab --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,51 @@ +# Working in this repository + +This file is read by Codex as `AGENTS.md` and by Claude as `CLAUDE.md`; the two are the same text. +Edit `AGENTS.md` and run `scripts/sync-agent-docs.sh` to regenerate `CLAUDE.md`. + +## Before you open a pull request + +- Run `scripts/dev/check.sh` (or `jpdev check` where the kit is installed) once. It runs the + checks listed in `dev.json` in order and prints one summary block. +- Paste that summary block in the pull request body, under a "Checks" heading. +- If a check fails: read its first lines in the block, fix the cause, re-run that one check with + `--only NAME`, then run the whole list once more. Do not loop on the whole list. +- Start no CI watch. Push, report once, and stop; CI owns everything deterministic. + +## Rules + +- Write test vectors and fixtures by hand. Never record them "from a run" of the code under test. +- Keep machine-local paths and personal names out of tracked files, commit messages and pull + request bodies. Refer to people by role or handle where a name is needed. +- Sign off every commit (`git commit -s`). Never force-push; never rewrite published history. +- End a process you started by its process id. Never use a pattern kill (`pkill`, `killall`). +- One pull request per issue or batch. State what changed and why in plain, factual wording. +- A change to a public surface (a schema, a contract, a command's output) says so in the pull + request and changes its vectors and documentation in the same pull request. +- If a safeguard stops you (a permission prompt, a refused command, a failed gate), stop and say + so in your report. Do not work around it. + +## Reports + +Your final report is short: the pull request and its head commit, what changed, the check +summary block, and anything you stopped on or chose not to do. + +## This repository + +- The kit is bash 5 and the python3 standard library. Write for bash 5; `shellcheck` runs on every + script and a finding is fixed, not silenced, unless the comment above the directive says why. +- `tests/run.sh` is the kit's own test run (no network; it needs git, python3, rsync, flock and go). + The templates are the product: a change to a file under `templates/`, to `lib/`, to `bin/jpdev` or + to a setting has its test in `tests/run.sh` in the same pull request. Break the change and watch + the test fail before you rely on it. +- `scripts/dev/check.sh` is a copy of `templates/check.sh`; after a change to the template, copy + it again. Line 2 of the template names the kit version (`# from judgment-pack-devkit vX.Y.Z + (templates/check.sh)`) and moves with `JPDEV_VERSION` in `lib/common.sh` at a release; a test + holds the two together. +- This file is `templates/AGENTS.md` plus this section. Change a shared rule in the template and + here together, then run `scripts/sync-agent-docs.sh .` for `CLAUDE.md`. +- Machine settings (paths, caps, name lists) never live in this repository. They are read from the + person's config file and environment (`config/env.example` documents each); a tracked file holds + no path of any machine. +- Only a maintainer releases (`docs/releasing.md`); a pull request does not change the version or + write the next version's notes. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..5af7bab --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,51 @@ +# Working in this repository + +This file is read by Codex as `AGENTS.md` and by Claude as `CLAUDE.md`; the two are the same text. +Edit `AGENTS.md` and run `scripts/sync-agent-docs.sh` to regenerate `CLAUDE.md`. + +## Before you open a pull request + +- Run `scripts/dev/check.sh` (or `jpdev check` where the kit is installed) once. It runs the + checks listed in `dev.json` in order and prints one summary block. +- Paste that summary block in the pull request body, under a "Checks" heading. +- If a check fails: read its first lines in the block, fix the cause, re-run that one check with + `--only NAME`, then run the whole list once more. Do not loop on the whole list. +- Start no CI watch. Push, report once, and stop; CI owns everything deterministic. + +## Rules + +- Write test vectors and fixtures by hand. Never record them "from a run" of the code under test. +- Keep machine-local paths and personal names out of tracked files, commit messages and pull + request bodies. Refer to people by role or handle where a name is needed. +- Sign off every commit (`git commit -s`). Never force-push; never rewrite published history. +- End a process you started by its process id. Never use a pattern kill (`pkill`, `killall`). +- One pull request per issue or batch. State what changed and why in plain, factual wording. +- A change to a public surface (a schema, a contract, a command's output) says so in the pull + request and changes its vectors and documentation in the same pull request. +- If a safeguard stops you (a permission prompt, a refused command, a failed gate), stop and say + so in your report. Do not work around it. + +## Reports + +Your final report is short: the pull request and its head commit, what changed, the check +summary block, and anything you stopped on or chose not to do. + +## This repository + +- The kit is bash 5 and the python3 standard library. Write for bash 5; `shellcheck` runs on every + script and a finding is fixed, not silenced, unless the comment above the directive says why. +- `tests/run.sh` is the kit's own test run (no network; it needs git, python3, rsync, flock and go). + The templates are the product: a change to a file under `templates/`, to `lib/`, to `bin/jpdev` or + to a setting has its test in `tests/run.sh` in the same pull request. Break the change and watch + the test fail before you rely on it. +- `scripts/dev/check.sh` is a copy of `templates/check.sh`; after a change to the template, copy + it again. Line 2 of the template names the kit version (`# from judgment-pack-devkit vX.Y.Z + (templates/check.sh)`) and moves with `JPDEV_VERSION` in `lib/common.sh` at a release; a test + holds the two together. +- This file is `templates/AGENTS.md` plus this section. Change a shared rule in the template and + here together, then run `scripts/sync-agent-docs.sh .` for `CLAUDE.md`. +- Machine settings (paths, caps, name lists) never live in this repository. They are read from the + person's config file and environment (`config/env.example` documents each); a tracked file holds + no path of any machine. +- Only a maintainer releases (`docs/releasing.md`); a pull request does not change the version or + write the next version's notes. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..f8ce728 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,30 @@ +# Code of conduct + +We want participation in this project to be rigorous, candid, and safe. + +## Expected behavior + +- Critique ideas, specifications, and evidence without attacking people. +- State assumptions and conflicts of interest. +- Make room for contributors with different levels of standards and domain experience. +- Treat security, privacy, accessibility, and harmful-use concerns as technical requirements. +- Preserve disagreement accurately; do not manufacture consensus. +- Respect confidential information and do not post customer or employer data without permission. + +## Unacceptable behavior + +- harassment, threats, discrimination, or sexualized conduct; +- deliberate intimidation or sustained personal disruption; +- publishing another person's private information; +- bad-faith manipulation of reviews or governance; +- knowingly submitting material without the right to license it; or +- using project channels to expose credentials, private packs, or security vulnerabilities. + +## Enforcement + +Report conduct concerns privately to the project maintainers through the organization contact +mechanism. Maintainers may warn, moderate, temporarily restrict, or permanently remove a participant +when necessary to protect the community. + +Enforcement decisions should be documented internally, handled consistently, and avoid unnecessary +disclosure about reporters or affected people. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..2c483a0 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,61 @@ +# Contributing + +Thank you for helping. This repository is the development kit of the Judgment Pack repositories: +the `jpdev` command, the manifest schema, the runner and agent instructions a repository vendors, +and the policy for agent work. It is plain bash 5 with the python3 standard library. + +## Working on an issue + +Say on the issue that you are taking it. Keep a pull request to one issue. A change to a public +surface (a command's output or options, `templates/dev.schema.json`, a template, a setting) says +so in the pull request and changes its tests and documentation in the same pull request. The +templates are what other repositories vendor, so a change to a template has its test in +`tests/run.sh`. + +## Local checks + +Run the repository's own check list once before you open a pull request, and paste its summary +block in the pull request body: + +```sh +scripts/dev/check.sh +``` + +It runs the steps of [`dev.json`](dev.json) in order: `git diff --check`, no machine-local paths in +tracked text, `CLAUDE.md` generated from `AGENTS.md`, `bash -n` over every script, the JSON +documents parse, `shellcheck`, and `tests/run.sh`. CI runs the same list through the same runner. +To run one step again, `scripts/dev/check.sh --only NAME`. + +Each tool by itself: + +```sh +tests/run.sh +shellcheck -x -P ".:SCRIPTDIR" bin/* lib/*.sh templates/*.sh scripts/*.sh scripts/dev/*.sh tests/*.sh tests/fixture/*.sh +``` + +`tests/run.sh` needs bash 5, git, python3, rsync, flock and go, and no network. `shellcheck` is the +native program (`apt install shellcheck`, or a release from its repository), not an npm package. + +## Agent instructions + +`AGENTS.md` and `CLAUDE.md` are one text: edit `AGENTS.md` and run `scripts/sync-agent-docs.sh .` +to regenerate `CLAUDE.md`. The shared rules come from `templates/AGENTS.md`; change them there and +in the root copy together. The repository's own rules sit under "This repository" in `AGENTS.md`. + +## Developer Certificate of Origin + +Every commit is signed off (`git commit -s`) under the +[Developer Certificate of Origin](https://developercertificate.org/). The sign-off name and email +must match the commit's author. + +## Licence + +Contributions are accepted under Apache-2.0, the repository's licence. + +## Releases + +Only maintainers release; see [docs/releasing.md](docs/releasing.md). + +## Conduct + +See [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md). diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..1dace5f --- /dev/null +++ b/NOTICE @@ -0,0 +1,5 @@ +Judgment Pack Devkit +Copyright 2026 The Judgment Pack contributors + +This product is the development kit of the Judgment Pack repositories +(https://github.com/Judgment-Pack), licensed under the Apache License, Version 2.0. diff --git a/README.md b/README.md index 9cb0f59..0d9b998 100644 --- a/README.md +++ b/README.md @@ -11,19 +11,56 @@ needs, its setup steps and its checks. The kit holds the schema, the public runn repository vendors, the agent instructions every repository carries, the CI step list and the operating policy. +## Public + +This repository is public. It is for the people and agent sessions who work on the Judgment Pack +repositories, and for anyone who wants the same pattern in their own: a repository lists its checks +in one manifest, anyone can run them with one vendored script and python3 alone, and a machine that +installs the kit adds resource caps and a lock around the same steps. Machine settings never live in +the kit: paths, caps and the list of names a posted record must not carry are read from the +person's own config file and environment (`config/env.example` documents each), and a tracked file +holds no path of any machine. See [CONTRIBUTING.md](CONTRIBUTING.md) to help, +[SECURITY.md](SECURITY.md) to report a vulnerability, and [docs/policy.md](docs/policy.md) for how +agent work on these repositories is run. + ## Install +From a release, after verifying it (the commands are in [docs/releasing.md](docs/releasing.md)): + +```sh +gh release download vX.Y.Z --repo Judgment-Pack/judgment-pack-devkit --dir release +sha256sum --check --strict release/checksums.txt +tar -xzf release/judgment-pack-devkit-X.Y.Z.tgz +ln -s "$PWD/judgment-pack-devkit-X.Y.Z/bin/jpdev" ~/.local/bin/jpdev +``` + +From a clone: + ```sh git clone https://github.com/Judgment-Pack/judgment-pack-devkit cd judgment-pack-devkit ln -s "$PWD/bin/jpdev" ~/.local/bin/jpdev +``` + +Then the settings, which are optional: + +```sh mkdir -p ~/.config/jpdev && cp config/env.example ~/.config/jpdev/env jpdev env ``` +## Releases + +Each release is a tarball and its `checksums.txt`, built by +[`release.yml`](.github/workflows/release.yml) from a signed tag on `main`, attested with a +build-provenance attestation, and published only after a maintainer approves it. +`gh attestation verify` checks a file against the workflow and the tag; the exact command, and what +the workflow does at each step, are in [docs/releasing.md](docs/releasing.md). + Needs bash 5, git, gh (logged in), python3, rsync and, for Go steps, flock. Node versions are read from nvm; a python version the manifest names is read from `python3.N` on PATH or from -pyenv. +pyenv; a Go version the manifest names is the go on PATH when it matches, else one under +`JPDEV_GO_SDK_DIR`, and `jpdev` says so when none does. ## Settings @@ -38,13 +75,14 @@ the same name overrides it. `config/env.example` documents each setting with its | `JPDEV_GO_ULIMIT_KB`, `JPDEV_GO_TIMEOUT_S` | 12000000, 3000 | the Go address-space cap and timeout | | `JPDEV_PY_ULIMIT_KB` | 4000000 | the python address-space cap | | `JPDEV_NVM_DIR` | `$NVM_DIR` or `~/.nvm` | where node versions live | +| `JPDEV_GO_SDK_DIR` | `~/sdk` | where a Go release the manifest names is looked for, when the go on PATH is another (`sdk/go1.26.5/bin/go`) | | `JPDEV_NAME_PATTERNS` | `~/.config/jpdev/name-patterns` | regexes the record gate refuses | ## Commands - `jpdev setup DIR [--repo OWNER/NAME] [--branch NAME] [--base origin/main] [--manifest FILE]`: clones with `gh repo clone` (or reuses `DIR`), makes the branch from the base, selects the - manifest's toolchains (node by nvm version, go, python and its virtual environment), runs the + manifest's toolchains (node by nvm version, go by its version when the manifest names one, python and its virtual environment), runs the setup steps under the caps and prints one summary block. - `jpdev check [--only NAME,...] [--skip NAME,...] [--all] [--manifest FILE]`: runs the checks in order, each under its toolchain's cap, and stops at the first failure unless `--all`. The @@ -66,8 +104,12 @@ the same name overrides it. `config/env.example` documents each setting with its | --- | --- | | `node` | `lib/rsscap.sh`: the process group's resident memory and a timeout; never `ulimit -v`, which breaks vite and vitest | | `go` | `flock` on the machine lock, `ulimit -v`, `GOFLAGS=-mod=mod`, a per-clone `GOCACHE`, `timeout` | -| `python` | `ulimit -v` and `timeout`; the clone's virtual environment first on PATH | -| `shell` | none | +| `python` | `ulimit -v` and `timeout` | +| `shell` | `timeout` only | + +When the manifest declares a virtual environment, its `bin` is first on PATH for every step, whatever +the step's toolchain (the toolchain still decides the cap), so a node or Go step that runs a python +script uses the environment's interpreter and packages. ## What is here @@ -81,5 +123,8 @@ the same name overrides it. `config/env.example` documents each setting with its repository adopts the kit. `examples/judgment-pack-agent.dev.json`: a worked manifest. - `tests/run.sh`: the kit's own tests (`tests/fixture/` holds a manifest of shell echoes and one deliberate failure). +- `dev.json`, `scripts/dev/check.sh`, `AGENTS.md`, `CLAUDE.md`: the kit on itself, the list of checks + CI runs. `CONTRIBUTING.md`, `SECURITY.md`, `docs/releasing.md`: contributing, reporting and + releasing. Apache-2.0; see `LICENSE`. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..3d4ee7d --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,29 @@ +# Security policy + +## Reporting a vulnerability + +Do not open a public issue for a vulnerability. Use GitHub's private vulnerability reporting for +this repository (Security -> Report a vulnerability), or open a minimal non-sensitive issue asking +a maintainer to establish a private channel. + +Report, for example: + +- a way for `jpdev record post` to publish a local path, or a line that matches the name-pattern + file, which it is meant to refuse; +- a way for a step to run something the repository's `dev.json` does not name, or outside the caps + its toolchain sets; +- a way for the release workflow to publish files other than the ones the run built from the tagged + commit, or to publish without the maintainer's approval at the `release` environment; +- a release file whose attestation does not verify against the release workflow and the tag; +- a setting, token or credential of the person running the kit reaching a log, a summary block or + a posted record. + +Include the command, the kit version and a reproduction. We confirm receipt within five working +days. + +## Scope + +The kit runs the commands a repository's `dev.json` names, as the person who runs it. A manifest +is code: running `jpdev check` or `scripts/dev/check.sh` in a repository you do not trust runs that +repository's commands. The caps limit memory and time, not authority, and are not a sandbox; a +report that only shows a manifest running its own commands is out of scope. diff --git a/bin/jpdev b/bin/jpdev index 18220ad..e3649e6 100755 --- a/bin/jpdev +++ b/bin/jpdev @@ -88,9 +88,15 @@ cmd_setup() { fi : "${manifest:=$root/dev.json}" jpdev_prepare_run "$root" "$manifest" - local go_v py_v venv pyv py + local go_v py_v venv pyv py gov if [ -n "$(jpdev_manifest_get "$manifest" toolchains go)" ]; then - go_v=$(go version 2>/dev/null) || jpdev_die "setup: the manifest names go, which is not on PATH" + go_v=$("${JPDEV_GO_BIN:+$JPDEV_GO_BIN/}go" version 2>/dev/null) || jpdev_die "setup: the manifest names go, which is not on PATH" + gov=$(jpdev_manifest_get "$manifest" toolchains go version) + if [ -n "$gov" ] && [ -n "${JPDEV_GO_BIN:-}" ]; then + go_v="$go_v (selected for the manifest's go $gov)" + elif [ -n "$gov" ]; then + go_v="$go_v (the manifest names go $gov and no matching Go is installed: this one is used)" + fi fi venv=$(jpdev_manifest_get "$manifest" toolchains python venv) pyv=$(jpdev_manifest_get "$manifest" toolchains python version) diff --git a/config/env.example b/config/env.example index 3e73d00..c6553bb 100644 --- a/config/env.example +++ b/config/env.example @@ -23,6 +23,10 @@ # The nvm directory node versions are read from ($NVM_DIR when set, else $HOME/.nvm). #JPDEV_NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +# Where Go releases are looked for when a manifest names a Go version (go.version in dev.json) and +# the go on PATH is another release: sdk/go1.26.5/bin/go, the layout golang.org/dl leaves. +#JPDEV_GO_SDK_DIR="$HOME/sdk" + # A file of extended regular expressions, one per line (# comments and blank lines ignored), # that `jpdev record post` refuses to post: put the personal names that must not appear in # public text there. `jpdev record post` refuses to post without this file (or --no-name-check), diff --git a/dev.json b/dev.json new file mode 100644 index 0000000..74e23ed --- /dev/null +++ b/dev.json @@ -0,0 +1,13 @@ +{ + "$schema": "./templates/dev.schema.json", + "version": 1, + "checks": [ + {"name": "diff-check", "run": "git diff --check \"$(git hash-object -t tree /dev/null)\"", "toolchain": "shell"}, + {"name": "local-paths", "run": "if git grep -nE '(^|[^A-Za-z0-9_])/(home|tmp)/[A-Za-z0-9_.-]+'; then echo 'a machine-local path is in tracked text'; exit 1; fi", "toolchain": "shell"}, + {"name": "agent-docs", "run": "scripts/sync-agent-docs.sh --check && scripts/sync-agent-docs.sh --check . && head -c \"$(wc -c < templates/AGENTS.md)\" AGENTS.md | cmp - templates/AGENTS.md", "toolchain": "shell", "summary": "^sync-agent-docs:"}, + {"name": "bash-n", "run": "set -eu; for file in bin/* lib/*.sh templates/*.sh scripts/*.sh scripts/dev/*.sh tests/*.sh tests/fixture/*.sh; do bash -n \"$file\"; done; echo 'bash -n: every script parses'", "toolchain": "shell", "summary": "^bash -n:"}, + {"name": "json", "run": "python3 -I -c 'import glob, json; files = sorted(glob.glob(\"*.json\") + glob.glob(\"templates/*.json\") + glob.glob(\"examples/*.json\") + glob.glob(\"tests/fixture/*.json\")); [json.load(open(f, encoding=\"utf-8\")) for f in files]; print(\"parsed %d JSON documents\" % len(files))'", "toolchain": "shell", "summary": "^parsed "}, + {"name": "shellcheck", "run": "shellcheck -x -P \".:SCRIPTDIR\" bin/* lib/*.sh templates/*.sh scripts/*.sh scripts/dev/*.sh tests/*.sh tests/fixture/*.sh && echo 'shellcheck: clean'", "toolchain": "shell", "summary": "^shellcheck:"}, + {"name": "tests", "run": "tests/run.sh", "toolchain": "shell", "summary": "^[0-9]+ passed, [0-9]+ failed", "timeout": 900} + ] +} diff --git a/docs/adopting.md b/docs/adopting.md index f62e7d1..90bc923 100644 --- a/docs/adopting.md +++ b/docs/adopting.md @@ -8,8 +8,12 @@ machine that has the kit adds the caps and the lock around the same steps. Start from `templates/dev.json` and keep what the repository runs. The manifest names: -- `toolchains`: `node` (`{"version": "22"}`, resolved from nvm), `go` (`{}`), `python` - (`{"version": "3.12", "venv": ".venv"}`; the version and the virtual environment are optional). +- `toolchains`: `node` (`{"version": "22"}`, resolved from nvm), `go` (`{}`, or + `{"version": "1.26.5"}` to name the Go release the checks are measured on: CI installs it, + `jpdev` selects an installed Go that matches it and says when none is installed, and the + workflow falls back to the `go.mod` version when the manifest names none), `python` + (`{"version": "3.12", "venv": ".venv"}`; the version and the virtual environment are optional; + a declared virtual environment is first on PATH for every step). - `setup`: steps `jpdev setup` and CI run once after a clone (`npm ci`, the python requirements). - `checks`: the check list, in the order it should run (cheap checks first). @@ -35,6 +39,10 @@ mkdir -p scripts/dev cp templates/check.sh scripts/dev/check.sh ``` +Line 2 of the template, `# from judgment-pack-devkit vX.Y.Z (templates/check.sh)`, records the kit +release the copy came from. A vendored copy keeps that line, so a later reader can tell which release +it is behind; when you update the copy, copy the whole file again. + `scripts/dev/check.sh [--setup] [--only A,B] [--skip A,B] [--all]` runs the manifest with no caps and prints the summary block. `jpdev check` uses the vendored copy when it exists and sets `JPDEV_WRAP` so that each step runs under its toolchain's cap. diff --git a/docs/releasing.md b/docs/releasing.md new file mode 100644 index 0000000..94f63f2 --- /dev/null +++ b/docs/releasing.md @@ -0,0 +1,190 @@ +# Releasing the kit + +Only maintainers release this repository. A release is a tag a person pushes on a commit already on +`main`. [`release.yml`](../.github/workflows/release.yml) packages the kit from that commit and +publishes no release until a maintainer approves it. + +There is no registry and no Go module. A person installs the attested tarball (see the README). + +## What a release is + +- `judgment-pack-devkit-.tgz`: `bin/`, `lib/`, `templates/`, `config/`, `scripts/`, `docs/`, + `README.md`, `LICENSE` and `NOTICE` of the tagged commit, in one directory named + `judgment-pack-devkit-/`, where `` is the tag without its `v`. The tarball is + written with `git archive` and `gzip -n`, so the same commit gives the same bytes. +- `checksums.txt`: the SHA-256 of the tarball, in the form `sha256sum` writes. +- A build-provenance attestation whose subjects are both files, each by its own digest, signed by + GitHub for this workflow at the tagged commit. +- The notes a person wrote, from `docs/releases/.md`. + +## Once, before the first release the workflow makes + +Both are done by a maintainer. The workflow reads the first one's reviewer rule and does not read +the second. + +1. The `release` environment exists with a required reviewer. The workflow reads the rule before it + writes anything and refuses a repository without it, because GitHub creates a missing + environment on first use with no protection. It reads that a reviewer rule exists, not who the + reviewers are, and it cannot stop an administrator from changing the rule later. With one + maintainer, the person who pushed the tag is the person who approves: the gate is a pause to read + the draft, not a second person. + + ```sh + gh api -X PUT repos/Judgment-Pack/judgment-pack-devkit/environments/release \ + -F 'reviewers[][type]=User' -F "reviewers[][id]=$(gh api user --jq .id)" + ``` + +2. **Immutable releases are turned on by the maintainer** in the repository settings (Settings, + General, Releases). It locks the tag and the assets once a release is published, not before. + Until it is on, the workflow compares the tag with the commit the run was started for, and a + published release can still be changed. + +## Prepare the release + +1. Open a pull request that: + - sets `JPDEV_VERSION` in `lib/common.sh` to the release's version, and line 2 of + `templates/check.sh` to `# from judgment-pack-devkit v (templates/check.sh)`, and + copies the template to `scripts/dev/check.sh` (a test holds the three together); + - writes `docs/releases/.md`: what the release holds and what it does not. It is the text + the release page carries, and the first line ends with the tag's name, for example + `# jpdev v0.2.0`. +2. Run the check list once locally, as `ci.yml` does: + + ```sh + scripts/dev/check.sh + ``` + +3. Merge the pull request. + +## Tag + +Tag the merge commit on `main`, signed, and push only that tag: + +```sh +git fetch origin +git tag -s -m "jpdev " "$(git rev-parse origin/main)" +git push origin +``` + +If no signing key is configured, stop and settle the signing policy. Do not replace a signed tag +with an unsigned one. Signing is the maintainer's practice; the workflow does not check it. + +The tag must be annotated (`-s` makes it so): the workflow refuses a lightweight tag, reading the +tag's object type. A tag is `vMAJOR.MINOR.PATCH`, in ASCII digits with no leading zeroes, and +nothing after: no prerelease and no build metadata. A tag is never moved or reused. A fix is a new +release. + +## What the workflow does + +Every job works on the commit the run was started for, by its digest, not on the tag's name. + +1. **Admits the tag.** It is in the grammar; it is an annotated tag; it names the commit the run + was started for; that commit is on `main`; `docs/releases/.md` exists, is not empty and ends + its first line with the tag; and the `release` environment has a reviewer rule. +2. **Runs CI at that commit.** The release calls the commit's own + [`ci.yml`](../.github/workflows/ci.yml): `lint` and `tests`. A check `main` has gained since that + commit is not run. +3. **Packages the kit.** It refuses unless `bin/jpdev --version` prints `jpdev `, line 2 of + `templates/check.sh` names ``, and `scripts/dev/check.sh` is a copy of the template. Then + `git archive` of the commit into `judgment-pack-devkit-.tgz`, `checksums.txt` written + over it and checked, and the unpacked tarball run once (`jpdev --version`, `jpdev env`). +4. **Attests and drafts.** Only if the tag still names the commit and no release exists under the + tag, both files are attested, each by its own path, and a draft release is created with the + notes from the commit. The job records the draft's release id for the publish job. +5. **Waits at the `release` gate.** Review the draft on the Releases page: the notes and the two + assets. Approve the pending deployment on the run. +6. **Compares the draft with the run, and publishes it by its id.** A draft can be changed while it + waits, by anyone who can write to the repository. After the gate, the tag is compared with the + commit once more, and the run's own files are held to their checksums. The release the run + drafted is then read through the releases API by its id, without downloading anything: it must + still be a draft under the tag, with the title the workflow gave it and the notes the commit + holds, and the digest the API reports for each of its assets must be the digest of the run's + file of that name, no asset more or fewer. Any difference refuses. The publish call follows that + last read at once, addressed to the same release id, not to the tag, and marks the release + latest. +7. **Reads the published release again.** Immediately after publication the job reads the + published release's asset digests through the API once more, and fails the step `Require the + published release to hold what this run wrote`, printing both digest sets, if they differ from + the run's files. A read that fails is a separate failure, "could not verify the published + release", and also fails the step. + +An approval given more than thirty days after the run finds the run's files gone, and the job +fails: release a new version. + +### The window that remains + +The API cannot publish a release on condition that its assets are the ones just read, so one +window remains: between the last read of the draft's asset digests and the publish call, a few +moments of the same step. Whoever can write to the repository's releases can act in it, which in +this organisation is a maintainer, the same people the `release` gate asks to approve. Nothing in +the workflow closes that window. A file replaced in it is published, and the read after publication +then fails the run by name; with release immutability on, the published release can no longer be +changed, so the replacement stays visible as a failed release and is not quietly corrected. + +No maintainer token and no repository secret is used. + +### What exists before the approval + +The gate is on the release. Three things exist before it and are not secret: + +- the files, as an artifact of the workflow run, downloadable for thirty days by anyone who can read + the repository's runs; +- the attestation of the tarball and `checksums.txt`, recorded in a public transparency log when it + is made. It says which workflow built the files, from which commit. It does not say a maintainer + approved them; +- the draft, visible to those who can write to the repository. + +### If a run fails + +Fix the cause on `main` and release a new version. Re-running is for a failure that was the hosted +runner's: re-run the failed jobs, not all jobs. + +What to do with a draft depends on which job failed: + +| The job that failed | The draft | What to do | +| --- | --- | --- | +| Any job before `Attest and draft release` | none was made | re-run the failed jobs; where the version did not match the tag, fix it on `main` and release a new version | +| `Attest and draft release` | may exist, and may lack assets | read it, delete it by hand, then re-run the failed jobs. The job refuses to run while a release under the tag exists, draft or published, and refuses when it cannot find out | +| `Publish release`, before publication | needs looking at: it may be as it was made, changed, incomplete or gone | read why the job failed, and look at the Releases page. **Keep** a draft that is as it was made, and re-run the failed jobs: that job only publishes the release id the run drafted. Do not publish a draft that was changed or is incomplete. Where the draft cannot be published, release a new version | +| `Publish release`, at `Require the published release to hold what this run wrote` | published, holding files this run did not write, or not published after the call | someone who can write to the repository changed the release in the window above. Treat it as an incident: tell the maintainers, point no one at the release, and release a new version from a new tag. Do not re-run | + +Delete a draft with `gh release delete --repo Judgment-Pack/judgment-pack-devkit`, which +leaves the tag. A published release is never deleted to make room for another. + +## Verifying a release by hand + +Both files are attestation subjects, and `checksums.txt` also names the tarball's digest. With the +release's files downloaded into a directory of their own: + +```sh +git clone https://github.com/Judgment-Pack/judgment-pack-devkit.git devkit +commit="$(git -C devkit rev-parse '^{commit}')" +gh release download --repo Judgment-Pack/judgment-pack-devkit --dir release- +cd release- +for file in judgment-pack-devkit-.tgz checksums.txt; do + gh attestation verify "${file}" \ + --repo Judgment-Pack/judgment-pack-devkit \ + --signer-workflow Judgment-Pack/judgment-pack-devkit/.github/workflows/release.yml \ + --source-ref refs/tags/ \ + --source-digest "${commit}" \ + --format json +done +sha256sum --check --strict checksums.txt +``` + +The loop checks each file against the attestation made in this repository, by this workflow, in a +run started for that tag, from that commit; with `--repo` alone it would check the repository and +nothing more. The checksum command checks the tarball against `checksums.txt`, itself an attested +file. `--format json` is for scripts: `gh attestation verify` prints nothing when its output is not +a terminal. On macOS use `shasum -a 256 --check checksums.txt`. + +To check that the tarball is what the tagged source packages, build it from the tag and compare. +The packing is reproducible, so the digests are equal: + +```sh +git -C devkit archive --format=tar --prefix=judgment-pack-devkit-/ '^{commit}' \ + bin lib templates config scripts docs README.md LICENSE NOTICE | gzip -n -9 > rebuilt.tgz +sha256sum rebuilt.tgz judgment-pack-devkit-.tgz +``` + +Neither check says a maintainer approved the release, which its being published says. diff --git a/examples/judgment-pack-agent.dev.json b/examples/judgment-pack-agent.dev.json index 3481c24..91b1786 100644 --- a/examples/judgment-pack-agent.dev.json +++ b/examples/judgment-pack-agent.dev.json @@ -3,7 +3,7 @@ "version": 1, "toolchains": { "node": {"version": "22"}, - "go": {}, + "go": {"version": "1.26.5"}, "python": {"version": "3.12", "venv": ".venv"} }, "setup": [ diff --git a/lib/common.sh b/lib/common.sh index d2d5b94..2511f08 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -7,7 +7,7 @@ JPDEV_VERSION=0.1.0 # Every setting jpdev reads. The environment wins over the config file, the config file over the default. JPDEV_SETTINGS=(JPDEV_SCRATCH JPDEV_LOCK JPDEV_NODE_RSS_MB JPDEV_NODE_TIMEOUT_S JPDEV_GO_ULIMIT_KB - JPDEV_GO_TIMEOUT_S JPDEV_PY_ULIMIT_KB JPDEV_NVM_DIR JPDEV_NAME_PATTERNS) + JPDEV_GO_TIMEOUT_S JPDEV_PY_ULIMIT_KB JPDEV_NVM_DIR JPDEV_GO_SDK_DIR JPDEV_NAME_PATTERNS) jpdev_log() { printf 'jpdev: %s\n' "$*" >&2; } jpdev_die() { printf 'jpdev: %s\n' "$*" >&2; exit 2; } @@ -26,6 +26,7 @@ jpdev_default() { JPDEV_GO_TIMEOUT_S) printf 3000 ;; JPDEV_PY_ULIMIT_KB) printf 4000000 ;; JPDEV_NVM_DIR) printf '%s' "${NVM_DIR:-$HOME/.nvm}" ;; + JPDEV_GO_SDK_DIR) printf '%s/sdk' "$HOME" ;; JPDEV_NAME_PATTERNS) printf '%s/jpdev/name-patterns' "${XDG_CONFIG_HOME:-$HOME/.config}" ;; esac } @@ -71,6 +72,28 @@ jpdev_node_bin() { printf '%s/bin' "$best" } +# jpdev_go_matches GO VERSION succeeds when the go command GO is release VERSION: 1.26 matches go1.26 +# and go1.26.5, 1.26.5 matches go1.26.5 alone. +jpdev_go_matches() { + local have + have=$("$1" version 2>/dev/null | awk '{ print $3 }') || return 1 + have=${have#go} + [ "$have" = "$2" ] || [[ $have == "$2".* ]] +} + +# jpdev_go_bin VERSION prints the bin directory of an installed Go of that release, or fails: the +# go on PATH when it matches, else the newest match under $JPDEV_GO_SDK_DIR (the layout +# `go install golang.org/dl/go1.26.5@latest && go1.26.5 download` leaves: sdk/go1.26.5/bin/go). +jpdev_go_bin() { + local found entry best="" + if found=$(command -v go) && jpdev_go_matches "$found" "$1"; then dirname "$found"; return 0; fi + for entry in "$JPDEV_GO_SDK_DIR"/go"$1" "$JPDEV_GO_SDK_DIR"/go"$1".*; do + if [ -x "$entry/bin/go" ] && jpdev_go_matches "$entry/bin/go" "$1"; then best=$(printf '%s\n%s\n' "$best" "$entry" | sort -V | tail -n 1); fi + done + [ -n "$best" ] || return 1 + printf '%s/bin' "$best" +} + # jpdev_python_bin MAJOR.MINOR prints the path of a working python of that version: python3.N on # PATH when it runs (a pyenv shim for a version that is not selected does not), else the newest # one under $PYENV_ROOT/versions (default ~/.pyenv/versions). @@ -111,9 +134,11 @@ print(v if isinstance(v, (str, int, float)) else json.dumps(v))' "$@" jpdev_git_net() { git -c credential.helper= -c 'credential.helper=!gh auth git-credential' "$@"; } # jpdev_prepare_run ROOT MANIFEST validates the manifest and exports what lib/wrap.sh and -# templates/check.sh read: JPDEV_ROOT, JPDEV_WRAP and JPDEV_NODE_BIN. +# templates/check.sh read: JPDEV_ROOT, JPDEV_WRAP, JPDEV_NODE_BIN and JPDEV_GO_BIN. When the manifest's +# go toolchain names a version, JPDEV_GO_BIN is the bin directory of an installed Go of that release; +# when none is installed it is unset, jpdev says so, and the steps run with the go on PATH. jpdev_prepare_run() { - local root=$1 manifest=$2 kit nodev + local root=$1 manifest=$2 kit nodev gov have kit=$(jpdev_kit_dir) [ -f "$manifest" ] || jpdev_die "no manifest at $manifest" command -v python3 >/dev/null || jpdev_die "python3 is needed to read the manifest" @@ -121,6 +146,17 @@ jpdev_prepare_run() { export JPDEV_ROOT=$root JPDEV_WRAP=$kit/lib/wrap.sh JPDEV_VENV=$(jpdev_manifest_get "$manifest" toolchains python venv) export JPDEV_VENV + unset JPDEV_GO_BIN + gov=$(jpdev_manifest_get "$manifest" toolchains go version) + if [ -n "$gov" ]; then + if JPDEV_GO_BIN=$(jpdev_go_bin "$gov"); then + export JPDEV_GO_BIN + else + unset JPDEV_GO_BIN + have=$(go version 2>/dev/null | awk '{ print $3 }') + jpdev_log "the manifest names go $gov and no matching Go is installed (the go on PATH is ${have:-absent}, and none is under $JPDEV_GO_SDK_DIR); the go steps run with the go on PATH" + fi + fi nodev=$(jpdev_manifest_get "$manifest" toolchains node version) if [ -n "$nodev" ]; then JPDEV_NODE_BIN=$(jpdev_node_bin "$nodev") || jpdev_die "node $nodev is not installed under $JPDEV_NVM_DIR (nvm install $nodev)" diff --git a/lib/validate_manifest.py b/lib/validate_manifest.py index c9d16fa..5b274af 100644 --- a/lib/validate_manifest.py +++ b/lib/validate_manifest.py @@ -3,10 +3,11 @@ Checks a dev.json against its JSON Schema when the jsonschema package is installed (it says which check ran), and always makes the checks the schema cannot: step names are unique, a node -step has a node version to run under, a setup step's name is not a check's. Standard library -otherwise. Exit 0 when valid, 1 with one line per problem, 2 on an unreadable file. +step has a node version to run under. The go version is also held to a release number here, +because the fallback without the jsonschema package has no schema. Standard library otherwise. Exit 0 when valid, 1 with one line per problem, 2 on an unreadable file. """ import json +import re import sys @@ -48,6 +49,9 @@ def main(): steps = [s for k in ("setup", "checks") for s in manifest.get(k, []) if isinstance(s, dict)] if isinstance(manifest.get("parity"), dict): steps.append(dict(manifest["parity"], name="parity")) + go_version = manifest.get("toolchains", {}).get("go", {}).get("version") if isinstance(manifest.get("toolchains"), dict) and isinstance(manifest["toolchains"].get("go"), dict) else None + if go_version is not None and not (isinstance(go_version, str) and re.fullmatch(r"[0-9]+\.[0-9]+(\.[0-9]+)?", go_version)): + problems.append("toolchains/go/version: %r is not a Go release number such as 1.26 or 1.26.5" % (go_version,)) seen = set() for step in steps: name = step.get("name") diff --git a/lib/wrap.sh b/lib/wrap.sh index fd189e9..981d0d6 100755 --- a/lib/wrap.sh +++ b/lib/wrap.sh @@ -6,7 +6,8 @@ # python ulimit -v and timeout # shell timeout only # JPDEV_STEP_TIMEOUT (seconds) replaces the toolchain's default timeout for this one command. -# JPDEV_NODE_BIN, when set, is put first on PATH; so is /$JPDEV_VENV/bin for python. +# PATH, in order: /$JPDEV_VENV/bin when the manifest declares a virtual environment, for +# every toolchain (the toolchain still decides the cap); then JPDEV_GO_BIN and JPDEV_NODE_BIN when set. set -u here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=lib/common.sh @@ -17,9 +18,10 @@ if ! { [ $# -ge 3 ] && [ "$2" = "--" ]; }; then echo "usage: wrap.sh TOOLCHAIN - toolchain=$1; shift 2 step_timeout=${JPDEV_STEP_TIMEOUT:-} +if [ -n "${JPDEV_GO_BIN:-}" ]; then PATH=$JPDEV_GO_BIN:$PATH; fi if [ -n "${JPDEV_NODE_BIN:-}" ]; then PATH=$JPDEV_NODE_BIN:$PATH; fi clone=${JPDEV_ROOT:-$(git rev-parse --show-toplevel 2>/dev/null || pwd)} -if [ "$toolchain" = python ] && [ -n "${JPDEV_VENV:-}" ] && [ -d "$clone/$JPDEV_VENV/bin" ]; then PATH=$clone/$JPDEV_VENV/bin:$PATH; fi +if [ -n "${JPDEV_VENV:-}" ] && [ -d "$clone/$JPDEV_VENV/bin" ]; then PATH=$clone/$JPDEV_VENV/bin:$PATH; fi export PATH case $toolchain in diff --git a/scripts/dev/check.sh b/scripts/dev/check.sh new file mode 100755 index 0000000..66d8986 --- /dev/null +++ b/scripts/dev/check.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# from judgment-pack-devkit v0.1.0 (templates/check.sh) +# scripts/dev/check.sh [--setup] [--only a,b] [--skip a,b] [--all] [--manifest FILE] +# Runs the checks of dev.json in order and prints one summary block; exit status = failures. +# Stops at the first failure unless --all. --setup runs the manifest's setup steps instead. +# A step whose requires_env variable is empty is skipped, and said to be; when CI is set that is a +# failure unless the step is marked optional: true. Standard library python3 only. jpdev sets JPDEV_WRAP to add its caps around each step. +root=${JPDEV_ROOT:-$(cd "$(dirname "$0")" && git rev-parse --show-toplevel)} || exit 2 +exec python3 -I - "$root" "$@" <<'PY' +import json, os, re, signal, subprocess, sys, time +root, args = sys.argv[1], sys.argv[2:] +opt = {'only': set(), 'skip': set(), 'all': False, 'setup': False, 'manifest': root + '/dev.json'} +while args: + a = args.pop(0) + if a in ('--only', '--skip'): opt[a[2:]] = set(args.pop(0).split(',')) + elif a == '--manifest': opt['manifest'] = args.pop(0) + elif a in ('--all', '--setup'): opt[a[2:]] = True + else: sys.exit('check.sh: unknown argument ' + a) +m = json.load(open(opt['manifest'], encoding='utf-8')) +steps = m.get('setup', []) if opt['setup'] else m['checks'] + ([dict(name='parity', **m['parity'])] if 'parity' in m else []) +unknown = (opt['only'] | opt['skip']) - set(s['name'] for s in steps) +if unknown: print('check.sh: no step named ' + ', '.join(sorted(unknown)), file=sys.stderr); sys.exit(2) +wrap, logdir, rows, failed = os.environ.get('JPDEV_WRAP'), os.environ.get('JPDEV_LOGDIR'), [], 0 +for c in steps: + n = c['name'] + if (opt['only'] and n not in opt['only']) or n in opt['skip']: continue + if failed and not opt['all']: rows.append((n, 'NOT RUN', '', [])); continue + if c.get('requires_env') and not os.environ.get(c['requires_env']): + # A skip is a failure under CI (CI is set) unless the step says optional: true. + if os.environ.get('CI') and not c.get('optional'): failed += 1; rows.append((n, 'FAIL', 'needs %s, which CI must set (or mark the step optional)' % c['requires_env'], [])) + else: rows.append((n, 'SKIP', 'needs ' + c['requires_env'], [])) + continue + cmd = ['bash', '-c', c['run']] + if wrap: cmd = [wrap, c.get('toolchain', 'shell'), '--'] + cmd + tmo = c.get('timeout', 1200) + t0 = time.time(); print('check: running ' + n, file=sys.stderr, flush=True) + p = subprocess.Popen(cmd, cwd=os.path.join(root, c.get('cwd', '.')), stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, errors='replace', start_new_session=True, env=dict(os.environ, JPDEV_STEP_TIMEOUT=str(tmo))) + try: out, _ = p.communicate(timeout=None if wrap else tmo) + except subprocess.TimeoutExpired: os.killpg(p.pid, signal.SIGKILL); out = p.communicate()[0] + '\ncheck: stopped for time\n' + lines, took = out.splitlines(), '%ds' % (time.time() - t0) + if logdir: os.makedirs(logdir, exist_ok=True); open('%s/%s.log' % (logdir, n), 'w').write(out) + hit = [l.strip() for l in lines if c.get('summary') and re.search(c['summary'], l)] + note = [' ' + l for l in (hit if len(hit) <= 3 else ['%d lines matched; last: %s' % (len(hit), hit[-1])])] + if p.returncode == 0: rows.append((n, 'PASS', took, note)); continue + failed += 1 + rows.append((n, 'FAIL', '%s, exit %d' % (took, p.returncode), note + [' --'] + [' ' + l for l in lines[:40]] + ([' ... %d more lines' % (len(lines) - 40)] if len(lines) > 40 else []))) +print('== %s summary (%s) ==' % ('setup' if opt['setup'] else 'check', os.path.basename(opt['manifest']))) +for n, s, head, extra in rows: print('\n'.join(['%-8s %-24s %s' % (s, n, head)] + extra)) +print('== %d passed, %d failed, %d skipped, %d not run ==' % tuple(sum(1 for r in rows if r[1] == k) for k in ('PASS', 'FAIL', 'SKIP', 'NOT RUN'))) +sys.exit(min(failed, 255)) +PY diff --git a/templates/check.sh b/templates/check.sh index 762a3dc..66d8986 100755 --- a/templates/check.sh +++ b/templates/check.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# from judgment-pack-devkit v0.1.0 (templates/check.sh) # scripts/dev/check.sh [--setup] [--only a,b] [--skip a,b] [--all] [--manifest FILE] # Runs the checks of dev.json in order and prints one summary block; exit status = failures. # Stops at the first failure unless --all. --setup runs the manifest's setup steps instead. diff --git a/templates/ci-check.yml b/templates/ci-check.yml index d70c767..a0e2da4 100644 --- a/templates/ci-check.yml +++ b/templates/ci-check.yml @@ -29,6 +29,7 @@ jobs: tc = m.get("toolchains", {}) print("node=" + tc.get("node", {}).get("version", "")) print("go=" + ("true" if "go" in tc else "false")) + print("goversion=" + tc.get("go", {}).get("version", "")) print("python=" + tc.get("python", {}).get("version", "3.12")) PY echo "gomod=$(git ls-files go.mod '*/go.mod' | head -n 1)" >> "$GITHUB_OUTPUT" @@ -36,8 +37,15 @@ jobs: if: steps.dev.outputs.node != '' with: node-version: ${{ steps.dev.outputs.node }} + # The Go release is the manifest's (toolchains.go.version); a manifest that names none falls + # back to the go directive of the repository's go.mod. - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - if: steps.dev.outputs.go == 'true' + if: steps.dev.outputs.go == 'true' && steps.dev.outputs.goversion != '' + with: + go-version: ${{ steps.dev.outputs.goversion }} + cache: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + if: steps.dev.outputs.go == 'true' && steps.dev.outputs.goversion == '' with: go-version-file: ${{ steps.dev.outputs.gomod }} cache: false diff --git a/templates/dev.json b/templates/dev.json index 2e35d12..bcaa9d8 100644 --- a/templates/dev.json +++ b/templates/dev.json @@ -3,7 +3,7 @@ "version": 1, "toolchains": { "node": {"version": "22"}, - "go": {}, + "go": {"version": "1.26.5"}, "python": {"version": "3.12", "venv": ".venv"} }, "setup": [ diff --git a/templates/dev.schema.json b/templates/dev.schema.json index 0cb058b..9906857 100644 --- a/templates/dev.schema.json +++ b/templates/dev.schema.json @@ -33,7 +33,14 @@ }, "go": { "type": "object", - "additionalProperties": false + "additionalProperties": false, + "properties": { + "version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+(\\.[0-9]+)?$", + "description": "the Go release the checks are measured on (1.26 or 1.26.5); CI installs it, jpdev selects an installed Go that matches it and says when none does; absent: the go on PATH, and CI reads the version from go.mod" + } + } }, "python": { "type": "object", @@ -42,7 +49,7 @@ "venv": { "type": "string", "minLength": 1, - "description": "a virtual environment in the clone, put first on PATH for python steps" + "description": "a virtual environment in the clone, put first on PATH for every step" }, "version": { "type": "string", diff --git a/tests/run.sh b/tests/run.sh index fd022b3..7b97e79 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -7,7 +7,9 @@ work=$(mktemp -d) trap 'rm -rf -- "${work:?}"' EXIT export JPDEV_SCRATCH=$work/scratch XDG_CONFIG_HOME=$work/config HOME=$work/home mkdir -p "$HOME" "$XDG_CONFIG_HOME" -unset CI JPDEV_VENV JPDEV_LOCK JPDEV_NODE_RSS_MB JPDEV_NODE_TIMEOUT_S JPDEV_GO_ULIMIT_KB JPDEV_GO_TIMEOUT_S JPDEV_PY_ULIMIT_KB JPDEV_NVM_DIR JPDEV_NAME_PATTERNS JPDEV_FIXTURE_UNSET +unset CI JPDEV_VENV JPDEV_LOCK JPDEV_NODE_RSS_MB JPDEV_NODE_TIMEOUT_S JPDEV_GO_ULIMIT_KB JPDEV_GO_TIMEOUT_S JPDEV_PY_ULIMIT_KB JPDEV_NVM_DIR JPDEV_GO_SDK_DIR JPDEV_NAME_PATTERNS JPDEV_FIXTURE_UNSET +# What jpdev check exports to its steps: a run of these tests under it must not inherit it. +unset JPDEV_ROOT JPDEV_WRAP JPDEV_NODE_BIN JPDEV_GO_BIN JPDEV_LOGDIR JPDEV_STEP_TIMEOUT jpdev=$kit/bin/jpdev fixture=$kit/tests/fixture/dev.json pass=0 fail=0 @@ -129,6 +131,102 @@ JSON out=$(cd "$vclone" && "$jpdev" check 2>/dev/null) && s=0 || s=$? expect "the python step runs in the virtual environment the manifest names" 0 'from-the-venv' "$out" "$s" +# the virtual environment is first on PATH for every step, whatever its toolchain +pyclone=$work/pyclone +git init -q -b main "$pyclone" +python3 -m venv --without-pip "$pyclone/pyvenv" +purelib=$("$pyclone/pyvenv/bin/python" -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])') +printf 'WHERE = "in-the-venv"\n' > "$purelib/jpdev_venv_marker.py" +mkdir -p "$work/nvm/versions/node/v99.0.0/bin" +printf '#!/bin/sh\necho v99.0.0\n' > "$work/nvm/versions/node/v99.0.0/bin/node" +chmod +x "$work/nvm/versions/node/v99.0.0/bin/node" +cat > "$pyclone/dev.json" <<'JSON' +{"version": 1, + "toolchains": {"node": {"version": "99"}, "go": {}, "python": {"venv": "pyvenv"}}, + "checks": [ + {"name": "node-step", "run": "python3 -c 'import jpdev_venv_marker as m; print(\"node:\", m.WHERE)'", "toolchain": "node", "summary": "^node:"}, + {"name": "go-step", "run": "python3 -c 'import jpdev_venv_marker as m; print(\"go:\", m.WHERE)'", "toolchain": "go", "summary": "^go:"}, + {"name": "shell-step", "run": "python3 -c 'import jpdev_venv_marker as m; print(\"shell:\", m.WHERE)'", "toolchain": "shell", "summary": "^shell:"}, + {"name": "python-step", "run": "python3 -c 'import jpdev_venv_marker as m; print(\"python:\", m.WHERE)'", "toolchain": "python", "summary": "^python:"}]} +JSON +out=$(cd "$pyclone" && JPDEV_NVM_DIR=$work/nvm "$jpdev" check --all 2>/dev/null) && s=0 || s=$? +expect "a node step imports a module installed in the manifest's virtual environment" 0 'node: in-the-venv' "$out" "$s" +expect "a go step imports a module installed in the manifest's virtual environment" 0 'go: in-the-venv' "$out" "$s" +expect "a shell step imports a module installed in the manifest's virtual environment" 0 'shell: in-the-venv' "$out" "$s" +expect "a python step imports a module installed in the manifest's virtual environment" 0 'python: in-the-venv' "$out" "$s" +expect "the four steps pass" 0 '== 4 passed, 0 failed' "$out" "$s" +# with no virtual environment declared, no step finds the module +sed 's/, "python": {"venv": "pyvenv"}//' "$pyclone/dev.json" | sed 's/"toolchain": "python"/"toolchain": "shell"/' > "$pyclone/novenv.json" +out=$(cd "$pyclone" && JPDEV_NVM_DIR=$work/nvm "$jpdev" check --all --manifest "$pyclone/novenv.json" 2>/dev/null) && s=0 || s=$? +expect "with no virtual environment declared the module is not found, in any step" 4 '== 0 passed, 4 failed' "$out" "$s" + +# --- the Go release the manifest names (toolchains.go.version) +# The stubs are releases no real Go has, so a match on the go installed here cannot pass for them. +mkdir -p "$work/sdk/go1.77.5/bin" "$work/sdk/go1.76.9/bin" +for rel in 1.77.5 1.76.9; do + cat > "$work/sdk/go$rel/bin/go" <<'STUB' +#!/bin/sh +[ "$1" = version ] && echo "go version goRELEASE linux/amd64" +exit 0 +STUB + sed -i "s/RELEASE/$rel/" "$work/sdk/go$rel/bin/go" + chmod +x "$work/sdk/go$rel/bin/go" +done +goclone=$work/goclone +mkdir -p "$goclone" +git init -q -b main "$goclone" +for want in 1.77.5 1.77 1.76; do + cat > "$goclone/dev.json" <&1) && s=0 || s=$? + case $want in 1.76) rel=1.76.9 ;; *) rel=1.77.5 ;; esac + expect "jpdev check selects an installed Go that matches go.version $want" 0 "go version go$rel linux/amd64" "$out" "$s" +done +cat > "$goclone/dev.json" <<'JSON' +{"version": 1, "toolchains": {"go": {"version": "1.7"}}, "checks": [{"name": "which-go", "run": "go version", "toolchain": "go", "summary": "^go version"}]} +JSON +# a go on PATH is one candidate too: 1.7 is not a prefix of 1.77 to it either +mkdir -p "$work/pathgo" +cp "$work/sdk/go1.77.5/bin/go" "$work/pathgo/go" +out=$(cd "$goclone" && PATH=$work/pathgo:$PATH JPDEV_GO_SDK_DIR=$work/none "$jpdev" check 2>&1) && s=0 || s=$? +expect "go.version 1.7 does not match a go1.77.5 on PATH" 0 'the manifest names go 1\.7 and no matching Go is installed' "$out" "$s" +sed -i 's/"1\.7"/"1.77"/' "$goclone/dev.json" +out=$(cd "$goclone" && PATH=$work/pathgo:$PATH JPDEV_GO_SDK_DIR=$work/none "$jpdev" check 2>&1) && s=0 || s=$? +expect "go.version 1.77 matches a go1.77.5 on PATH" 0 'go version go1\.77\.5 linux/amd64' "$out" "$s" +absent "go.version 1.77 on PATH needs no warning" 'no matching Go is installed' "$out" +sed -i 's/"1\.77"/"1.7"/' "$goclone/dev.json" +out=$(cd "$goclone" && JPDEV_GO_SDK_DIR=$work/sdk "$jpdev" check 2>&1) && s=0 || s=$? +expect "go.version 1.7 does not match go1.76 or go1.77" 0 'the manifest names go 1\.7 and no matching Go is installed' "$out" "$s" +absent "go.version 1.7 selects no stub" 'go version go1\.7[67]\.' "$out" +cat > "$goclone/dev.json" <<'JSON' +{"version": 1, "toolchains": {"go": {"version": "1.99.0"}}, "checks": [{"name": "which-go", "run": "go version", "toolchain": "go", "summary": "^go version"}]} +JSON +out=$(cd "$goclone" && JPDEV_GO_SDK_DIR=$work/sdk "$jpdev" check 2>&1) && s=0 || s=$? +expect "jpdev check says so when no installed Go matches go.version, and runs the go on PATH" 0 'the manifest names go 1\.99\.0 and no matching Go is installed' "$out" "$s" +expect "the go steps still run when no Go matches" 0 'PASS +which-go' "$out" "$s" +absent "no stub Go is used when none matches" 'go version go1\.7[67]\.' "$out" +git -C "$goclone" add dev.json +git -C "$goclone" -c user.name=test -c user.email=test@example.com commit -q -m go +git init -q --bare -b main "$work/goorigin.git" +git -C "$goclone" remote add origin "$work/goorigin.git" +git -C "$goclone" push -q origin main +out=$(JPDEV_GO_SDK_DIR=$work/sdk "$jpdev" setup "$goclone" 2>&1) && s=0 || s=$? +expect "setup says no matching Go is installed" 0 'go +go version .*\(the manifest names go 1\.99\.0 and no matching Go is installed: this one is used\)' "$out" "$s" +sed -i 's/1\.99\.0/1.77.5/' "$goclone/dev.json" +out=$(JPDEV_GO_SDK_DIR=$work/sdk "$jpdev" setup "$goclone" 2>&1) && s=0 || s=$? +expect "setup says which Go it selected" 0 "go +go version go1\.77\.5 linux/amd64 \\(selected for the manifest's go 1\.77\.5\\)" "$out" "$s" +printf '{"version": 1, "toolchains": {"go": {"version": "latest"}}, "checks": [{"name": "a", "run": "true"}]}\n' > "$work/badgo.json" +out=$(python3 "$kit/lib/validate_manifest.py" "$work/badgo.json" "$kit/templates/dev.schema.json" 2>&1) && s=0 || s=$? +expect "the schema refuses a Go version that is not a release number" 1 "dev.json:" "$out" "$s" + +# --- the runner template records the kit version it came from +line2=$(sed -n 2p "$kit/templates/check.sh") +want="# from judgment-pack-devkit v$("$jpdev" --version | awk '{ print $2 }') (templates/check.sh)" +if [ "$line2" = "$want" ]; then ok "templates/check.sh line 2 names the kit version jpdev --version prints"; else bad "templates/check.sh line 2 names the kit version jpdev --version prints" "line 2: $line2 +wanted: $want"; fi +if cmp -s "$kit/templates/check.sh" "$kit/scripts/dev/check.sh"; then ok "scripts/dev/check.sh is the template, line 2 included"; else bad "scripts/dev/check.sh is the template, line 2 included"; fi + # --- the lock lock=$work/test.lock ( flock 9; sleep 4 ) 9> "$lock" &