From 40b8200f27cd408d8b7c04892cc751253de744b1 Mon Sep 17 00:00:00 2001 From: kikashy <35789537+kikashy@users.noreply.github.com> Date: Thu, 29 Oct 2026 04:05:37 -0400 Subject: [PATCH 1/3] Vendor Spec v0.3.0 and the retained-column leaf at its release identity Re-import the Core 0.3.0 bundle from the published v0.3.0 tag (bytes and bundle digest unchanged; the lock's ref and commit move to the tag). Remove the v0.3.0-rc.2 leaf import and import the leaf from the verified v0.3.0 archive; install its provenance pin and exact tuple, and switch the selected reader identity to 0.3.0. The installed reader refuses the candidate's release identity and selector. Co-Authored-By: Claude Opus 5.5 Signed-off-by: kikashy <35789537+kikashy@users.noreply.github.com> --- CHANGELOG.md | 2 +- CONFORMANCE.md | 7 ++-- README.md | 5 +-- VERSIONING.md | 15 +++---- docs/architecture.md | 4 +- docs/dialect-retained-column-draft.md | 2 +- ...lumn-rc2.md => dialect-retained-column.md} | 24 +++++++---- docs/releasing.md | 32 +++++++++----- internal/artifacts/artifacts_test.go | 4 +- internal/artifacts/jps/0.3.0/lock.json | 4 +- internal/cli/dialect.go | 8 ++-- internal/cli/dialect_retained_column_test.go | 4 +- internal/dialect/retained-column/README.md | 4 +- .../retained-column/official/provenance.json | 42 +++++++++---------- .../official/release-identity.json | 2 +- ...olumn.json => v0.3.0.retained-column.json} | 12 +++--- .../retained-column/official/spec/dialect.md | 4 +- .../official/spec/judgment-pack-core.md | 6 +-- .../official/spec/retained-column.md | 8 ++-- .../tools/dialect-verify-ts/src/retained.ts | 10 +++-- .../official/tools/retained_column.py | 13 +++--- internal/dialect/retained_column_artifacts.go | 6 +-- .../dialect/retained_column_artifacts_test.go | 13 ++++-- internal/dialect/retained_column_draft.go | 6 +-- ...est.go => retained_column_release_test.go} | 20 ++++++--- internal/mcp/dialect.go | 12 +++--- internal/retainedcolumn/artifacts.go | 14 ++++--- internal/retainedcolumn/artifacts_test.go | 21 +++++++++- tools/sync-retained-column/main.go | 10 ++--- tools/sync-retained-column/main_test.go | 2 +- 30 files changed, 186 insertions(+), 130 deletions(-) rename docs/{dialect-retained-column-rc2.md => dialect-retained-column.md} (85%) rename internal/dialect/retained-column/official/releases/{v0.3.0-rc.2.retained-column.json => v0.3.0.retained-column.json} (84%) rename internal/dialect/{retained_column_rc2_test.go => retained_column_release_test.go} (87%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 234daf9..92b1ee5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ All notable changes to tagged releases are documented here. and supports document validation/declared inventory coverage for Base/Extended. Generated retained values now admit up to 524288 UTF-8 bytes; asserted values remain bounded at 16000 in both profiles. Historical Draft fixtures remain pinned. - See the [RC2 reader](docs/dialect-retained-column-rc2.md) and + See the [RC2 reader](https://github.com/Judgment-Pack/judgment-pack-runtime/blob/v0.39.0-alpha.2/docs/dialect-retained-column-rc2.md) and [release notes](docs/releases/v0.39.0-alpha.2.md). Runtime's conformance claim is stated, in full and only, in diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 4fc610f..e1c0534 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -179,9 +179,10 @@ above — for every input admitted under that exact contract, not merely for the ## Core 0.3.0 preparation evidence and outstanding review -The implementation bundles exact Spec commit -`65c7cf5598a0f0fc532c80a9e2215090951104ce`, with per-file hashes and an immutable -commit provenance lock. No published Core `0.3.0` tag is represented by that pin. +The implementation bundles the published Spec tag `v0.3.0`, commit +`27cb29bf13e4a2d58d69fe19c7ebfeb762b014fc`, with per-file hashes and an immutable +provenance lock. Its bytes equal those of the earlier commit pin +`65c7cf5598a0f0fc532c80a9e2215090951104ce`; the bundle digest is unchanged. The implemented Core `0.3.0` reader/evaluator supports only `base`; the optional `extended` profile is refused as `profile-unsupported` before schema interpretation. diff --git a/README.md b/README.md index 3a8e3e2..00236ed 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,8 @@ it names are required evidence for such a claim and are not exhaustive evidence The runtime **validates documents**. It does not fetch a source, authorize an action, or establish truth, organizational authority, safety, or operational fitness. It bundles `0.1.0-draft`, `0.2.0-draft`, and Core `0.3.0`, and validates a document against its exact declaration. -Core `0.3.0` resources are pinned to Spec commit -`65c7cf5598a0f0fc532c80a9e2215090951104ce`; this is an immutable source pin, not a -published Core release tag. Core `0.3.0` support is base-only. Runtime discovery reports +Core `0.3.0` resources are imported from the published Spec tag `v0.3.0`, commit +`27cb29bf13e4a2d58d69fe19c7ebfeb762b014fc`. Core `0.3.0` support is base-only. Runtime discovery reports `evaluatorCapabilities` separately from document `supportedSpecVersions`, with `supportedProfiles: ["base"]` scoped to the Core `0.3.0` evaluator entry. This says nothing about Policy or Dialect profile support. diff --git a/VERSIONING.md b/VERSIONING.md index 87bb954..479c5d0 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -61,18 +61,19 @@ Matrix carrier versions `1`–`3` are unchanged: the explicit contract selects t portable disposition grammar, rather than adding a closed matrix member. Additive skip metadata is omitted from old suite reports when no case is skipped. -The Core `0.3.0` bundle uses full Spec commit -`65c7cf5598a0f0fc532c80a9e2215090951104ce`, not a published Core release tag. +The Core `0.3.0` bundle is imported from the published Spec tag `v0.3.0` at +commit `27cb29bf13e4a2d58d69fe19c7ebfeb762b014fc`; its bytes are those of the +earlier commit pin `65c7cf5598a0f0fc532c80a9e2215090951104ce`, unchanged. This source provenance does not by itself establish a conformance claim; see [CONFORMANCE.md](CONFORMANCE.md). Selected retained-column leaf support has a separate exact distribution/source/schema/corpus -identity. Core `specVersion: "0.3.0"` does not select or imply this support. The explicit CLI/MCP reader selects the installed official `v0.3.0-rc.2` -leaf, source `bcac15b7537ce21879201b5e5dfe795496225121`, with maturity `draft` +identity. Core `specVersion: "0.3.0"` does not select or imply this support. The explicit CLI/MCP reader selects the installed official `v0.3.0` +leaf, source `27cb29bf13e4a2d58d69fe19c7ebfeb762b014fc`, with maturity `draft` and document-validation role for Base/Extended. Compatibility flag names -`--draft-contract`/`draftContract` require the exact installed RC2 selector; -historical Draft selectors refuse. Historical Draft test resources retain their -original labels. See [the RC2 reader](docs/dialect-retained-column-rc2.md). +`--draft-contract`/`draftContract` require the exact installed release selector; +historical Draft selectors and the `v0.3.0-rc.2` candidate selector refuse. Historical Draft test resources retain their +original labels. See [the selected reader](docs/dialect-retained-column.md). ## Compatibility dimensions diff --git a/docs/architecture.md b/docs/architecture.md index 0d9e92c..e0357e8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -216,8 +216,8 @@ client rather than in a runtime store -- is walked end to end in ## Core contract boundaries -The `0.3.0` resources are an exact immutable Spec commit pin, -`65c7cf5598a0f0fc532c80a9e2215090951104ce`, rather than a published Core tag. +The `0.3.0` resources are imported from the published Spec tag `v0.3.0`, commit +`27cb29bf13e4a2d58d69fe19c7ebfeb762b014fc`. The conformance runner registers the exact Core schema as an explicit in-memory resource for manifest references. Local suites must match the bundled manifest schema; they cannot retrieve remote schemas or traverse to sibling resources. diff --git a/docs/dialect-retained-column-draft.md b/docs/dialect-retained-column-draft.md index 7358455..9c5c821 100644 --- a/docs/dialect-retained-column-draft.md +++ b/docs/dialect-retained-column-draft.md @@ -1,7 +1,7 @@ # Historical retained-column Draft reader This page records the historical Draft experiment and its original pins. Current -selected support is documented in [the RC2 reader](dialect-retained-column-rc2.md). +selected support is documented in [the selected reader](dialect-retained-column.md). The historical selector below is no longer accepted as a selected identity. That historical Runtime had an explicit experimental entry for RFC 0029 at Spec revision diff --git a/docs/dialect-retained-column-rc2.md b/docs/dialect-retained-column.md similarity index 85% rename from docs/dialect-retained-column-rc2.md rename to docs/dialect-retained-column.md index 5261586..ec2bc50 100644 --- a/docs/dialect-retained-column-rc2.md +++ b/docs/dialect-retained-column.md @@ -1,17 +1,18 @@ -# Selected retained-column RC2 reader +# Selected retained-column reader Runtime installs the retained-column/1 leaf from the official immutable Spec -`v0.3.0-rc.2` distribution at source revision -`bcac15b7537ce21879201b5e5dfe795496225121`. Its maturity is `draft`; selected +`v0.3.0` release at source revision +`27cb29bf13e4a2d58d69fe19c7ebfeb762b014fc`. Its maturity is `draft`; selected support is limited to document validation and declared inventory coverage. Validation makes no preparation, source authenticity or execution claim. The existing dialect operations retain their default contract and limits. Select exactly -`--draft-contract retained-column/1@bcac15b7537ce21879201b5e5dfe795496225121` +`--draft-contract retained-column/1@27cb29bf13e4a2d58d69fe19c7ebfeb762b014fc` and `--draft-profile base` or `--draft-profile extended` on `dialect contract`, `canonicalize`, `digest`, `validate` or `check`. Both flags are required. The `draft` flag names are compatibility adapter -spellings; their exact accepted value selects only this installed RC2 identity. Unsupported selection refuses before opening input files. A document +spellings; their exact accepted value selects only this installed release identity. Unsupported selection, including the +`v0.3.0-rc.2` candidate selector, refuses before opening input files. A document must declare the selected profile; omission means base. Extended requires explicit `"profile":"extended"` in a validated document. No selection is inferred from leaves, carrier versions or caller-supplied specification hashes. @@ -43,8 +44,8 @@ response bounded without reducing accepted input or changing ordinary reports. The selected contract response reports only role `document-validation` in `capabilities.conversionContracts`, with fixed trusted identity: -- Version label: `0.3.0-rc.2`. -- Revision: `bcac15b7537ce21879201b5e5dfe795496225121`. +- Version label: `0.3.0`. +- Revision: `27cb29bf13e4a2d58d69fe19c7ebfeb762b014fc`. - Schema: `sha256:0cb49f238c78391cb163a65474cc9af962df2bb1024a55c5310c2b784b2f0557`. - Corpus: `sha256:17b16ff7a9b74fe4cc81eb6f6129a6aabdd9834621d9e756aa2d4c1a3f245863`. @@ -95,7 +96,12 @@ Preview arithmetic, producer/history, preparation/export/verification and capabi consumer cases remain their owners' work. Synthetic published fixture identities remain preparation-only and are never promoted into signed full-input proof. Historical Draft test resources remain byte-exact under their original pins. +The schema, corpus and capability table are byte-identical to the `v0.3.0-rc.2` +candidate's, so the published tables keep the distribution label they were +published with; the installed release identity and the `v0.3.0` inventory carry +the release. The installed reader refuses the candidate's release identity. -Existing Runtime releases, including 0.39.0-alpha.1, gain no support from this -source change. Stable integrated checks, CI, review and distribution gates remain +Runtime 0.39.0-alpha.2 installed the `v0.3.0-rc.2` candidate; earlier releases, +including 0.39.0-alpha.1, installed none. No existing release gains support from +this source change. Stable integrated checks, CI, review and distribution gates remain required before a new Runtime release. No final Core conformance claim follows. diff --git a/docs/releasing.md b/docs/releasing.md index 6108d99..0d4f07b 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -56,33 +56,45 @@ the retained prototype or historical candidate trees or the Core locks. Policy source schema IDs stay distinct from derived Runtime editing schema IDs. See [ADR-0061](adr/0061-adopt-official-policy-draft-resources.md) for compatibility. -## Refresh the selected retained-column RC2 leaf import +## Refresh the selected retained-column leaf import -The current opt-in CLI/MCP reader selects the installed official `v0.3.0-rc.2` +The current opt-in CLI/MCP reader selects the installed official `v0.3.0` leaf with draft maturity and document-validation role. Its exact source/schema/corpus and independent provenance pin are compiled in `retained_column_artifacts.go`. Core locks, Policy resources and the general Core registry remain independent. -Historical Draft fixtures retain their original labels; they are not RC2 evidence. +Historical Draft fixtures retain their original labels; they are not release evidence. +Runtime 0.39.0-alpha.2 installed the `v0.3.0-rc.2` candidate of this leaf; the +installed reader now refuses that candidate's identity. + +A Spec pre-release takes one of two forms: `-rc.N` is a candidate of the complete +milestone, and `-alpha.N` is a scoped cut for a named consumer that never graduates; +the development kit's release policy defines both. After the maintainer confirms adoption and immutable publication of the official -Spec `v0.3.0-rc.2`, verify its exact tag/source commit, immutable release state, +Spec release, verify its exact tag/source commit, immutable release state, asset checksums and attestations using the Spec release process. Record those observations separately: the offline importer cannot establish publication, immutability or source authenticity from local bytes or a Git remote URL. Review the release inventory digest independently. Use a clean official checkout -at that exact tag and the verified tar.gz asset, then run: +at that exact tag and the verified tar.gz asset. A reviewed artifact update never +overwrites a nonempty set: remove the previous `official/` import in the same +change, then run: ```bash env GO111MODULE=on go run ./tools/sync-retained-column \ --source ../judgment-pack-spec \ - --source-ref v0.3.0-rc.2 \ - --source-commit "$RC2_SOURCE_COMMIT" \ - --archive "$RC2_VERIFIED_ARCHIVE" \ - --archive-sha256 "$RC2_VERIFIED_ARCHIVE_DIGEST" \ - --inventory-sha256 "$RC2_REVIEWED_INVENTORY_DIGEST" \ + --source-ref v0.3.0 \ + --source-commit "$SOURCE_COMMIT" \ + --archive "$VERIFIED_ARCHIVE" \ + --archive-sha256 "$VERIFIED_ARCHIVE_DIGEST" \ + --inventory-sha256 "$REVIEWED_INVENTORY_DIGEST" \ --destination ./internal/dialect/retained-column/official ``` +The importer admits only the release named by `internal/retainedcolumn` +(`Release`, `Version`, `InventoryPath`); adopting another release changes those +constants, the reader identity and its tests in the same reviewed change. + Digest arguments include `sha256:`. The destination must be new and exclusively owned during import. The helper follows the existing staged import pattern; the portable directory rename can replace an empty directory created concurrently, diff --git a/internal/artifacts/artifacts_test.go b/internal/artifacts/artifacts_test.go index bdf1b8c..cf552d0 100644 --- a/internal/artifacts/artifacts_test.go +++ b/internal/artifacts/artifacts_test.go @@ -249,11 +249,11 @@ func TestCore030ExactProvenanceAndCompleteIndexedFixtures(t *testing.T) { t.Fatal(err) } lock := set.Lock() - const pin = "65c7cf5598a0f0fc532c80a9e2215090951104ce" + const tag, pin = "v0.3.0", "27cb29bf13e4a2d58d69fe19c7ebfeb762b014fc" if len(lock.Files) != 165 { t.Fatalf("Core 0.3 lock files: %d", len(lock.Files)) } - if lock.Source.Repository != "https://github.com/Judgment-Pack/judgment-pack-spec" || lock.Source.Ref != pin || lock.Source.BaseCommit != pin || lock.Source.Kind != "immutable-git-ref" || lock.Source.WorktreeDirty { + if lock.Source.Repository != "https://github.com/Judgment-Pack/judgment-pack-spec" || lock.Source.Ref != tag || lock.Source.BaseCommit != pin || lock.Source.Kind != "immutable-git-ref" || lock.Source.WorktreeDirty { t.Fatalf("unexpected source: %+v", lock.Source) } if lock.BundleDigest.Value != "018b8221c424fa530d7482b160f4d7faa59cd9d2e80d3f6e2f8748dfb0d7e743" { diff --git a/internal/artifacts/jps/0.3.0/lock.json b/internal/artifacts/jps/0.3.0/lock.json index d905cb8..1ca9a82 100644 --- a/internal/artifacts/jps/0.3.0/lock.json +++ b/internal/artifacts/jps/0.3.0/lock.json @@ -4,8 +4,8 @@ "source": { "repository": "https://github.com/Judgment-Pack/judgment-pack-spec", "kind": "immutable-git-ref", - "baseCommit": "65c7cf5598a0f0fc532c80a9e2215090951104ce", - "ref": "65c7cf5598a0f0fc532c80a9e2215090951104ce", + "baseCommit": "27cb29bf13e4a2d58d69fe19c7ebfeb762b014fc", + "ref": "v0.3.0", "worktreeDirty": false }, "bundleDigest": { diff --git a/internal/cli/dialect.go b/internal/cli/dialect.go index 0d3d19e..5c35683 100644 --- a/internal/cli/dialect.go +++ b/internal/cli/dialect.go @@ -151,10 +151,10 @@ func (a *App) dialectOperation(operation string) *cobra.Command { c.Flags().StringVar(&packPath, "pack", "", "pack file or - for standard input; required") } if operation == "contract" || operation == "canonicalize" || operation == "digest" || operation == "validate" || operation == "check" { - c.Flags().StringVar(&draftContract, "draft-contract", "", "explicit installed RC2 selector (compatibility flag spelling): "+dialect.RetainedColumnDraftSelector) + c.Flags().StringVar(&draftContract, "draft-contract", "", "explicit installed leaf selector (compatibility flag spelling): "+dialect.RetainedColumnDraftSelector) c.Flags().StringVar(&draftProfile, "draft-profile", "", "explicit selected profile: base or extended; requires --draft-contract") if operation == "validate" { - c.Flags().BoolVar(&retainedFrame, "retained-frame", false, "validate a standalone retained frame; requires both selected RC2 flags") + c.Flags().BoolVar(&retainedFrame, "retained-frame", false, "validate a standalone retained frame; requires both selected leaf flags") } } return c @@ -220,9 +220,9 @@ func (a *App) runRetainedColumnDraft(command, operation string, args []string, f } } case operation == "contract": - _, err = fmt.Fprintf(a.out, "Selected RC2 %s · %s\nProfile: %s; role: document-validation\nOperations: contract, canonicalize, digest, validate (document or retained frame), check\nDocument limit: %d UTF-8 bytes; frame limit: %d UTF-8 bytes\nCanonicalize and digest check encoding only. Pack checks cover declared inventory.\n", selected.Contract, selected.Spec.Revision, selected.Profile, selected.Limits.DocumentBytes, selected.Limits.FrameBytes) + _, err = fmt.Fprintf(a.out, "Selected %s v%s · %s\nProfile: %s; role: document-validation\nOperations: contract, canonicalize, digest, validate (document or retained frame), check\nDocument limit: %d UTF-8 bytes; frame limit: %d UTF-8 bytes\nCanonicalize and digest check encoding only. Pack checks cover declared inventory.\n", selected.Contract, selected.Spec.Version, selected.Spec.Revision, selected.Profile, selected.Limits.DocumentBytes, selected.Limits.FrameBytes) case operation == "validate": - _, err = fmt.Fprintln(a.out, "Selected RC2 checks passed: "+report.ValidationScope+".") + _, err = fmt.Fprintln(a.out, "Selected leaf checks passed: "+report.ValidationScope+".") case operation == "digest": _, err = fmt.Fprintln(a.out, report.Digest) default: diff --git a/internal/cli/dialect_retained_column_test.go b/internal/cli/dialect_retained_column_test.go index d4ad9c0..114fb5a 100644 --- a/internal/cli/dialect_retained_column_test.go +++ b/internal/cli/dialect_retained_column_test.go @@ -150,7 +150,7 @@ func TestRetainedColumnEntryCLIFrameAndByteBoundaries(t *testing.T) { // Use the installed published recipes through both actual application handlers. // Independent maximum generated values are frame API checks, not snapshot claims. -func TestRetainedColumnRC2CLIAndMCPRawValues(t *testing.T) { +func TestRetainedColumnReleaseCLIAndMCPRawValues(t *testing.T) { raw, err := os.ReadFile("../dialect/retained-column/official/conformance/retained-column/1/cases.json") if err != nil { t.Fatal(err) @@ -224,7 +224,7 @@ func TestRetainedColumnRC2CLIAndMCPRawValues(t *testing.T) { t.Fatal("raw frame wire preservation failed", code) } spec := cliReport["spec"].(map[string]any) - if cliReport["maturity"] != "draft" || cliReport["role"] != "document-validation" || spec["version"] != "0.3.0-rc.2" || spec["revision"] != "bcac15b7537ce21879201b5e5dfe795496225121" || spec["schemaSha256"] != "sha256:0cb49f238c78391cb163a65474cc9af962df2bb1024a55c5310c2b784b2f0557" || spec["corpusSha256"] != "sha256:17b16ff7a9b74fe4cc81eb6f6129a6aabdd9834621d9e756aa2d4c1a3f245863" { + if cliReport["maturity"] != "draft" || cliReport["role"] != "document-validation" || spec["version"] != "0.3.0" || spec["revision"] != "27cb29bf13e4a2d58d69fe19c7ebfeb762b014fc" || spec["schemaSha256"] != "sha256:0cb49f238c78391cb163a65474cc9af962df2bb1024a55c5310c2b784b2f0557" || spec["corpusSha256"] != "sha256:17b16ff7a9b74fe4cc81eb6f6129a6aabdd9834621d9e756aa2d4c1a3f245863" { t.Fatal("public installed identity changed") } } diff --git a/internal/dialect/retained-column/README.md b/internal/dialect/retained-column/README.md index f813cef..262ffc7 100644 --- a/internal/dialect/retained-column/README.md +++ b/internal/dialect/retained-column/README.md @@ -1,7 +1,7 @@ # Retained-column installation state -`official/` retains the exact original paths from official Spec `v0.3.0-rc.2`, -source `bcac15b7537ce21879201b5e5dfe795496225121`. `provenance.json` records the +`official/` retains the exact original paths from official Spec `v0.3.0`, +source `27cb29bf13e4a2d58d69fe19c7ebfeb762b014fc`. `provenance.json` records the verified archive/inventory identities and each imported file's bytes and digest. The independently compiled provenance pin and selected tuple are in `retained_column_artifacts.go`; admission and release checks verify them. diff --git a/internal/dialect/retained-column/official/provenance.json b/internal/dialect/retained-column/official/provenance.json index 4e92f06..28bc23c 100644 --- a/internal/dialect/retained-column/official/provenance.json +++ b/internal/dialect/retained-column/official/provenance.json @@ -1,14 +1,14 @@ { "sourceRepository": "https://github.com/Judgment-Pack/judgment-pack-spec", - "sourceRelease": "v0.3.0-rc.2", - "sourceRevision": "bcac15b7537ce21879201b5e5dfe795496225121", + "sourceRelease": "v0.3.0", + "sourceRevision": "27cb29bf13e4a2d58d69fe19c7ebfeb762b014fc", "sourceKind": "immutable-git-ref", "worktreeDirty": false, - "sourceArchiveSha256": "sha256:0762a27d68492bcc2313872b7ae566e8b3cd65f3a45e02d596cb686259f3cb61", - "inventorySha256": "sha256:b4d62b3259b24dee68d926750ab45247d974004eec2bd15814f7d24c89140160", + "sourceArchiveSha256": "sha256:6469a866b9c0e1671de3462c1b1823a1738561f6b84e52b470382fc620530ce9", + "inventorySha256": "sha256:c8e489b2cabd0ee0b794a8b6a817f06f901e338348e32ec23c8c83d7c40dfe58", "identity": { - "version": "0.3.0-rc.2", - "revision": "bcac15b7537ce21879201b5e5dfe795496225121", + "version": "0.3.0", + "revision": "27cb29bf13e4a2d58d69fe19c7ebfeb762b014fc", "schemaSha256": "sha256:0cb49f238c78391cb163a65474cc9af962df2bb1024a55c5310c2b784b2f0557", "corpusSha256": "sha256:17b16ff7a9b74fe4cc81eb6f6129a6aabdd9834621d9e756aa2d4c1a3f245863" }, @@ -45,13 +45,13 @@ }, { "path": "release-identity.json", - "byteLength": 186, - "sha256": "sha256:5b107b898fe9a1fb41f7c4ae18ec1227f06b3e9624ed98d86ea74cfbb8bcbb4e" + "byteLength": 141, + "sha256": "sha256:9432a9f5e4b32eff2b231baeb817bfab99a8e77726c9c47c25a76906d109c121" }, { - "path": "releases/v0.3.0-rc.2.retained-column.json", - "byteLength": 3651, - "sha256": "sha256:b4d62b3259b24dee68d926750ab45247d974004eec2bd15814f7d24c89140160" + "path": "releases/v0.3.0.retained-column.json", + "byteLength": 3646, + "sha256": "sha256:c8e489b2cabd0ee0b794a8b6a817f06f901e338348e32ec23c8c83d7c40dfe58" }, { "path": "schema/retained-column/1/contract.schema.json", @@ -60,18 +60,18 @@ }, { "path": "spec/dialect.md", - "byteLength": 148952, - "sha256": "sha256:a10d6fc0f50b1cf450b2bd9156175529bed2a7d926d24567b47677076066cb5a" + "byteLength": 148961, + "sha256": "sha256:729e491a0d855a51f69e9b9e7aadc24131209c388ee29e8820c57a7a7c845d5a" }, { "path": "spec/judgment-pack-core.md", - "byteLength": 67285, - "sha256": "sha256:0ceb66c15ce6b8268e02f319720eb40766c616423cfa08caaeea56ae8ba3d2ef" + "byteLength": 67317, + "sha256": "sha256:151d64678db709f91ab0729c2f75516b89a027e24a3e22e1788dcdf0aa6faabc" }, { "path": "spec/retained-column.md", - "byteLength": 58020, - "sha256": "sha256:f17747d5daeba6953cef335900569c5b1cf1052b7f982c0146b6322687375b7d" + "byteLength": 58329, + "sha256": "sha256:c1c9dd34063b9056175b7740d161341a5529e9642eaead3da263cd29a0b4b5c6" }, { "path": "tools/_retained_column.py", @@ -125,8 +125,8 @@ }, { "path": "tools/dialect-verify-ts/src/retained.ts", - "byteLength": 18165, - "sha256": "sha256:5ae44bc9782168cdae87467a0eb9f4229382f46058ac64ef6369b1236fbe905d" + "byteLength": 18246, + "sha256": "sha256:156ca057e967e3ae36ba4c8a740ee0f41e87588826c9f55af70bd00688163302" }, { "path": "tools/dialect-verify-ts/src/schema.ts", @@ -155,8 +155,8 @@ }, { "path": "tools/retained_column.py", - "byteLength": 19582, - "sha256": "sha256:0598c3b31a0a4115eb4efdf2e8fccc6b75c558dfe7d09180b3acf2241e937447" + "byteLength": 19654, + "sha256": "sha256:ba3e97b1a9f4e46fff808f9ac79bc208d5b2a0a4c4f71ea0d760b259ba12bc5c" } ] } diff --git a/internal/dialect/retained-column/official/release-identity.json b/internal/dialect/retained-column/official/release-identity.json index 2a3a32a..4f32f5e 100644 --- a/internal/dialect/retained-column/official/release-identity.json +++ b/internal/dialect/retained-column/official/release-identity.json @@ -1 +1 @@ -{"distribution":"release-candidate","maturity":"draft","releaseVersion":"0.3.0-rc.2","sourceCommit":"bcac15b7537ce21879201b5e5dfe795496225121","specVersion":"0.3.0","tag":"v0.3.0-rc.2"} +{"maturity":"draft","releaseVersion":"0.3.0","sourceCommit":"27cb29bf13e4a2d58d69fe19c7ebfeb762b014fc","specVersion":"0.3.0","tag":"v0.3.0"} diff --git a/internal/dialect/retained-column/official/releases/v0.3.0-rc.2.retained-column.json b/internal/dialect/retained-column/official/releases/v0.3.0.retained-column.json similarity index 84% rename from internal/dialect/retained-column/official/releases/v0.3.0-rc.2.retained-column.json rename to internal/dialect/retained-column/official/releases/v0.3.0.retained-column.json index d261821..9128136 100644 --- a/internal/dialect/retained-column/official/releases/v0.3.0-rc.2.retained-column.json +++ b/internal/dialect/retained-column/official/releases/v0.3.0.retained-column.json @@ -1,6 +1,6 @@ { "contract": "retained-column/1", - "distribution": "v0.3.0-rc.2", + "distribution": "v0.3.0", "maturity": "draft", "schema": "schema/retained-column/1/contract.schema.json", "schemaSha256": "sha256:0cb49f238c78391cb163a65474cc9af962df2bb1024a55c5310c2b784b2f0557", @@ -14,9 +14,9 @@ "conformance/retained-column/1/manifest.json": "sha256:b78cdac121b4b936614c9e652af3c78bde8b84184fd0a04f84effdd2c3eaf2c8", "conformance/retained-column/1/preparations.json": "sha256:ff9450279cc65db4422f3644975b5b1128396e2e9a1f17effd4dcc732bb5317a", "schema/retained-column/1/contract.schema.json": "sha256:0cb49f238c78391cb163a65474cc9af962df2bb1024a55c5310c2b784b2f0557", - "spec/dialect.md": "sha256:a10d6fc0f50b1cf450b2bd9156175529bed2a7d926d24567b47677076066cb5a", - "spec/judgment-pack-core.md": "sha256:0ceb66c15ce6b8268e02f319720eb40766c616423cfa08caaeea56ae8ba3d2ef", - "spec/retained-column.md": "sha256:f17747d5daeba6953cef335900569c5b1cf1052b7f982c0146b6322687375b7d", + "spec/dialect.md": "sha256:729e491a0d855a51f69e9b9e7aadc24131209c388ee29e8820c57a7a7c845d5a", + "spec/judgment-pack-core.md": "sha256:151d64678db709f91ab0729c2f75516b89a027e24a3e22e1788dcdf0aa6faabc", + "spec/retained-column.md": "sha256:c1c9dd34063b9056175b7740d161341a5529e9642eaead3da263cd29a0b4b5c6", "tools/_retained_column.py": "sha256:65c91f1363769ecfdfbebb5b42ade4258aaaf7b61620b23ee68b4000fee96233", "tools/check_retained_column.py": "sha256:946709a979f9dd9fc1aa10ab4500d77128c053ad3e46f653537f83c29becad18", "tools/dialect-verify-ts/src/bounds.ts": "sha256:c90d6c5db69206d4361409e502bb54a5e81505e4ac5842b987b44e60fbdb53c1", @@ -27,13 +27,13 @@ "tools/dialect-verify-ts/src/main.ts": "sha256:ef1e5f28c2608d51f719adc74bad03266ce5f2fb5526673e878229dc260b99be", "tools/dialect-verify-ts/src/prepare.ts": "sha256:e185b78a8f4d51cac63557f6973ef8b4af99660c3cef15b1509156acc8bbf027", "tools/dialect-verify-ts/src/retained-engine.mjs": "sha256:4ab0e7bcae864552a987a53fcec97f834e280c4dbf8b3ae37b6ae950739a9825", - "tools/dialect-verify-ts/src/retained.ts": "sha256:5ae44bc9782168cdae87467a0eb9f4229382f46058ac64ef6369b1236fbe905d", + "tools/dialect-verify-ts/src/retained.ts": "sha256:156ca057e967e3ae36ba4c8a740ee0f41e87588826c9f55af70bd00688163302", "tools/dialect-verify-ts/src/schema.ts": "sha256:938ad0ffefc378a7b1cc3bc7777063f4ef2929fbf8b90132ca5eb76537981dd3", "tools/dialect-verify-ts/src/time.ts": "sha256:54afe326be3d163d325ea64ccaf907e83530a38250824d1877cc1b4726c92ca7", "tools/dialect-verify-ts/src/vector.ts": "sha256:289b68479edf607df1480e46b0e6a6ee04091feb5dd56ddee696fbb2a337a1cb", "tools/dialect-verify-ts/src/verify.ts": "sha256:5ae002b07b0f9d3755c9d00bc828420e27b9927948402d8e4b266540c4f5cc1b", "tools/dialect_verify.py": "sha256:fe3b773ce30ad8c395fe85500714a12d6cc4262ccc6bca05c44f7b3b40a60b60", - "tools/retained_column.py": "sha256:0598c3b31a0a4115eb4efdf2e8fccc6b75c558dfe7d09180b3acf2241e937447" + "tools/retained_column.py": "sha256:ba3e97b1a9f4e46fff808f9ac79bc208d5b2a0a4c4f71ea0d760b259ba12bc5c" }, "claim": "selected-leaf-support" } diff --git a/internal/dialect/retained-column/official/spec/dialect.md b/internal/dialect/retained-column/official/spec/dialect.md index e1a6125..297ce32 100644 --- a/internal/dialect/retained-column/official/spec/dialect.md +++ b/internal/dialect/retained-column/official/spec/dialect.md @@ -139,9 +139,9 @@ be interpreted as described by BCP 14 when, and only when, they appear in all ca ## Selected retained-column support [Selected retained columns](retained-column.md) is accepted at draft maturity for exact -`retained-column/1` selection in RC2. It adds no implicit support to ordinary inputs. +`retained-column/1` selection, adopted in `v0.3.0` (candidate distribution `v0.3.0-rc.2`). It adds no implicit support to ordinary inputs. Its new leaf corpus/claim does not publish the forty-seven staged recordings or establish the -generic §14 preparation-verifier conformance claim. Immutable RC2 publication remains separate. +generic §14 preparation-verifier conformance claim. ## 1. Terms and the canonical forms diff --git a/internal/dialect/retained-column/official/spec/judgment-pack-core.md b/internal/dialect/retained-column/official/spec/judgment-pack-core.md index cb25571..41eea19 100644 --- a/internal/dialect/retained-column/official/spec/judgment-pack-core.md +++ b/internal/dialect/retained-column/official/spec/judgment-pack-core.md @@ -51,8 +51,8 @@ The artifacts in this repository have distinct roles: - [`spec/dialect.md`](dialect.md), a companion document, is the normative prose for the preparation-verifier reader conformance class it defines, and for nothing else; - [selected retained columns](retained-column.md) is accepted at draft maturity as normative prose - for the exact RC2 selected leaf support scope only; it establishes no implicit ordinary support - or generic preparation-verifier claim, and publication requires the immutable RC2 release; + for the selected leaf support scope only, adopted in `v0.3.0` (candidate distribution `v0.3.0-rc.2`); + it establishes no implicit ordinary support or generic preparation-verifier claim; - the evaluation corpus — the manifest and case fixtures under [`conformance/evaluation/`](../conformance/evaluation/README.md), not its README — is normative for evaluator conformance (§3.4) and for nothing else. This is the normative status the bullet below @@ -368,7 +368,7 @@ For the proposed exact selected `retained-column/1` contract only, [the retained-column companion](retained-column.md) specifies scoped Dialect artifact ceilings after its own selection and independent role admission. Absence of that selection uses this table. It does not raise Core pack, global transport, canonical parser, target-count or independent -work ceilings. This is an RC2 adoption candidate, not a support claim for an existing release. +work ceilings. It is adopted at draft maturity in `v0.3.0` (candidate distribution `v0.3.0-rc.2`), not a Core conformance claim. The following is the ceiling table for these formats. Numbers elsewhere in the companions state `base`; except for the exactly selected retained-column companion above, their diff --git a/internal/dialect/retained-column/official/spec/retained-column.md b/internal/dialect/retained-column/official/spec/retained-column.md index 02c141d..a6e88ed 100644 --- a/internal/dialect/retained-column/official/spec/retained-column.md +++ b/internal/dialect/retained-column/official/spec/retained-column.md @@ -3,7 +3,7 @@ ## Status and exact scope This companion is accepted at **draft maturity** through [RFC0029](../rfcs/0029-retained-column-conversions.md) -for the exact `v0.3.0-rc.2` distribution. Acceptance is separate from immutable publication: +in the `v0.3.0` release, first distributed as the `v0.3.0-rc.2` candidate. Acceptance is separate from immutable publication: the exact tag, maintainer-published GitHub release and verified artifacts are required before published support can be claimed. RFC0029 is its informative design record. @@ -20,12 +20,12 @@ below are required behavior of a selected supporting role, not optional implemen ## Installed identity and certification The complete trusted support tuple is contract `retained-column/1`, Spec distribution version -`0.3.0-rc.2`, the exact immutable Spec source commit, the new schema SHA-256, complete selected +`0.3.0`, the exact immutable Spec source commit, the new schema SHA-256, complete selected corpus SHA-256, profile and role (plus producerGrammar where applicable). Core packs continue to declare exact specVersion `0.3.0`; that does not select this leaf. RC2 evidence remains provisional for final Core conformance and is never described as a final Core release claim. -`releases/v0.3.0-rc.2.retained-column.json` inventories the new schema and all corpus inputs. +`releases/v0.3.0.retained-column.json` inventories the new schema and all corpus inputs. `corpusSha256` hashes the exact selected `conformance/retained-column/1/cases.json` bytes; that file names every normative case group and its external companion inputs by digest. The release inventory also binds the handwritten full-preparation fixtures and their driver. @@ -263,7 +263,7 @@ or a boolean support flag. Each entry is a closed object with required `contract producer section below for its three applicable roles. `contract` is exactly `"retained-column/1"`; `profile` is exactly `"base"` or `"extended"`. `spec` is closed with exactly `version` (nonempty version label), `revision` (40 lower-case hex characters identifying the exact Spec commit), `schemaSha256` -and `corpusSha256` (each `sha256:` plus 64 lower-case hex characters). The normative support version is exactly `0.3.0-rc.2`; the final immutable source revision +and `corpusSha256` (each `sha256:` plus 64 lower-case hex characters). The normative support version is exactly `0.3.0`, the `v0.3.0` distribution; `v0.3.0-rc.2` was its candidate distribution and remains published: the schema, the corpus and their digests are the candidate's, while this companion page and the two offline readers name the release; a leaf support claim is made against a `v0.3.0` installation. The final immutable source revision and new schema/corpus digests come from the external release inventory described above. Test example pins are synthetic and grant no real support. Consumers compare all identity members exactly to trusted expected pins, never values learned solely from the report. Member order does not affect comparison. diff --git a/internal/dialect/retained-column/official/tools/dialect-verify-ts/src/retained.ts b/internal/dialect/retained-column/official/tools/dialect-verify-ts/src/retained.ts index cda9ed4..5877358 100644 --- a/internal/dialect/retained-column/official/tools/dialect-verify-ts/src/retained.ts +++ b/internal/dialect/retained-column/official/tools/dialect-verify-ts/src/retained.ts @@ -19,16 +19,18 @@ const digest = (text: Uint8Array | string) => "sha256:" + createHash("sha256").u const plain = (v: JVal): any => engine.parse(decode(preparationForm(v))); const jval = (v: any): JVal => parse(JSON.stringify(v)); const bad = (code = "input-malformed"): never => { throw new Refusal(code, "selected retained-column contract"); }; +const TAG = "v0.3.0", VERSION = "0.3.0", INVENTORY = "releases/v0.3.0.retained-column.json"; export const installation = { identity(): Record { const identity = JSON.parse(readFileSync(pathResolve(ROOT, "release-identity.json"), "utf8")); - const inventory = JSON.parse(readFileSync(pathResolve(ROOT, "releases/v0.3.0-rc.2.retained-column.json"), "utf8")); - if (identity.tag !== "v0.3.0-rc.2" || identity.releaseVersion !== "0.3.0-rc.2" || identity.specVersion !== "0.3.0" - || identity.maturity !== "draft" || !/^[0-9a-f]{40}$/.test(identity.sourceCommit)) bad("capability-unsupported"); + const inventory = JSON.parse(readFileSync(pathResolve(ROOT, INVENTORY), "utf8")); + if (identity.tag !== TAG || identity.releaseVersion !== VERSION || identity.specVersion !== "0.3.0" + || identity.maturity !== "draft" || !/^[0-9a-f]{40}$/.test(identity.sourceCommit) + || inventory.distribution !== TAG) bad("capability-unsupported"); for (const [name, sha] of Object.entries(inventory.files)) { if (digest(readFileSync(pathResolve(ROOT, name))) !== sha) bad("capability-unsupported"); } - return { version: "0.3.0-rc.2", revision: identity.sourceCommit, schemaSha256: inventory.schemaSha256, corpusSha256: inventory.corpusSha256 }; + return { version: VERSION, revision: identity.sourceCommit, schemaSha256: inventory.schemaSha256, corpusSha256: inventory.corpusSha256 }; }, }; type Selection = { selector: any; raw: WeakMap; origin: "asserted" | "generated"; profilesBytes: number }; diff --git a/internal/dialect/retained-column/official/tools/retained_column.py b/internal/dialect/retained-column/official/tools/retained_column.py index f1a6f50..1b7e8b2 100644 --- a/internal/dialect/retained-column/official/tools/retained_column.py +++ b/internal/dialect/retained-column/official/tools/retained_column.py @@ -20,22 +20,25 @@ import _retained_column as engine ROOT = Path(__file__).resolve().parents[1] -TAG = "v0.3.0-rc.2" +TAG = "v0.3.0" +VERSION = "0.3.0" +INVENTORY = "releases/v0.3.0.retained-column.json" ROLES = tuple(engine.ROLE_LIMITS) CAPS = engine.PROFILE_CAPS def installed_spec() -> dict[str, str]: identity = json.loads((ROOT / "release-identity.json").read_text("utf-8")) - inventory = json.loads((ROOT / "releases/v0.3.0-rc.2.retained-column.json").read_text("utf-8")) - if (identity.get("tag") != TAG or identity.get("releaseVersion") != "0.3.0-rc.2" + inventory = json.loads((ROOT / INVENTORY).read_text("utf-8")) + if (identity.get("tag") != TAG or identity.get("releaseVersion") != VERSION or identity.get("specVersion") != "0.3.0" or identity.get("maturity") != "draft" - or not re.fullmatch("[0-9a-f]{40}", identity.get("sourceCommit", ""))): + or not re.fullmatch("[0-9a-f]{40}", identity.get("sourceCommit", "")) + or inventory.get("distribution") != TAG): engine.fail("capability-unsupported") for name, digest in inventory["files"].items(): if "sha256:" + hashlib.sha256((ROOT / name).read_bytes()).hexdigest() != digest: engine.fail("capability-unsupported") - return {"version": "0.3.0-rc.2", "revision": identity["sourceCommit"], + return {"version": VERSION, "revision": identity["sourceCommit"], "schemaSha256": inventory["schemaSha256"], "corpusSha256": inventory["corpusSha256"]} diff --git a/internal/dialect/retained_column_artifacts.go b/internal/dialect/retained_column_artifacts.go index cbb83e1..2317457 100644 --- a/internal/dialect/retained_column_artifacts.go +++ b/internal/dialect/retained_column_artifacts.go @@ -11,15 +11,15 @@ import ( // A reviewed import goes below retained-column/official and requires an // independent provenance pin and exact supported tuple in this file. An empty -// pin means no RC2 installation, not a placeholder RC2 capability. +// pin means no installation, not a placeholder capability. // //go:embed all:retained-column var retainedColumnArtifacts embed.FS -const retainedColumnProvenancePin = "sha256:a13a5c76ef3e010b31266d0fd779450ba985c41df779280a3da42c3a79acdfd4" +const retainedColumnProvenancePin = "sha256:271bfd2b1858885a4748d8630e0cb98a3dcf3d93b015798ab4d7ad804dcb257b" var retainedColumnInstalledIdentity = retainedcolumn.Identity{ - Version: "0.3.0-rc.2", Revision: "bcac15b7537ce21879201b5e5dfe795496225121", + Version: "0.3.0", Revision: "27cb29bf13e4a2d58d69fe19c7ebfeb762b014fc", SchemaSHA256: "sha256:0cb49f238c78391cb163a65474cc9af962df2bb1024a55c5310c2b784b2f0557", CorpusSHA256: "sha256:17b16ff7a9b74fe4cc81eb6f6129a6aabdd9834621d9e756aa2d4c1a3f245863", } diff --git a/internal/dialect/retained_column_artifacts_test.go b/internal/dialect/retained_column_artifacts_test.go index 3840dfa..69b6424 100644 --- a/internal/dialect/retained_column_artifacts_test.go +++ b/internal/dialect/retained_column_artifacts_test.go @@ -23,9 +23,9 @@ func TestRetainedColumnInstallationGate(t *testing.T) { t.Fatal("unpinned resources accepted") } delete(files, "retained-column/official/provenance.json") - selected.Version = "0.3.0-rc.2" + selected.Version = "0.3.0" if err := verifyRetainedColumnArtifacts(files, "", retainedcolumn.Identity{}, selected); err == nil { - t.Fatal("uninstalled selected RC2 identity accepted") + t.Fatal("uninstalled selected release identity accepted") } installed := retainedcolumn.Identity{Version: selected.Version, Revision: selected.Revision, SchemaSHA256: selected.SchemaSHA256, CorpusSHA256: selected.CorpusSHA256} selected.Revision = strings.Repeat("2", 40) @@ -62,7 +62,7 @@ func TestRetainedColumnAdmissionRequiresInstalledIdentity(t *testing.T) { } } -func TestRetainedColumnRC2InstalledInventoryRefusals(t *testing.T) { +func TestRetainedColumnReleaseInstalledInventoryRefusals(t *testing.T) { files := fstest.MapFS{} if err := fs.WalkDir(retainedColumnArtifacts, "retained-column", func(path string, entry fs.DirEntry, err error) error { if err != nil { @@ -80,7 +80,7 @@ func TestRetainedColumnRC2InstalledInventoryRefusals(t *testing.T) { }); err != nil { t.Fatal(err) } - for _, name := range []string{"valid", "missing", "changed", "extra", "outside inventory", "pin", "unsupported"} { + for _, name := range []string{"valid", "missing", "changed", "extra", "outside inventory", "pin", "unsupported", "candidate"} { t.Run(name, func(t *testing.T) { copy := fstest.MapFS{} for path, file := range files { @@ -103,6 +103,11 @@ func TestRetainedColumnRC2InstalledInventoryRefusals(t *testing.T) { case "unsupported": installed.Version = "0.3.0-rc.1" selected.Version = installed.Version + case "candidate": + // The v0.3.0-rc.2 tuple, selected and installed alike, is refused + // against the final release's installed bytes. + installed.Version, installed.Revision = "0.3.0-rc.2", "bcac15b7537ce21879201b5e5dfe795496225121" + selected.Version, selected.Revision = installed.Version, installed.Revision } err := verifyRetainedColumnArtifacts(copy, pin, installed, selected) if (err == nil) != (name == "valid") { diff --git a/internal/dialect/retained_column_draft.go b/internal/dialect/retained_column_draft.go index 7c34eea..b90a176 100644 --- a/internal/dialect/retained_column_draft.go +++ b/internal/dialect/retained_column_draft.go @@ -1,7 +1,7 @@ package dialect // This opt-in internal reader implements document/retained-frame structure from -// the installed immutable RC2 leaf below. It executes no conversions. +// the installed immutable official release leaf below. It executes no conversions. import ( "crypto/sha256" "encoding/hex" @@ -16,7 +16,7 @@ import ( const ( RetainedColumnDraftContract = "retained-column/1" RetainedColumnDraftSelector = RetainedColumnDraftContract + "@" + RetainedColumnDraftRevision - RetainedColumnDraftRevision = "bcac15b7537ce21879201b5e5dfe795496225121" + RetainedColumnDraftRevision = "27cb29bf13e4a2d58d69fe19c7ebfeb762b014fc" RetainedColumnDraftSchema = "sha256:0cb49f238c78391cb163a65474cc9af962df2bb1024a55c5310c2b784b2f0557" RetainedColumnDraftCorpus = "sha256:17b16ff7a9b74fe4cc81eb6f6129a6aabdd9834621d9e756aa2d4c1a3f245863" RetainedColumnDraftDocumentBytes = 1048576 @@ -44,7 +44,7 @@ type RetainedColumnDraftSelection struct { func RetainedColumnDraftBase() RetainedColumnDraftSelection { return RetainedColumnDraftSelection{RetainedColumnDraftContract, RetainedColumnDraftIdentity{ - "0.3.0-rc.2", RetainedColumnDraftRevision, RetainedColumnDraftSchema, RetainedColumnDraftCorpus}, "base"} + "0.3.0", RetainedColumnDraftRevision, RetainedColumnDraftSchema, RetainedColumnDraftCorpus}, "base"} } type RetainedColumnDraftLimits struct { diff --git a/internal/dialect/retained_column_rc2_test.go b/internal/dialect/retained_column_release_test.go similarity index 87% rename from internal/dialect/retained_column_rc2_test.go rename to internal/dialect/retained_column_release_test.go index f6c398a..319607d 100644 --- a/internal/dialect/retained_column_rc2_test.go +++ b/internal/dialect/retained_column_release_test.go @@ -10,7 +10,7 @@ import ( // Exercise the published bytes in this owner's document-validation role. Typed // conversion expectations belong to preview/preparation owners, not this reader. -func TestRetainedColumnRC2PublishedCorpus(t *testing.T) { +func TestRetainedColumnReleasePublishedCorpus(t *testing.T) { raw, err := fs.ReadFile(retainedColumnArtifacts, "retained-column/official/conformance/retained-column/1/cases.json") if err != nil { t.Fatal(err) @@ -130,7 +130,7 @@ func TestRetainedColumnRC2PublishedCorpus(t *testing.T) { } } -func TestRetainedColumnRC2PublishedRawValueBounds(t *testing.T) { +func TestRetainedColumnReleasePublishedRawValueBounds(t *testing.T) { raw, err := fs.ReadFile(retainedColumnArtifacts, "retained-column/official/conformance/retained-column/1/cases.json") if err != nil { t.Fatal(err) @@ -191,17 +191,22 @@ func TestRetainedColumnRC2PublishedRawValueBounds(t *testing.T) { } } -func TestRetainedColumnRC2Identity(t *testing.T) { +func TestRetainedColumnReleaseIdentity(t *testing.T) { r, code := ContractRetainedColumnDraft("test", RetainedColumnDraftBase()) - if code != 0 || r.Spec.Version != "0.3.0-rc.2" || r.Maturity != "draft" || r.Specification != fmt.Sprintf("https://github.com/Judgment-Pack/judgment-pack-spec/blob/%s/spec/retained-column.md", RetainedColumnDraftRevision) || r.Role != "document-validation" { + if code != 0 || r.Spec.Version != "0.3.0" || r.Spec.Revision != "27cb29bf13e4a2d58d69fe19c7ebfeb762b014fc" || r.Maturity != "draft" || r.Specification != fmt.Sprintf("https://github.com/Judgment-Pack/judgment-pack-spec/blob/%s/spec/retained-column.md", RetainedColumnDraftRevision) || r.Role != "document-validation" { t.Fatal("wrong installed role identity", r) } if _, _, code := SelectRetainedColumnDraft("test", "retained-column/1@ade3914b0b81ef299e5b9a8d6147f37ff3f0976c", "base"); code == 0 { - t.Fatal("historical selector accepted as installed RC2") + t.Fatal("historical selector accepted as the installed release") + } + // The v0.3.0-rc.2 candidate's selector names its own source commit. The + // installed final reader refuses it, as the Spec's own readers now do. + if _, _, code := SelectRetainedColumnDraft("test", "retained-column/1@bcac15b7537ce21879201b5e5dfe795496225121", "base"); code == 0 { + t.Fatal("release-candidate selector accepted as the installed release") } } -func TestRetainedColumnRC2PublishedCapabilityRole(t *testing.T) { +func TestRetainedColumnReleasePublishedCapabilityRole(t *testing.T) { raw, err := fs.ReadFile(retainedColumnArtifacts, "retained-column/official/conformance/retained-column/1/capabilities.json") if err != nil { t.Fatal(err) @@ -217,6 +222,9 @@ func TestRetainedColumnRC2PublishedCapabilityRole(t *testing.T) { t.Fatal(err) } role := table.Roles["document-validation"] + // The corpus files are byte-identical to the v0.3.0-rc.2 candidate's, so the + // table keeps the distribution label it was published with; the release + // identity is carried by release-identity.json and the v0.3.0 inventory. if table.Contract != "retained-column/1" || table.Distribution != "v0.3.0-rc.2" || table.Maturity != "draft" || len(role) != 2 || role[0] != "documentBytes" || role[1] != "frameBytes" { t.Fatal("published role table changed") } diff --git a/internal/mcp/dialect.go b/internal/mcp/dialect.go index 0d79f5f..3d9e313 100644 --- a/internal/mcp/dialect.go +++ b/internal/mcp/dialect.go @@ -44,7 +44,7 @@ func dialectDefinition(operation string) map[string]any { } if operation == "contract" || operation == "canonicalize" || operation == "digest" || operation == "validate" || operation == "check" { dialectDraftSchema(operation, schema, properties) - descriptions[operation] += fmt.Sprintf(" Explicit compatibility draftContract/draftProfile selects the installed RC2 document/frame reader; no preparation, authenticity or execution claim. The %d-byte stdio request limit includes JSON escaping and delimiter; decoded byte limits do not guarantee transport admission.", maxMessageBytes) + descriptions[operation] += fmt.Sprintf(" Explicit compatibility draftContract/draftProfile selects the installed leaf document/frame reader; no preparation, authenticity or execution claim. The %d-byte stdio request limit includes JSON escaping and delimiter; decoded byte limits do not guarantee transport admission.", maxMessageBytes) } return map[string]any{"name": name, "description": descriptions[operation], "inputSchema": schema} } @@ -172,14 +172,14 @@ func dialectDraftSchema(operation string, schema, properties map[string]any) { properties["draftProfile"] = map[string]any{"type": "string", "enum": []string{"base", "extended"}} selectorMembers := []any{map[string]any{"required": []string{"draftContract"}}, map[string]any{"required": []string{"draftProfile"}}} if operation == "validate" { - properties["retainedFrame"] = map[string]any{"type": "boolean", "description": "With exact installed RC2 selection, check document text as a standalone retained frame; no conversion or provenance verification."} + properties["retainedFrame"] = map[string]any{"type": "boolean", "description": "With exact installed leaf selection, check document text as a standalone retained frame; no conversion or provenance verification."} selectorMembers = append(selectorMembers, map[string]any{"required": []string{"retainedFrame"}}) } legacy := map[string]any{"not": map[string]any{"anyOf": selectorMembers}} if operation != "contract" { document := properties["document"].(map[string]any) delete(document, "maxLength") - document["description"] = "Exact JSON text, never a path. The selected legacy/RC2 branch bounds decoded UTF-8 bytes; stdio escaping and delimiter count against the separate request envelope." + document["description"] = "Exact JSON text, never a path. The selected legacy/leaf branch bounds decoded UTF-8 bytes; stdio escaping and delimiter count against the separate request envelope." legacy["properties"] = map[string]any{"document": map[string]any{"maxLength": dialect.MaxDocumentBytes}} } branches := []any{legacy} @@ -236,11 +236,11 @@ func (s *Server) toolRetainedColumnDraft(name string, args json.RawMessage) any } var fields map[string]json.RawMessage if json.Unmarshal(args, &fields) != nil || json.Unmarshal(args, &input) != nil { - return toolError("Expected exact installed RC2 selectors and JSON text arguments.") + return toolError("Expected exact installed leaf selectors and JSON text arguments.") } for _, key := range []string{"draftContract", "draftProfile", "retainedFrame"} { if value, present := fields[key]; present && bytes.Equal(bytes.TrimSpace(value), []byte("null")) { - return toolError("RC2 selection members cannot be null.") + return toolError("Leaf selection members cannot be null.") } } selector, profile := "", "" @@ -290,7 +290,7 @@ func (s *Server) toolRetainedColumnDraft(name string, args json.RawMessage) any encoder := json.NewEncoder(&text) encoder.SetEscapeHTML(false) if err := encoder.Encode(output); err != nil { - return toolError("The selected RC2 report could not be encoded.") + return toolError("The selected leaf report could not be encoded.") } return map[string]any{ "content": []map[string]any{{"type": "text", "text": strings.TrimSuffix(text.String(), "\n")}}, diff --git a/internal/retainedcolumn/artifacts.go b/internal/retainedcolumn/artifacts.go index 3320f28..ab3cd3a 100644 --- a/internal/retainedcolumn/artifacts.go +++ b/internal/retainedcolumn/artifacts.go @@ -14,10 +14,10 @@ import ( const ( Repository = "https://github.com/Judgment-Pack/judgment-pack-spec" - Release = "v0.3.0-rc.2" - Version = "0.3.0-rc.2" + Release = "v0.3.0" + Version = "0.3.0" Contract = "retained-column/1" - InventoryPath = "releases/v0.3.0-rc.2.retained-column.json" + InventoryPath = "releases/v0.3.0.retained-column.json" SchemaPath = "schema/retained-column/1/contract.schema.json" CorpusPath = "conformance/retained-column/1/cases.json" ) @@ -36,8 +36,10 @@ type ReleaseIdentity struct { Tag string `json:"tag"` SpecVersion string `json:"specVersion"` Maturity string `json:"maturity"` - Distribution string `json:"distribution"` - SourceCommit string `json:"sourceCommit"` + // Distribution is present only on a release candidate's identity + // ("release-candidate"). The installed final release carries none. + Distribution string `json:"distribution,omitempty"` + SourceCommit string `json:"sourceCommit"` } type Inventory struct { Contract string `json:"contract"` @@ -85,7 +87,7 @@ func Inspect(source fs.FS, revision, inventoryPin string) (Identity, Inventory, } raw, err := fs.ReadFile(source, "release-identity.json") var release ReleaseIdentity - if err != nil || json.Unmarshal(raw, &release) != nil || release.ReleaseVersion != Version || release.Tag != Release || release.SpecVersion != "0.3.0" || release.Maturity != "draft" || release.Distribution != "release-candidate" || release.SourceCommit != revision { + if err != nil || json.Unmarshal(raw, &release) != nil || release.ReleaseVersion != Version || release.Tag != Release || release.SpecVersion != "0.3.0" || release.Maturity != "draft" || release.Distribution != "" || release.SourceCommit != revision { return fail("release identity differs from the exact installation") } raw, err = fs.ReadFile(source, InventoryPath) diff --git a/internal/retainedcolumn/artifacts_test.go b/internal/retainedcolumn/artifacts_test.go index a8faa27..1075677 100644 --- a/internal/retainedcolumn/artifacts_test.go +++ b/internal/retainedcolumn/artifacts_test.go @@ -30,7 +30,7 @@ func syntheticInstallation(t *testing.T) (fstest.MapFS, string, Identity) { files[name] = &fstest.MapFile{Data: raw} } encode(InventoryPath, inventory) - encode("release-identity.json", ReleaseIdentity{Version, Release, "0.3.0", "draft", "release-candidate", revision}) + encode("release-identity.json", ReleaseIdentity{Version, Release, "0.3.0", "draft", "", revision}) identity := Identity{Version, revision, inventory.SchemaSHA256, inventory.CorpusSHA256} p := Provenance{SourceRepository: Repository, SourceRelease: Release, SourceRevision: revision, SourceKind: "immutable-git-ref", SourceArchiveSHA256: "sha256:" + strings.Repeat("2", 64), InventorySHA256: Digest(files[InventoryPath].Data), Identity: identity} for name, file := range files { @@ -40,7 +40,7 @@ func syntheticInstallation(t *testing.T) (fstest.MapFS, string, Identity) { return files, Digest(files["provenance.json"].Data), identity } func TestVerifySyntheticInstallationAndRefusals(t *testing.T) { - for _, name := range []string{"valid", "manifest pin", "selected identity", "archive identity", "dirty", "moving source", "wrong repository", "missing schema", "changed schema", "changed corpus", "changed inventory", "changed release identity", "extra file", "unmanifested inventory", "duplicate inventory", "bad length"} { + for _, name := range []string{"valid", "manifest pin", "selected identity", "archive identity", "dirty", "moving source", "wrong repository", "missing schema", "changed schema", "changed corpus", "changed inventory", "changed release identity", "candidate release identity", "earlier candidate identity", "extra file", "unmanifested inventory", "duplicate inventory", "bad length"} { t.Run(name, func(t *testing.T) { files, pin, identity := syntheticInstallation(t) editManifest := func(edit func(*Provenance)) { @@ -76,6 +76,23 @@ func TestVerifySyntheticInstallationAndRefusals(t *testing.T) { files[InventoryPath].Data = append(files[InventoryPath].Data, ' ') case "changed release identity": files["release-identity.json"].Data = []byte("{}") + case "candidate release identity", "earlier candidate identity": + // The candidate form carries "distribution"; the second is the exact + // identity v0.3.0-rc.2 generated. The installed final reader refuses + // both, as the Spec's own readers do. The provenance is re-pinned so + // only the release identity check can refuse. + raw := []byte(`{"distribution":"release-candidate","maturity":"draft","releaseVersion":"` + Version + `","sourceCommit":"` + identity.Revision + `","specVersion":"0.3.0","tag":"` + Release + `"}`) + if name == "earlier candidate identity" { + raw = []byte(`{"distribution":"release-candidate","maturity":"draft","releaseVersion":"0.3.0-rc.2","sourceCommit":"` + identity.Revision + `","specVersion":"0.3.0","tag":"v0.3.0-rc.2"}`) + } + files["release-identity.json"].Data = raw + editManifest(func(p *Provenance) { + for i := range p.Files { + if p.Files[i].Path == "release-identity.json" { + p.Files[i] = File{"release-identity.json", len(raw), Digest(raw)} + } + } + }) case "extra file": files["extra.json"] = &fstest.MapFile{Data: []byte("{}")} case "unmanifested inventory": diff --git a/tools/sync-retained-column/main.go b/tools/sync-retained-column/main.go index 8d69da9..e876d47 100644 --- a/tools/sync-retained-column/main.go +++ b/tools/sync-retained-column/main.go @@ -1,4 +1,4 @@ -// Command sync-retained-column imports reviewed bytes from a verified RC2 +// Command sync-retained-column imports reviewed bytes from a verified release // archive and its clean immutable source checkout. It performs no network fetch, // publication verification, registry update or capability selection. package main @@ -23,7 +23,7 @@ import ( "github.com/Judgment-Pack/judgment-pack-runtime/internal/retainedcolumn" ) -const archivePrefix = "judgment-pack-spec-0.3.0-rc.2/" +const archivePrefix = "judgment-pack-spec-" + retainedcolumn.Version + "/" const maxArchiveBytes = 256 << 20 // Maintainer import budget, not a Runtime admission limit. type options struct{ source, destination, archive, archivePin, revision, inventoryPin, sourceRef string } @@ -32,11 +32,11 @@ func main() { var o options flag.StringVar(&o.source, "source", "", "clean official Spec checkout at the verified release tag") flag.StringVar(&o.destination, "destination", "", "new leaf artifact directory (must not exist)") - flag.StringVar(&o.archive, "archive", "", "independently verified official RC2 tar.gz release asset") + flag.StringVar(&o.archive, "archive", "", "independently verified official tar.gz release asset") flag.StringVar(&o.archivePin, "archive-sha256", "", "independently verified sha256: archive digest") flag.StringVar(&o.inventoryPin, "inventory-sha256", "", "reviewed sha256: release inventory digest") flag.StringVar(&o.revision, "source-commit", "", "verified immutable 40-hex source commit") - flag.StringVar(&o.sourceRef, "source-ref", "", "exact release tag v0.3.0-rc.2") + flag.StringVar(&o.sourceRef, "source-ref", "", "exact release tag "+retainedcolumn.Release) flag.Parse() pin, err := run(o) if err != nil { @@ -57,7 +57,7 @@ func git(source string, args ...string) ([]byte, error) { func verifySource(o options) error { return verifySourceWith(o, git) } func verifySourceWith(o options, read func(string, ...string) ([]byte, error)) error { if o.source == "" || o.sourceRef != retainedcolumn.Release { - return errors.New("clean official source checkout and exact RC2 release tag are required") + return errors.New("clean official source checkout and the exact supported release tag are required") } origin, err := read(o.source, "remote", "get-url", "origin") if err != nil { diff --git a/tools/sync-retained-column/main_test.go b/tools/sync-retained-column/main_test.go index 428ed89..0a27c35 100644 --- a/tools/sync-retained-column/main_test.go +++ b/tools/sync-retained-column/main_test.go @@ -37,7 +37,7 @@ func syntheticArchive(t *testing.T) ([]byte, options, fstest.MapFS) { } encode(retainedcolumn.InventoryPath, inv) revision := strings.Repeat("1", 40) - encode("release-identity.json", retainedcolumn.ReleaseIdentity{ReleaseVersion: retainedcolumn.Version, Tag: retainedcolumn.Release, SpecVersion: "0.3.0", Maturity: "draft", Distribution: "release-candidate", SourceCommit: revision}) + encode("release-identity.json", retainedcolumn.ReleaseIdentity{ReleaseVersion: retainedcolumn.Version, Tag: retainedcolumn.Release, SpecVersion: "0.3.0", Maturity: "draft", SourceCommit: revision}) raw := makeArchive(t, files, nil) return raw, options{archivePin: retainedcolumn.Digest(raw), revision: revision, inventoryPin: retainedcolumn.Digest(files[retainedcolumn.InventoryPath].Data)}, files } From 15bc1de14c84eb39d0631b22ed34e0bcd08c5db9 Mon Sep 17 00:00:00 2001 From: kikashy <35789537+kikashy@users.noreply.github.com> Date: Thu, 29 Oct 2026 04:12:00 -0400 Subject: [PATCH 2/3] Fingerprint the audit trail's persisted line formats (#327) Add a generated catalog of every persisted audit-trail line type (evaluations records and discontinuities, signature-sidecar record signatures and key rotations, stamps) with its normalized JSON shape and SHA-256 digest; hand-kept change records naming the format decision behind each digest; and a separate format-status record (writes, baseline, released) checked against the writer constants. The audit-formats check (dev.json, and the CI matrix with the other Go steps) regenerates the catalog and fails on a stale catalog, a digest without a change record or a status the writer constants contradict. Co-Authored-By: Claude Opus 5.5 Signed-off-by: kikashy <35789537+kikashy@users.noreply.github.com> --- .github/workflows/ci.yml | 2 + dev.json | 2 + docs/releasing.md | 21 ++ internal/audit/formats.go | 28 ++ internal/auditformat/auditformat.go | 379 +++++++++++++++++++++++ internal/auditformat/auditformat_test.go | 155 +++++++++ internal/auditformat/catalog.json | 36 +++ internal/auditformat/changes.json | 44 +++ internal/auditformat/status.json | 8 + jpdev.json | 6 +- tools/audit-formats/main.go | 53 ++++ 11 files changed, 733 insertions(+), 1 deletion(-) create mode 100644 internal/audit/formats.go create mode 100644 internal/auditformat/auditformat.go create mode 100644 internal/auditformat/auditformat_test.go create mode 100644 internal/auditformat/catalog.json create mode 100644 internal/auditformat/changes.json create mode 100644 internal/auditformat/status.json create mode 100644 tools/audit-formats/main.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03c1749..de1dd2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,8 @@ jobs: run: go build -trimpath ./cmd/jpack env: CGO_ENABLED: "0" + - name: Check audit line formats + run: go run ./tools/audit-formats check: name: check diff --git a/dev.json b/dev.json index 615fc56..d9569cd 100644 --- a/dev.json +++ b/dev.json @@ -17,6 +17,8 @@ "summary": "^(ok|FAIL|---) ", "timeout": 3000}, {"name": "conformance", "run": "go run ./cmd/jpack spec test-conformance --quiet", "toolchain": "go", "timeout": 1200}, {"name": "build", "run": "CGO_ENABLED=0 go build -trimpath -o /dev/null ./cmd/jpack", "toolchain": "go", "timeout": 1200}, + {"name": "audit-formats", "run": "go run ./tools/audit-formats", "toolchain": "go", + "summary": "^audit-formats:", "timeout": 600}, {"name": "registry-template", "run": "python3 scripts/check-registry-template.py", "toolchain": "python", "summary": "^registry template:", "timeout": 300} ], diff --git a/docs/releasing.md b/docs/releasing.md index 0d4f07b..89cc43c 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -120,6 +120,26 @@ refuses unpinned resources, changed or missing installed bytes, extra inventory, unsupported identities and disagreement with the selected reader. No final Core, producer or preparation-verifier claim follows from this document-validation role. +## Audit line formats + +The audit trail's persisted lines (`evaluations.jsonl` records and discontinuities, +`signatures.jsonl` record signatures and key rotations, `stamps.jsonl` stamps) are +fingerprinted in `internal/auditformat/`. `catalog.json` is generated: each line +type's normalized JSON shape and its SHA-256 digest. `changes.json` is written by +hand and only appended to: every digest a line type has had, with the decision that +admitted it (`baseline`, `compatible-addition` at the same format version, or +`new-format-version` at the next integer with its `readingRule`) and a reason. +`status.json` is kept apart: per trail file, the format the code `writes` (checked +against the writer constant), the last accepted compatibility `baseline`, and the +last format `released`. A format that shipped in any release, a pre-release +included, carries compatibility obligations, so `baseline <= released <= writes`. + +`go run ./tools/audit-formats` (the `audit-formats` check) regenerates the catalog in +memory and fails on a stale catalog, on a digest no change record names, and on a +status the writer constants contradict. After changing a persisted line, run it with +`-write`, append the change record, and update `status.json` when the format version +moves. When a release ships a new format version, set `released` to it. + ## Validate locally Install the exact GoReleaser version used by the workflow, then run: @@ -132,6 +152,7 @@ env GO111MODULE=on go test ./... env GO111MODULE=on go run ./cmd/jpack spec test-conformance --quiet env GO111MODULE=on go run ./cmd/jpack spec test-conformance --spec-version 0.2.0-draft --quiet env GO111MODULE=on go run ./cmd/jpack experimental evaluate-corpus +env GO111MODULE=on go run ./tools/audit-formats env GO111MODULE=on go run ./tools/check-release --tag goreleaser check goreleaser release --snapshot --clean diff --git a/internal/audit/formats.go b/internal/audit/formats.go new file mode 100644 index 0000000..fa711b2 --- /dev/null +++ b/internal/audit/formats.go @@ -0,0 +1,28 @@ +package audit + +import "reflect" + +// LineFormat is one persisted line type of a project's audit trail: the file +// it is written to, its name, the format version the writer stamps on that +// file's lines, and the Go type it is encoded from. The catalog in +// internal/auditformat fingerprints each one (#327), so a change to a member +// of a persisted line cannot pass without a committed format decision. +type LineFormat struct { + File string + Line string + Version string + Type reflect.Type +} + +// LineFormats lists every line type this runtime persists, in file order. A +// new persisted line type is added here, or the catalog cannot fingerprint it. +func LineFormats() []LineFormat { + return []LineFormat{ + // One shape carries both decision kinds, evaluation and graph-composite. + {FileName, "record", RecordVersion, reflect.TypeOf(Record{})}, + {FileName, KindDiscontinuity, RecordVersion, reflect.TypeOf(discontinuityRecord{})}, + {SidecarName, KindRecordSignature, SidecarVersion, reflect.TypeOf(recordSignatureLine{})}, + {SidecarName, KindKeyRotation, SidecarVersion, reflect.TypeOf(keyRotationLine{})}, + {StampsName, "stamp", StampVersion, reflect.TypeOf(stampLine{})}, + } +} diff --git a/internal/auditformat/auditformat.go b/internal/auditformat/auditformat.go new file mode 100644 index 0000000..be6bb70 --- /dev/null +++ b/internal/auditformat/auditformat.go @@ -0,0 +1,379 @@ +// Package auditformat fingerprints the audit trail's persisted line formats +// (#327). It keeps three records beside this file: +// +// - catalog.json, generated: every persisted line type, its normalized type +// and that type's SHA-256 digest. +// - changes.json, written by hand: the format decision behind every digest a +// line type has had, appended and never rewritten. +// - status.json, written by hand and kept apart: per trail file, the format +// the code writes, the last accepted compatibility baseline and the last +// format released. A format that shipped in any release, a pre-release +// included, carries compatibility obligations. +// +// Check regenerates the catalog from the Go types and refuses a catalog that +// differs, a digest no change record names, and a status the writer constants +// contradict. +package auditformat + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/Judgment-Pack/judgment-pack-runtime/internal/audit" +) + +// Normalization names the grammar Normalize writes. Changing the grammar +// changes every digest, so it is versioned with the catalog. +const Normalization = "audit-line-type/1" + +// The three hand-kept files and the generated catalog, by name. +const ( + CatalogFile = "catalog.json" + ChangesFile = "changes.json" + StatusFile = "status.json" +) + +// The decisions a change record may name. +const ( + DecisionBaseline = "baseline" + DecisionCompatibleAddition = "compatible-addition" + DecisionNewFormatVersion = "new-format-version" +) + +// Entry is one line type in the catalog. +type Entry struct { + File string `json:"file"` + Line string `json:"line"` + Type string `json:"type"` + SHA256 string `json:"sha256"` +} + +// Catalog is the generated inventory. +type Catalog struct { + CatalogVersion int `json:"catalogVersion"` + Normalization string `json:"normalization"` + Lines []Entry `json:"lines"` +} + +// Change is one committed format decision for one line type's digest. +type Change struct { + File string `json:"file"` + Line string `json:"line"` + SHA256 string `json:"sha256"` + FormatVersion string `json:"formatVersion"` + Decision string `json:"decision"` + // ReadingRule is required for a new format version: how a reader tells + // the versions apart and what it does with each. + ReadingRule string `json:"readingRule,omitempty"` + Reason string `json:"reason"` +} + +// Changes is the change-record file. +type Changes struct { + Changes []Change `json:"changes"` +} + +// FileStatus is the format status of one trail file. +type FileStatus struct { + File string `json:"file"` + Writes string `json:"writes"` + Baseline string `json:"baseline"` + Released string `json:"released"` +} + +// Status is the format-status file. +type Status struct { + StatusVersion int `json:"statusVersion"` + Files []FileStatus `json:"files"` +} + +// Generate fingerprints every line type the audit package persists. +func Generate() (Catalog, map[string]string) { + catalog := Catalog{CatalogVersion: 1, Normalization: Normalization} + writers := map[string]string{} + for _, format := range audit.LineFormats() { + catalog.Lines = append(catalog.Lines, entry(format.File, format.Line, format.Type)) + writers[format.File] = format.Version + } + return catalog, writers +} + +func entry(file, line string, t reflect.Type) Entry { + normalized := Normalize(t) + sum := sha256.Sum256([]byte(normalized)) + return Entry{file, line, normalized, "sha256:" + hex.EncodeToString(sum[:])} +} + +// Encode is the catalog file's exact bytes. +func Encode(catalog Catalog) []byte { + var buffer bytes.Buffer + encoder := json.NewEncoder(&buffer) + encoder.SetEscapeHTML(false) + encoder.SetIndent("", " ") + _ = encoder.Encode(catalog) // strings and integers only + return buffer.Bytes() +} + +var rawMessage = reflect.TypeOf(json.RawMessage(nil)) +var marshaler = reflect.TypeOf((*json.Marshaler)(nil)).Elem() + +// Normalize is the JSON shape encoding/json writes for t: members in encoding +// order with their names (bare when an identifier, quoted otherwise), +// omitempty marked by "?" and ",string" by "~", "|null" where a nil pointer, +// slice or map encodes as null, and +// member types reduced to JSON kinds. Go type and field names do not appear, +// so a rename that keeps the encoded shape keeps the digest. A type with its +// own MarshalJSON is marked "custom" and normalized by its fields, so a change +// to its fields is caught; a change to the method's code alone is not. +func Normalize(t reflect.Type) string { + return normalize(t, map[reflect.Type]bool{}) +} + +func normalize(t reflect.Type, open map[reflect.Type]bool) string { + if t == rawMessage { + return "json" + } + prefix := "" + if t.Kind() != reflect.Pointer && t.Kind() != reflect.Interface && (t.Implements(marshaler) || reflect.PointerTo(t).Implements(marshaler)) { + prefix = "custom" + } + switch t.Kind() { + case reflect.Bool: + return prefix + "boolean" + case reflect.String: + return prefix + "string" + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return prefix + "integer" + case reflect.Float32, reflect.Float64: + return prefix + "number" + case reflect.Interface: + return "any" + case reflect.Pointer: + return normalize(t.Elem(), open) + "|null" + case reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { + return prefix + "base64|null" + } + return prefix + "[" + normalize(t.Elem(), open) + "]|null" + case reflect.Array: + return prefix + "[" + normalize(t.Elem(), open) + "]" + case reflect.Map: + return prefix + "{" + normalize(t.Key(), open) + ":" + normalize(t.Elem(), open) + "}|null" + case reflect.Struct: + if open[t] { + return "recursive" + } + open[t] = true + defer delete(open, t) + var members []string + for _, field := range fields(t) { + name := field.name + if field.omitempty { + name += "?" + } + if field.quoted { + name += "~" + } + if !plainName.MatchString(field.name) { + name = strconv.Quote(name) + } + members = append(members, name+":"+normalize(field.typ, open)) + } + return prefix + "{" + strings.Join(members, ",") + "}" + } + return "unsupported:" + t.Kind().String() +} + +type field struct { + name string + omitempty bool + quoted bool + typ reflect.Type +} + +// fields is the members encoding/json writes for a struct, in its order. +// Untagged embedded structs are promoted; the persisted line types use none +// with conflicting names, so dominance rules are not reproduced. +func fields(t reflect.Type) []field { + var out []field + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + tag := f.Tag.Get("json") + if tag == "-" { + continue + } + name, options, _ := strings.Cut(tag, ",") + if f.Anonymous && name == "" { + embedded := f.Type + if embedded.Kind() == reflect.Pointer { + embedded = embedded.Elem() + } + if embedded.Kind() == reflect.Struct { + out = append(out, fields(embedded)...) + continue + } + } + if !f.IsExported() { + continue + } + if name == "" { + name = f.Name + } + option := func(want string) bool { + for _, o := range strings.Split(options, ",") { + if o == want { + return true + } + } + return false + } + out = append(out, field{name, option("omitempty"), option("string"), f.Type}) + } + return out +} + +var digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) + +// plainName is a member name written bare; any other is written quoted. +var plainName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +// Check compares the committed catalog, change records and status with the +// generated catalog and the writer constants (file -> version). It returns +// every disagreement, so one run names them all. +func Check(catalogRaw, changesRaw, statusRaw []byte, generated Catalog, writers map[string]string) []error { + var problems []error + fail := func(format string, args ...any) { problems = append(problems, fmt.Errorf(format, args...)) } + var committed Catalog + if err := strictDecode(catalogRaw, &committed); err != nil { + fail("%s: %v", CatalogFile, err) + } else if !reflect.DeepEqual(committed, generated) { + fail("%s differs from the line types the code persists; regenerate it with `go run ./tools/audit-formats -write` and commit a change record for each changed digest", CatalogFile) + } + var changes Changes + if err := strictDecode(changesRaw, &changes); err != nil { + fail("%s: %v", ChangesFile, err) + return problems + } + var status Status + if err := strictDecode(statusRaw, &status); err != nil { + fail("%s: %v", StatusFile, err) + return problems + } + known := map[string]Entry{} + for _, line := range generated.Lines { + known[line.File+"\x00"+line.Line] = line + } + // Walk each line type's records in order: the first is its baseline, each + // later one names a new digest and the decision that admitted it. + last := map[string]Change{} + for i, change := range changes.Changes { + key := change.File + "\x00" + change.Line + where := fmt.Sprintf("%s change %d (%s %s)", ChangesFile, i+1, change.File, change.Line) + if _, ok := known[key]; !ok { + fail("%s: no persisted line type has this name", where) + continue + } + version, err := formatVersion(change.FormatVersion) + if err != nil || !digestPattern.MatchString(change.SHA256) { + fail("%s: needs a sha256: digest and a positive integer format version", where) + continue + } + previous, seen := last[key] + switch { + case !seen && change.Decision != DecisionBaseline: + fail("%s: a line type's first record is its baseline", where) + case seen && change.Decision == DecisionBaseline: + fail("%s: a line type has one baseline", where) + case seen && change.SHA256 == previous.SHA256: + fail("%s: names the digest the previous record already admitted", where) + case seen && strings.TrimSpace(change.Reason) == "": + fail("%s: a change record names its reason", where) + case change.Decision == DecisionCompatibleAddition: + if prior, _ := formatVersion(previous.FormatVersion); version != prior { + fail("%s: a compatible addition keeps format version %s", where, previous.FormatVersion) + } + case change.Decision == DecisionNewFormatVersion: + if prior, _ := formatVersion(previous.FormatVersion); version != prior+1 || strings.TrimSpace(change.ReadingRule) == "" { + fail("%s: a new format version is the next integer and states its reading rule", where) + } + case change.Decision != DecisionBaseline: + fail("%s: decision %q is none of %s, %s, %s", where, change.Decision, DecisionBaseline, DecisionCompatibleAddition, DecisionNewFormatVersion) + } + last[key] = change + } + files := map[string][]Entry{} + for _, line := range generated.Lines { + files[line.File] = append(files[line.File], line) + change, ok := last[line.File+"\x00"+line.Line] + switch { + case !ok: + fail("%s %s: no change record admits this line type; add its baseline to %s", line.File, line.Line, ChangesFile) + case change.SHA256 != line.SHA256: + fail("%s %s: digest %s has no change record naming its format decision (the last admitted is %s)", line.File, line.Line, line.SHA256, change.SHA256) + case change.FormatVersion != writers[line.File]: + fail("%s %s: the last change record is format version %s, the writer writes %s", line.File, line.Line, change.FormatVersion, writers[line.File]) + } + } + statuses := map[string]bool{} + for _, file := range status.Files { + where := fmt.Sprintf("%s %s", StatusFile, file.File) + if _, ok := files[file.File]; !ok || statuses[file.File] { + fail("%s: not a trail file, or named twice", where) + continue + } + statuses[file.File] = true + writes, errW := formatVersion(file.Writes) + baseline, errB := formatVersion(file.Baseline) + released, errR := formatVersion(file.Released) + switch { + case errW != nil || errB != nil || errR != nil: + fail("%s: writes, baseline and released are positive integer format versions", where) + case file.Writes != writers[file.File]: + fail("%s: says the code writes format %s; the writer constant is %s", where, file.Writes, writers[file.File]) + case baseline > released || released > writes: + // A released format carries compatibility obligations: the + // baseline readers keep accepting reaches back to it. + fail("%s: needs baseline <= released <= writes", where) + } + } + names := make([]string, 0, len(files)) + for name := range files { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + if !statuses[name] { + fail("%s: %s has no format status", StatusFile, name) + } + } + return problems +} + +func formatVersion(text string) (int, error) { + value, err := strconv.Atoi(text) + if err != nil || value < 1 || strconv.Itoa(value) != text { + return 0, fmt.Errorf("not a positive integer: %q", text) + } + return value, nil +} + +func strictDecode(raw []byte, into any) error { + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(into); err != nil { + return err + } + if decoder.More() { + return fmt.Errorf("trailing data") + } + return nil +} diff --git a/internal/auditformat/auditformat_test.go b/internal/auditformat/auditformat_test.go new file mode 100644 index 0000000..13ad246 --- /dev/null +++ b/internal/auditformat/auditformat_test.go @@ -0,0 +1,155 @@ +package auditformat + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "reflect" + "strings" + "testing" +) + +// The committed catalog, change records and status agree with the code. +func TestCommittedFormatsAgree(t *testing.T) { + generated, writers := Generate() + if problems := Check(read(t, CatalogFile), read(t, ChangesFile), read(t, StatusFile), generated, writers); len(problems) != 0 { + t.Fatal(problems) + } + if len(generated.Lines) != 5 { + t.Fatalf("persisted line types: %d", len(generated.Lines)) + } +} + +func read(t *testing.T, name string) []byte { + t.Helper() + raw, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + return raw +} + +type customString string + +func (customString) MarshalJSON() ([]byte, error) { return []byte(`""`), nil } + +type promotedMembers struct { + Promoted string `json:"promoted"` +} + +// handwrittenShape exercises each normalization rule once. +type handwrittenShape struct { + A string `json:"a"` + B int64 `json:"b,omitempty"` + C int `json:"c,string"` + D bool `json:"d-e"` + F *struct { + G string `json:"g"` + } `json:"f"` + H []string `json:"h"` + I map[string]float64 `json:"i"` + J json.RawMessage `json:"j"` + K []byte `json:"k"` + L customString `json:"l"` + Hidden string `json:"-"` + hidden string + promotedMembers + M any `json:"m"` +} + +// The expected text and digest are written by hand; the digest was computed +// outside this code from the text. +func TestNormalizeHandwrittenShape(t *testing.T) { + const want = `{a:string,b?:integer,c~:integer,"d-e":boolean,f:{g:string}|null,h:[string]|null,i:{string:number}|null,j:json,k:base64|null,l:customstring,promoted:string,m:any}` + got := entry("f", "l", reflect.TypeOf(handwrittenShape{})) + if got.Type != want { + t.Fatalf("normalized:\n got %s\nwant %s", got.Type, want) + } + if got.SHA256 != "sha256:93884f1a0c8f9f11c275fe0df93b9fc24d98db29c4c2d884bc641e7db74b7e94" { + t.Fatal("digest", got.SHA256) + } + renamed := struct { + Renamed string `json:"a"` + }{} + if Normalize(reflect.TypeOf(renamed)) != "{a:string}" { + t.Fatal("a Go field rename changed the encoded shape") + } +} + +// Changing a member of any persisted line without a change record fails, even +// after the catalog is regenerated. +func TestChangedMemberWithoutRecordFails(t *testing.T) { + baseline, writers := Generate() + for i, line := range baseline.Lines { + changed, _ := Generate() + changed.Lines[i].Type = strings.TrimSuffix(line.Type, "}") + ",added?:string}" + sum := sha256.Sum256([]byte(changed.Lines[i].Type)) + changed.Lines[i].SHA256 = "sha256:" + hex.EncodeToString(sum[:]) + problems := Check(Encode(changed), read(t, ChangesFile), read(t, StatusFile), changed, writers) + if len(problems) != 1 || !strings.Contains(problems[0].Error(), line.File+" "+line.Line+": digest") { + t.Fatalf("%s %s: %v", line.File, line.Line, problems) + } + // The stale catalog is refused as well when it is not regenerated. + if problems := Check(read(t, CatalogFile), read(t, ChangesFile), read(t, StatusFile), changed, writers); len(problems) != 2 { + t.Fatalf("%s %s stale catalog: %v", line.File, line.Line, problems) + } + } +} + +type syntheticV1 struct { + Kind string `json:"kind"` +} +type syntheticV2 struct { + Kind string `json:"kind"` + Extra string `json:"extra,omitempty"` +} + +func TestCheckDecisionsAndStatus(t *testing.T) { + v1 := entry("trail.jsonl", "line", reflect.TypeOf(syntheticV1{})) + v2 := entry("trail.jsonl", "line", reflect.TypeOf(syntheticV2{})) + baseline := `{"file":"trail.jsonl","line":"line","sha256":"` + v1.SHA256 + `","formatVersion":"1","decision":"baseline","reason":"first"}` + addition := `{"file":"trail.jsonl","line":"line","sha256":"` + v2.SHA256 + `","formatVersion":"1","decision":"compatible-addition","reason":"optional member"}` + newVersion := `{"file":"trail.jsonl","line":"line","sha256":"` + v2.SHA256 + `","formatVersion":"2","decision":"new-format-version","readingRule":"version 1 lines lack extra","reason":"new member"}` + status := func(writes, baseline, released string) string { + return `{"statusVersion":1,"files":[{"file":"trail.jsonl","writes":"` + writes + `","baseline":"` + baseline + `","released":"` + released + `"}]}` + } + for _, tc := range []struct { + name string + entry Entry + changes string + status string + writer string + ok bool + }{ + {"baseline", v1, baseline, status("1", "1", "1"), "1", true}, + {"no record for change", v2, baseline, status("1", "1", "1"), "1", false}, + {"compatible addition", v2, baseline + "," + addition, status("1", "1", "1"), "1", true}, + {"new version", v2, baseline + "," + newVersion, status("2", "1", "1"), "2", true}, + {"new version without reading rule", v2, baseline + "," + strings.Replace(newVersion, `"readingRule":"version 1 lines lack extra",`, "", 1), status("2", "1", "1"), "2", false}, + {"new version skipping one", v2, baseline + "," + strings.Replace(newVersion, `"formatVersion":"2"`, `"formatVersion":"3"`, 1), status("3", "1", "1"), "3", false}, + {"new version, writer constant behind", v2, baseline + "," + newVersion, status("2", "1", "1"), "1", false}, + {"addition changing version", v2, baseline + "," + strings.Replace(addition, `"formatVersion":"1"`, `"formatVersion":"2"`, 1), status("2", "1", "1"), "2", false}, + {"no reason", v2, baseline + "," + strings.Replace(addition, `"optional member"`, `" "`, 1), status("1", "1", "1"), "1", false}, + {"unknown decision", v2, baseline + "," + strings.Replace(addition, "compatible-addition", "rename", 1), status("1", "1", "1"), "1", false}, + {"second baseline", v2, baseline + "," + strings.Replace(addition, "compatible-addition", "baseline", 1), status("1", "1", "1"), "1", false}, + {"first record not a baseline", v2, addition, status("1", "1", "1"), "1", false}, + {"repeated digest", v1, baseline + "," + strings.Replace(addition, v2.SHA256, v1.SHA256, 1), status("1", "1", "1"), "1", false}, + {"unknown line", v1, baseline + "," + strings.Replace(baseline, `"line":"line"`, `"line":"other"`, 1), status("1", "1", "1"), "1", false}, + {"unknown member", v1, strings.Replace(baseline, `"reason"`, `"note":"x","reason"`, 1), status("1", "1", "1"), "1", false}, + {"writer ahead of change records", v1, baseline, status("2", "1", "1"), "2", false}, + {"status contradicts writer", v1, baseline, status("2", "1", "1"), "1", false}, + {"baseline after released", v1, baseline, status("1", "2", "1"), "1", false}, + {"released after writes", v2, baseline + "," + newVersion, status("2", "1", "3"), "2", false}, + {"version not canonical", v1, baseline, status("01", "1", "1"), "1", false}, + {"no status", v1, baseline, `{"statusVersion":1,"files":[]}`, "1", false}, + } { + t.Run(tc.name, func(t *testing.T) { + generated := Catalog{CatalogVersion: 1, Normalization: Normalization, Lines: []Entry{tc.entry}} + problems := Check(Encode(generated), []byte(`{"changes":[`+tc.changes+`]}`), []byte(tc.status), generated, map[string]string{"trail.jsonl": tc.writer}) + if (len(problems) == 0) != tc.ok { + t.Fatal(problems) + } + }) + } +} diff --git a/internal/auditformat/catalog.json b/internal/auditformat/catalog.json new file mode 100644 index 0000000..de2c1ac --- /dev/null +++ b/internal/auditformat/catalog.json @@ -0,0 +1,36 @@ +{ + "catalogVersion": 1, + "normalization": "audit-line-type/1", + "lines": [ + { + "file": "evaluations.jsonl", + "line": "record", + "type": "{recordVersion:string,trail?:string,sequence?:integer,previous?:string,run:string,at:string,kind:string,surface:string,tool:{name:string,version:string,digest?:string},evaluatorSpecVersion:string,pack?:{id:string,version:string,specVersion:string,digest:string}|null,graph?:{id:string,version:string,formatVersion?:string,digest?:string,node?:string,resultNode?:string}|null,inputs?:{facts:json,evidence:json,evidenceSupplied:boolean}|null,draftPrototype?:{rfc:string,status:string,operators:[string]|null,outcomes?:[string]|null,packValidUnderSpecVersion:boolean,note:string}|null,artifact?:{specVersion:string,bundleDigest:string,provenance:string}|null,reviewed?:boolean|null,reviewedSet?:{lockDigest:string,lockVersion:string,configDigest:string}|null,cites?:[{sessionId:string,callIndex:integer,signature:string}]|null,binding?:{kind:string,digest:string}|null,unknownCauses?:[{path?:string|null,evidenceRequirement?:string,within?:string|null,cause:string,factType?:string}]|null,typeMismatches?:[{path:string,within?:string|null,operator:string,factType:string,operandTypes:[string]|null}]|null,contributors?:[{kind:string,id:string}]|null,disposition:json}", + "sha256": "sha256:3748655634fc2b135437c83631d6739db971516d3fd44d131f988183dd69549d" + }, + { + "file": "evaluations.jsonl", + "line": "discontinuity", + "type": "{recordVersion:string,trail:string,sequence:integer,previous:string,run:string,at:string,kind:string,surface:string,tool:{name:string,version:string,digest?:string},discontinuity:{reason:string,line:integer,bytes:integer,digest:string}}", + "sha256": "sha256:45b453544f10842bdaa934fd3443dea52c0f5c5d8b3c1dcba860b6c6cd7d8480" + }, + { + "file": "signatures.jsonl", + "line": "record-signature", + "type": "{keyId:string,kind:string,record:string,sequence:integer,sidecarVersion:string,signature:string,trail:string}", + "sha256": "sha256:f3d096d4fdebc66d951fa7592bc86336fa0c0352f30aaffefabd1a9f5cfa2582" + }, + { + "file": "signatures.jsonl", + "line": "key-rotation", + "type": "{at:integer,keyId:string,kind:string,next:string,sidecarVersion:string,signature:string,trail:string}", + "sha256": "sha256:8d7d8e7c05f8fa3b067ce82ac13e9adff1a6fcc3448eb1d001f0a8459cbfb782" + }, + { + "file": "stamps.jsonl", + "line": "stamp", + "type": "{checkpoint:{checkpointVersion:string,recordDigest:string,sequence:integer,trail:string},stampVersion:string,token:string}", + "sha256": "sha256:316f48ea2f46150a2e28f9583037700125d77ba606ea7c6b2e7f4f097800f5eb" + } + ] +} diff --git a/internal/auditformat/changes.json b/internal/auditformat/changes.json new file mode 100644 index 0000000..c17ee6e --- /dev/null +++ b/internal/auditformat/changes.json @@ -0,0 +1,44 @@ +{ + "changes": [ + { + "file": "evaluations.jsonl", + "line": "record", + "sha256": "sha256:3748655634fc2b135437c83631d6739db971516d3fd44d131f988183dd69549d", + "formatVersion": "1", + "decision": "baseline", + "reason": "The evaluation and graph-composite record shape as of v0.39.0, written under recordVersion 1." + }, + { + "file": "evaluations.jsonl", + "line": "discontinuity", + "sha256": "sha256:45b453544f10842bdaa934fd3443dea52c0f5c5d8b3c1dcba860b6c6cd7d8480", + "formatVersion": "1", + "decision": "baseline", + "reason": "The discontinuity record audit repair appends (ADR-0047), released since v0.26.0." + }, + { + "file": "signatures.jsonl", + "line": "record-signature", + "sha256": "sha256:f3d096d4fdebc66d951fa7592bc86336fa0c0352f30aaffefabd1a9f5cfa2582", + "formatVersion": "1", + "decision": "baseline", + "reason": "The record-signature sidecar line, released since v0.26.0." + }, + { + "file": "signatures.jsonl", + "line": "key-rotation", + "sha256": "sha256:8d7d8e7c05f8fa3b067ce82ac13e9adff1a6fcc3448eb1d001f0a8459cbfb782", + "formatVersion": "1", + "decision": "baseline", + "reason": "The key-rotation sidecar line, released since v0.26.0." + }, + { + "file": "stamps.jsonl", + "line": "stamp", + "sha256": "sha256:316f48ea2f46150a2e28f9583037700125d77ba606ea7c6b2e7f4f097800f5eb", + "formatVersion": "1", + "decision": "baseline", + "reason": "The time-stamp line, released since v0.26.0." + } + ] +} diff --git a/internal/auditformat/status.json b/internal/auditformat/status.json new file mode 100644 index 0000000..0064944 --- /dev/null +++ b/internal/auditformat/status.json @@ -0,0 +1,8 @@ +{ + "statusVersion": 1, + "files": [ + {"file": "evaluations.jsonl", "writes": "1", "baseline": "1", "released": "1"}, + {"file": "signatures.jsonl", "writes": "1", "baseline": "1", "released": "1"}, + {"file": "stamps.jsonl", "writes": "1", "baseline": "1", "released": "1"} + ] +} diff --git a/jpdev.json b/jpdev.json index 4f51294..7c23dfc 100644 --- a/jpdev.json +++ b/jpdev.json @@ -4,7 +4,11 @@ { "paths": ["**/*.go", "go.mod", "go.sum"], "checks": ["local-paths", "agent-docs", "diff-check", - "go-fmt", "go-vet", "go-test", "conformance", "build"] + "go-fmt", "go-vet", "go-test", "conformance", "build", "audit-formats"] + }, + { + "paths": ["internal/auditformat/*.json"], + "checks": ["local-paths", "agent-docs", "diff-check", "go-test", "audit-formats"] }, { "paths": ["internal/artifacts/jps/**", diff --git a/tools/audit-formats/main.go b/tools/audit-formats/main.go new file mode 100644 index 0000000..65e69f2 --- /dev/null +++ b/tools/audit-formats/main.go @@ -0,0 +1,53 @@ +// Command audit-formats regenerates the audit trail's line-format catalog and +// checks it, the committed change records and the format status (#327). Run it +// from the repository root. -write rewrites the generated catalog only; change +// records and the status are written by hand. +package main + +import ( + "bytes" + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/Judgment-Pack/judgment-pack-runtime/internal/auditformat" +) + +func main() { + dir := flag.String("dir", filepath.Join("internal", "auditformat"), "directory holding the catalog, change records and status") + write := flag.Bool("write", false, "rewrite the generated catalog before checking") + flag.Parse() + os.Exit(run(*dir, *write)) +} + +func run(dir string, write bool) int { + generated, writers := auditformat.Generate() + catalogPath := filepath.Join(dir, auditformat.CatalogFile) + if write { + current, _ := os.ReadFile(catalogPath) + if encoded := auditformat.Encode(generated); !bytes.Equal(current, encoded) { + if err := os.WriteFile(catalogPath, encoded, 0o644); err != nil { + fmt.Fprintln(os.Stderr, "audit-formats:", err) + return 1 + } + fmt.Println("audit-formats: rewrote", catalogPath) + } + } + read := func(name string) []byte { + raw, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + fmt.Fprintln(os.Stderr, "audit-formats:", err) + } + return raw + } + problems := auditformat.Check(read(auditformat.CatalogFile), read(auditformat.ChangesFile), read(auditformat.StatusFile), generated, writers) + for _, problem := range problems { + fmt.Fprintln(os.Stderr, "audit-formats:", problem) + } + if len(problems) != 0 { + return 1 + } + fmt.Printf("audit-formats: %d persisted line types match their committed format decisions\n", len(generated.Lines)) + return 0 +} From 4760af696b0f531604013b3f2852efedd0c31845 Mon Sep 17 00:00:00 2001 From: kikashy <35789537+kikashy@users.noreply.github.com> Date: Thu, 29 Oct 2026 04:49:37 -0400 Subject: [PATCH 3/3] audit-formats: fingerprint the disposition, pin released records (#327 review) F1: the record's canonical disposition is built from maps, so it was an opaque json value in the fingerprint. Declare its members in result.CanonicalDispositionShape and fingerprint the record through it; the check now holds exemplar lines from the writers' own encoders to the declared shapes and grammars (an undeclared member, a declared member no exemplar writes, or a stamp token that is not standard padded base64 fails). F2: status.json pins the released change records' count and digest; rewriting, reordering or removing one fails. F3: state that the fingerprint covers shape and declared grammars only. F5: remove a CI comment left behind by #333. Co-Authored-By: Claude Opus 5.5 Signed-off-by: kikashy <35789537+kikashy@users.noreply.github.com> --- .github/workflows/ci.yml | 1 - docs/releasing.md | 43 ++-- internal/audit/formats.go | 74 +++++- internal/audit/stamp.go | 5 +- internal/auditformat/auditformat.go | 283 +++++++++++++++++++++-- internal/auditformat/auditformat_test.go | 115 ++++++++- internal/auditformat/catalog.json | 8 +- internal/auditformat/changes.json | 8 +- internal/auditformat/status.json | 1 + internal/result/disposition_shape.go | 21 ++ tools/audit-formats/main.go | 7 +- 11 files changed, 501 insertions(+), 65 deletions(-) create mode 100644 internal/result/disposition_shape.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de1dd2d..57ae4a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,7 +68,6 @@ jobs: with: fetch-depth: 0 persist-credentials: false - # First, so that a manifest that does not parse fails before the long steps start. - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: diff --git a/docs/releasing.md b/docs/releasing.md index 89cc43c..9679062 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -125,20 +125,35 @@ producer or preparation-verifier claim follows from this document-validation rol The audit trail's persisted lines (`evaluations.jsonl` records and discontinuities, `signatures.jsonl` record signatures and key rotations, `stamps.jsonl` stamps) are fingerprinted in `internal/auditformat/`. `catalog.json` is generated: each line -type's normalized JSON shape and its SHA-256 digest. `changes.json` is written by -hand and only appended to: every digest a line type has had, with the decision that -admitted it (`baseline`, `compatible-addition` at the same format version, or -`new-format-version` at the next integer with its `readingRule`) and a reason. -`status.json` is kept apart: per trail file, the format the code `writes` (checked -against the writer constant), the last accepted compatibility `baseline`, and the -last format `released`. A format that shipped in any release, a pre-release -included, carries compatibility obligations, so `baseline <= released <= writes`. - -`go run ./tools/audit-formats` (the `audit-formats` check) regenerates the catalog in -memory and fails on a stale catalog, on a digest no change record names, and on a -status the writer constants contradict. After changing a persisted line, run it with -`-write`, append the change record, and update `status.json` when the format version -moves. When a release ships a new format version, set `released` to it. +type's normalized JSON shape and its SHA-256 digest. A member the runtime builds +itself as raw JSON, the record's canonical `disposition`, is fingerprinted through +the type declaring its members (`result.CanonicalDispositionShape`), and a string +member may declare a grammar (the stamp `token` is standard padded `base64`). +Exemplar lines produced by the writers' own encoders are held to those declared +shapes and grammars: a member the builder writes but the shape does not declare, +a declared member no exemplar writes, or a token in another encoding fails. +The fingerprint covers shape and those declared grammars, not other value +encodings or spellings, such as a timestamp's. + +`changes.json` is written by hand and only appended to: every digest a line type +has had, with the decision that admitted it (`baseline`, `compatible-addition` at +the same format version, or `new-format-version` at the next integer with its +`readingRule`) and a reason. `status.json` is kept apart: per trail file, the +format the code `writes` (checked against the writer constant), the last accepted +compatibility `baseline`, and the last format `released`. A format that shipped in +any release, a pre-release included, carries compatibility obligations, so +`baseline <= released <= writes`. Its `releasedChanges` pins the count and digest +of the change records already released; they must stay unchanged and in order, so +a released record cannot be rewritten in place. + +`go run ./tools/audit-formats` (the `audit-formats` check) regenerates the catalog +in memory and fails on a stale catalog, on a digest no change record names, on a +rewritten released record, on a status the writer constants contradict, and on an +exemplar that does not conform. After changing a persisted line, run it with +`-write`, append the change record, and update `status.json` when the format +version moves. When a release ships, set `released` to the format it carries and +move `releasedChanges` to cover every record then present (the count and the +digest the check reports for that prefix). ## Validate locally diff --git a/internal/audit/formats.go b/internal/audit/formats.go index fa711b2..a6d82d1 100644 --- a/internal/audit/formats.go +++ b/internal/audit/formats.go @@ -1,17 +1,31 @@ package audit -import "reflect" +import ( + "reflect" + + "github.com/Judgment-Pack/judgment-pack-runtime/internal/result" +) // LineFormat is one persisted line type of a project's audit trail: the file // it is written to, its name, the format version the writer stamps on that // file's lines, and the Go type it is encoded from. The catalog in -// internal/auditformat fingerprints each one (#327), so a change to a member +// internal/auditformat fingerprints each one (#327), so a change to the shape // of a persisted line cannot pass without a committed format decision. +// +// Embedded names a json.RawMessage member the runtime builds itself, by its +// JSON name, with the type that declares its shape; the fingerprint uses that +// type in place of an opaque JSON value. Exemplars are lines produced by the +// writer's own encoder, which the check holds to the declared shape and to +// any `format` grammar a string member declares. The fingerprint covers shape +// and those declared grammars, not every value spelling (a timestamp's, for +// one). type LineFormat struct { - File string - Line string - Version string - Type reflect.Type + File string + Line string + Version string + Type reflect.Type + Embedded map[string]reflect.Type + Exemplars func() ([][]byte, error) } // LineFormats lists every line type this runtime persists, in file order. A @@ -19,10 +33,48 @@ type LineFormat struct { func LineFormats() []LineFormat { return []LineFormat{ // One shape carries both decision kinds, evaluation and graph-composite. - {FileName, "record", RecordVersion, reflect.TypeOf(Record{})}, - {FileName, KindDiscontinuity, RecordVersion, reflect.TypeOf(discontinuityRecord{})}, - {SidecarName, KindRecordSignature, SidecarVersion, reflect.TypeOf(recordSignatureLine{})}, - {SidecarName, KindKeyRotation, SidecarVersion, reflect.TypeOf(keyRotationLine{})}, - {StampsName, "stamp", StampVersion, reflect.TypeOf(stampLine{})}, + {FileName, "record", RecordVersion, reflect.TypeOf(Record{}), + map[string]reflect.Type{"disposition": reflect.TypeOf(result.CanonicalDispositionShape{})}, recordExemplars}, + {FileName, KindDiscontinuity, RecordVersion, reflect.TypeOf(discontinuityRecord{}), nil, nil}, + {SidecarName, KindRecordSignature, SidecarVersion, reflect.TypeOf(recordSignatureLine{}), nil, nil}, + {SidecarName, KindKeyRotation, SidecarVersion, reflect.TypeOf(keyRotationLine{}), nil, nil}, + {StampsName, "stamp", StampVersion, reflect.TypeOf(stampLine{}), nil, stampExemplars}, + } +} + +// recordExemplars are records whose dispositions, built by the canonical +// encoder, together carry every disposition member: an outcome with a value +// under the legacy contract, and a routed requested handoff under Core 0.3. +func recordExemplars() ([][]byte, error) { + dispositions := []result.Disposition{ + {Contract: result.LegacyEvaluatorContract(), Kind: "outcome", OutcomeID: "approve", + Handoff: result.Handoff{State: "none"}, Value: map[string]any{"limit": "10", "final": true}}, + {Contract: result.LegacyEvaluatorContract(), Kind: "unresolved", Reasons: []string{"conflict"}, + Handoff: result.Handoff{State: "requested", TriggeredBy: []string{"conflict"}}}, + {Contract: result.Core030EvaluatorContract(), Kind: "unresolved", Reasons: []string{"conflict"}, + Handoff: result.Handoff{State: "requested", TriggeredBy: []string{"conflict"}, + Routes: []result.HandoffTarget{{Kind: "queue", Name: "review"}}}}, } + var lines [][]byte + for _, disposition := range dispositions { + canonical, err := disposition.Canonical() + if err != nil { + return nil, err + } + line, err := encodeLines([]Record{{RecordVersion: RecordVersion, Run: "exemplar", At: "exemplar", + Kind: KindEvaluation, Surface: "exemplar", Tool: Tool{Name: "exemplar", Version: "exemplar"}, + EvaluatorSpecVersion: disposition.Contract.Version(), Disposition: canonical}}) + if err != nil { + return nil, err + } + lines = append(lines, line) + } + return lines, nil +} + +// stampExemplars is one stamp line from the stamp encoder, with token bytes +// whose standard base64 spelling uses both "+" and "/" and padding. +func stampExemplars() ([][]byte, error) { + checkpoint := result.AuditCheckpoint{CheckpointVersion: "1", RecordDigest: Digest(nil), Sequence: 1, Trail: "exemplar"} + return [][]byte{encodeStampLine(checkpoint, []byte{0xfb, 0xff})}, nil } diff --git a/internal/audit/stamp.go b/internal/audit/stamp.go index 27e8a1b..a338e13 100644 --- a/internal/audit/stamp.go +++ b/internal/audit/stamp.go @@ -89,11 +89,12 @@ func CheckpointDigest(checkpoint result.AuditCheckpoint) []byte { return sum[:] } -// stampLine is a stamps line, its members in code-point order. +// stampLine is a stamps line, its members in code-point order. The token is +// standard padded base64; the format tag declares it to the audit-formats check. type stampLine struct { Checkpoint result.AuditCheckpoint `json:"checkpoint"` StampVersion string `json:"stampVersion"` - Token string `json:"token"` + Token string `json:"token" format:"base64"` } // encodeStampLine is a stamps line, newline included. diff --git a/internal/auditformat/auditformat.go b/internal/auditformat/auditformat.go index be6bb70..716f0c3 100644 --- a/internal/auditformat/auditformat.go +++ b/internal/auditformat/auditformat.go @@ -8,16 +8,21 @@ // - status.json, written by hand and kept apart: per trail file, the format // the code writes, the last accepted compatibility baseline and the last // format released. A format that shipped in any release, a pre-release -// included, carries compatibility obligations. +// included, carries compatibility obligations. It also pins the released +// prefix of change records, which stays unchanged and in order. // -// Check regenerates the catalog from the Go types and refuses a catalog that -// differs, a digest no change record names, and a status the writer constants -// contradict. +// CheckFiles regenerates the catalog from the Go types and refuses a catalog +// that differs, a digest no change record names, a rewritten released record, +// a status the writer constants contradict, and an exemplar line from the +// writers' own encoders that does not conform to its declared shape. The +// fingerprint covers shape and declared grammars, not other value encodings or +// spellings, such as a timestamp's. package auditformat import ( "bytes" "crypto/sha256" + "encoding/base64" "encoding/hex" "encoding/json" "fmt" @@ -91,23 +96,28 @@ type FileStatus struct { // Status is the format-status file. type Status struct { - StatusVersion int `json:"statusVersion"` - Files []FileStatus `json:"files"` + StatusVersion int `json:"statusVersion"` + ReleasedChanges ReleasedChanges `json:"releasedChanges"` + Files []FileStatus `json:"files"` } // Generate fingerprints every line type the audit package persists. func Generate() (Catalog, map[string]string) { + return generate(audit.LineFormats()) +} + +func generate(formats []audit.LineFormat) (Catalog, map[string]string) { catalog := Catalog{CatalogVersion: 1, Normalization: Normalization} writers := map[string]string{} - for _, format := range audit.LineFormats() { - catalog.Lines = append(catalog.Lines, entry(format.File, format.Line, format.Type)) + for _, format := range formats { + catalog.Lines = append(catalog.Lines, entry(format.File, format.Line, format.Type, format.Embedded)) writers[format.File] = format.Version } return catalog, writers } -func entry(file, line string, t reflect.Type) Entry { - normalized := Normalize(t) +func entry(file, line string, t reflect.Type, embedded map[string]reflect.Type) Entry { + normalized := NormalizeLine(t, embedded) sum := sha256.Sum256([]byte(normalized)) return Entry{file, line, normalized, "sha256:" + hex.EncodeToString(sum[:])} } @@ -132,12 +142,20 @@ var marshaler = reflect.TypeOf((*json.Marshaler)(nil)).Elem() // member types reduced to JSON kinds. Go type and field names do not appear, // so a rename that keeps the encoded shape keeps the digest. A type with its // own MarshalJSON is marked "custom" and normalized by its fields, so a change -// to its fields is caught; a change to the method's code alone is not. +// to its fields is caught; a change to the method's code alone is not. A +// string member's `format` tag is written after its kind, as "string(base64)". func Normalize(t reflect.Type) string { - return normalize(t, map[reflect.Type]bool{}) + return NormalizeLine(t, nil) +} + +// NormalizeLine is Normalize with the line's embedded members (a top-level +// member name mapped to the type declaring its shape) in place of their own +// types, so a json.RawMessage the runtime builds is not an opaque "json". +func NormalizeLine(t reflect.Type, embedded map[string]reflect.Type) string { + return normalize(t, map[reflect.Type]bool{}, embedded) } -func normalize(t reflect.Type, open map[reflect.Type]bool) string { +func normalize(t reflect.Type, open map[reflect.Type]bool, embedded map[string]reflect.Type) string { if t == rawMessage { return "json" } @@ -158,16 +176,16 @@ func normalize(t reflect.Type, open map[reflect.Type]bool) string { case reflect.Interface: return "any" case reflect.Pointer: - return normalize(t.Elem(), open) + "|null" + return normalize(t.Elem(), open, nil) + "|null" case reflect.Slice: if t.Elem().Kind() == reflect.Uint8 { return prefix + "base64|null" } - return prefix + "[" + normalize(t.Elem(), open) + "]|null" + return prefix + "[" + normalize(t.Elem(), open, nil) + "]|null" case reflect.Array: - return prefix + "[" + normalize(t.Elem(), open) + "]" + return prefix + "[" + normalize(t.Elem(), open, nil) + "]" case reflect.Map: - return prefix + "{" + normalize(t.Key(), open) + ":" + normalize(t.Elem(), open) + "}|null" + return prefix + "{" + normalize(t.Key(), open, nil) + ":" + normalize(t.Elem(), open, nil) + "}|null" case reflect.Struct: if open[t] { return "recursive" @@ -186,7 +204,15 @@ func normalize(t reflect.Type, open map[reflect.Type]bool) string { if !plainName.MatchString(field.name) { name = strconv.Quote(name) } - members = append(members, name+":"+normalize(field.typ, open)) + typ := field.typ + if declared, ok := embedded[field.name]; ok { + typ = declared + } + text := normalize(typ, open, nil) + if field.format != "" { + text += "(" + field.format + ")" + } + members = append(members, name+":"+text) } return prefix + "{" + strings.Join(members, ",") + "}" } @@ -197,6 +223,7 @@ type field struct { name string omitempty bool quoted bool + format string typ reflect.Type } @@ -236,7 +263,7 @@ func fields(t reflect.Type) []field { } return false } - out = append(out, field{name, option("omitempty"), option("string"), f.Type}) + out = append(out, field{name, option("omitempty"), option("string"), f.Tag.Get("format"), f.Type}) } return out } @@ -268,6 +295,11 @@ func Check(catalogRaw, changesRaw, statusRaw []byte, generated Catalog, writers fail("%s: %v", StatusFile, err) return problems } + // Released history is append-only: the pinned prefix of change records is + // unchanged and in order. + if pin := status.ReleasedChanges; pin.Count < 0 || pin.Count > len(changes.Changes) || PrefixDigest(changes.Changes, pin.Count) != pin.SHA256 { + fail("%s: the %d released change records pinned by %s are not unchanged and in order; append new records after them instead (all %d records present digest to %s)", ChangesFile, pin.Count, StatusFile, len(changes.Changes), PrefixDigest(changes.Changes, len(changes.Changes))) + } known := map[string]Entry{} for _, line := range generated.Lines { known[line.File+"\x00"+line.Line] = line @@ -358,6 +390,17 @@ func Check(catalogRaw, changesRaw, statusRaw []byte, generated Catalog, writers return problems } +// CheckFiles is the whole audit-formats check over the committed files: Check +// against the generated catalog and the writer constants, then Exemplars. +func CheckFiles(catalogRaw, changesRaw, statusRaw []byte) []error { + return checkFiles(catalogRaw, changesRaw, statusRaw, audit.LineFormats()) +} + +func checkFiles(catalogRaw, changesRaw, statusRaw []byte, formats []audit.LineFormat) []error { + generated, writers := generate(formats) + return append(Check(catalogRaw, changesRaw, statusRaw, generated, writers), exemplars(formats)...) +} + func formatVersion(text string) (int, error) { value, err := strconv.Atoi(text) if err != nil || value < 1 || strconv.Itoa(value) != text { @@ -377,3 +420,205 @@ func strictDecode(raw []byte, into any) error { } return nil } + +// Exemplars holds every exemplar line the audit package's own encoders produce +// to its line type's declared shape and grammars, and requires each member an +// embedded shape declares to appear in at least one exemplar. A builder that +// writes a member its shape does not declare, or stops writing one it does, +// fails here even when no Go type changed. +func Exemplars() []error { + return exemplars(audit.LineFormats()) +} + +func exemplars(formats []audit.LineFormat) []error { + var problems []error + for _, format := range formats { + if format.Exemplars == nil { + continue + } + lines, err := format.Exemplars() + if err != nil { + problems = append(problems, fmt.Errorf("%s %s: exemplar: %v", format.File, format.Line, err)) + continue + } + if err := Conform(lines, format.Type, format.Embedded); err != nil { + problems = append(problems, fmt.Errorf("%s %s: %v", format.File, format.Line, err)) + } + } + return problems +} + +// Conform checks lines against t (with its embedded shapes): each line is one +// JSON object whose members are all declared, with every non-omitempty member +// present and every value of its declared kind and `format` grammar. Within +// embedded shapes, every declared member must appear in at least one line. +func Conform(lines [][]byte, t reflect.Type, embedded map[string]reflect.Type) error { + seen := map[string]bool{} + for i, line := range lines { + decoder := json.NewDecoder(bytes.NewReader(line)) + decoder.UseNumber() + var value any + if err := decoder.Decode(&value); err != nil { + return fmt.Errorf("exemplar %d: %v", i+1, err) + } + if err := conform(value, t, "", "", false, embedded, seen); err != nil { + return fmt.Errorf("exemplar %d: %v", i+1, err) + } + } + for name, declared := range embedded { + for _, path := range declaredPaths(declared, name) { + if !seen[path] { + return fmt.Errorf("no exemplar writes the declared member %s", path) + } + } + } + return nil +} + +// conform checks one value at member path where. Below an embedded shape +// (tracked), the path is recorded in seen for the coverage rule. +func conform(value any, t reflect.Type, format, where string, tracked bool, embedded map[string]reflect.Type, seen map[string]bool) error { + if tracked { + seen[where] = true + } + if t == rawMessage || t.Kind() == reflect.Interface { + return nil + } + if t.Kind() == reflect.Pointer || t.Kind() == reflect.Slice || t.Kind() == reflect.Map { + if value == nil { + return nil + } + } + switch t.Kind() { + case reflect.Pointer: + return conform(value, t.Elem(), format, where, tracked, nil, seen) + case reflect.Bool: + if _, ok := value.(bool); ok { + return nil + } + case reflect.String: + text, ok := value.(string) + if ok && format == "base64" { + decoded, err := base64.StdEncoding.Strict().DecodeString(text) + if err != nil || base64.StdEncoding.EncodeToString(decoded) != text { + return fmt.Errorf("member %s is not standard padded base64", where) + } + } + if ok { + return nil + } + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + if number, ok := value.(json.Number); ok { + if _, err := strconv.ParseInt(number.String(), 10, 64); err == nil { + return nil + } + } + case reflect.Float32, reflect.Float64: + if _, ok := value.(json.Number); ok { + return nil + } + case reflect.Slice, reflect.Array: + if t.Elem().Kind() == reflect.Uint8 { + return conform(value, reflect.TypeOf(""), "base64", where, tracked, nil, seen) + } + items, ok := value.([]any) + if !ok { + break + } + for _, item := range items { + if err := conform(item, t.Elem(), "", where, tracked, nil, seen); err != nil { + return err + } + } + return nil + case reflect.Map: + members, ok := value.(map[string]any) + if !ok { + break + } + for _, member := range members { + if err := conform(member, t.Elem(), "", where, tracked, nil, seen); err != nil { + return err + } + } + return nil + case reflect.Struct: + members, ok := value.(map[string]any) + if !ok { + break + } + declared := map[string]field{} + for _, f := range fields(t) { + declared[f.name] = f + if _, present := members[f.name]; !present && !f.omitempty { + return fmt.Errorf("required member %s is absent", join(where, f.name)) + } + } + for name, member := range members { + f, ok := declared[name] + if !ok { + return fmt.Errorf("member %s is not declared by the line's shape", join(where, name)) + } + typ, next, nextTracked := f.typ, join(where, name), tracked + if shape, ok := embedded[name]; ok { + typ, nextTracked = shape, true + } + if f.quoted { + if _, ok := member.(string); !ok { + return fmt.Errorf("member %s is not a quoted value", join(where, name)) + } + continue + } + if err := conform(member, typ, f.format, next, nextTracked, nil, seen); err != nil { + return err + } + } + return nil + } + return fmt.Errorf("member %s is not of its declared kind %s", where, t.Kind()) +} + +func join(path, name string) string { + if path == "" { + return name + } + return path + "." + name +} + +// declaredPaths is every member path a struct shape declares, below prefix. +func declaredPaths(t reflect.Type, prefix string) []string { + for t.Kind() == reflect.Pointer || t.Kind() == reflect.Slice || t.Kind() == reflect.Array { + t = t.Elem() + } + if t.Kind() != reflect.Struct || t == rawMessage { + return nil + } + var paths []string + for _, f := range fields(t) { + path := join(prefix, f.name) + paths = append(paths, path) + paths = append(paths, declaredPaths(f.typ, path)...) + } + return paths +} + +// ReleasedChanges pins the change records a release has shipped: their count +// and the digest of their content, in order. Records are appended after the +// pinned prefix; the prefix itself is never rewritten. +type ReleasedChanges struct { + Count int `json:"count"` + SHA256 string `json:"sha256"` +} + +// PrefixDigest is the digest of the first count change records' content: each +// record's compact JSON encoding, one per line, so a whitespace edit to the +// file does not move it and a content edit or reordering does. +func PrefixDigest(changes []Change, count int) string { + hash := sha256.New() + for _, change := range changes[:count] { + encoded, _ := json.Marshal(change) // strings only + hash.Write(append(encoded, '\n')) + } + return "sha256:" + hex.EncodeToString(hash.Sum(nil)) +} diff --git a/internal/auditformat/auditformat_test.go b/internal/auditformat/auditformat_test.go index 13ad246..4718a39 100644 --- a/internal/auditformat/auditformat_test.go +++ b/internal/auditformat/auditformat_test.go @@ -8,12 +8,14 @@ import ( "reflect" "strings" "testing" + + "github.com/Judgment-Pack/judgment-pack-runtime/internal/audit" ) // The committed catalog, change records and status agree with the code. func TestCommittedFormatsAgree(t *testing.T) { - generated, writers := Generate() - if problems := Check(read(t, CatalogFile), read(t, ChangesFile), read(t, StatusFile), generated, writers); len(problems) != 0 { + generated, _ := Generate() + if problems := CheckFiles(read(t, CatalogFile), read(t, ChangesFile), read(t, StatusFile)); len(problems) != 0 { t.Fatal(problems) } if len(generated.Lines) != 5 { @@ -62,7 +64,7 @@ type handwrittenShape struct { // outside this code from the text. func TestNormalizeHandwrittenShape(t *testing.T) { const want = `{a:string,b?:integer,c~:integer,"d-e":boolean,f:{g:string}|null,h:[string]|null,i:{string:number}|null,j:json,k:base64|null,l:customstring,promoted:string,m:any}` - got := entry("f", "l", reflect.TypeOf(handwrittenShape{})) + got := entry("f", "l", reflect.TypeOf(handwrittenShape{}), nil) if got.Type != want { t.Fatalf("normalized:\n got %s\nwant %s", got.Type, want) } @@ -106,13 +108,13 @@ type syntheticV2 struct { } func TestCheckDecisionsAndStatus(t *testing.T) { - v1 := entry("trail.jsonl", "line", reflect.TypeOf(syntheticV1{})) - v2 := entry("trail.jsonl", "line", reflect.TypeOf(syntheticV2{})) + v1 := entry("trail.jsonl", "line", reflect.TypeOf(syntheticV1{}), nil) + v2 := entry("trail.jsonl", "line", reflect.TypeOf(syntheticV2{}), nil) baseline := `{"file":"trail.jsonl","line":"line","sha256":"` + v1.SHA256 + `","formatVersion":"1","decision":"baseline","reason":"first"}` addition := `{"file":"trail.jsonl","line":"line","sha256":"` + v2.SHA256 + `","formatVersion":"1","decision":"compatible-addition","reason":"optional member"}` newVersion := `{"file":"trail.jsonl","line":"line","sha256":"` + v2.SHA256 + `","formatVersion":"2","decision":"new-format-version","readingRule":"version 1 lines lack extra","reason":"new member"}` status := func(writes, baseline, released string) string { - return `{"statusVersion":1,"files":[{"file":"trail.jsonl","writes":"` + writes + `","baseline":"` + baseline + `","released":"` + released + `"}]}` + return `{"statusVersion":1,"releasedChanges":{"count":0,"sha256":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},"files":[{"file":"trail.jsonl","writes":"` + writes + `","baseline":"` + baseline + `","released":"` + released + `"}]}` } for _, tc := range []struct { name string @@ -142,7 +144,7 @@ func TestCheckDecisionsAndStatus(t *testing.T) { {"baseline after released", v1, baseline, status("1", "2", "1"), "1", false}, {"released after writes", v2, baseline + "," + newVersion, status("2", "1", "3"), "2", false}, {"version not canonical", v1, baseline, status("01", "1", "1"), "1", false}, - {"no status", v1, baseline, `{"statusVersion":1,"files":[]}`, "1", false}, + {"no status", v1, baseline, `{"statusVersion":1,"releasedChanges":{"count":0,"sha256":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},"files":[]}`, "1", false}, } { t.Run(tc.name, func(t *testing.T) { generated := Catalog{CatalogVersion: 1, Normalization: Normalization, Lines: []Entry{tc.entry}} @@ -153,3 +155,102 @@ func TestCheckDecisionsAndStatus(t *testing.T) { }) } } + +// Released change records are append-only: rewriting one in place, reordering +// or removing them fails; appending after them passes. +func TestReleasedChangeRecordsAreAppendOnly(t *testing.T) { + generated, writers := Generate() + var changes Changes + if err := json.Unmarshal(read(t, ChangesFile), &changes); err != nil { + t.Fatal(err) + } + encode := func(c Changes) []byte { raw, _ := json.Marshal(c); return raw } + // A member added to the record line, admitted by an appended record. + changed, _ := Generate() + changed.Lines[0].Type = strings.TrimSuffix(changed.Lines[0].Type, "}") + ",added?:string}" + sum := sha256.Sum256([]byte(changed.Lines[0].Type)) + changed.Lines[0].SHA256 = "sha256:" + hex.EncodeToString(sum[:]) + appended := Changes{append(append([]Change(nil), changes.Changes...), Change{File: changed.Lines[0].File, Line: changed.Lines[0].Line, SHA256: changed.Lines[0].SHA256, FormatVersion: "1", Decision: DecisionCompatibleAddition, Reason: "optional member"})} + if problems := Check(Encode(changed), encode(appended), read(t, StatusFile), changed, writers); len(problems) != 0 { + t.Fatal("append refused:", problems) + } + // The same change admitted by rewriting the released baseline in place. + rewritten := Changes{append([]Change(nil), changes.Changes...)} + rewritten.Changes[0].SHA256 = changed.Lines[0].SHA256 + problems := Check(Encode(changed), encode(rewritten), read(t, StatusFile), changed, writers) + if len(problems) != 1 || !strings.Contains(problems[0].Error(), "released change records") { + t.Fatal("in-place rewrite of a released record:", problems) + } + swapped := Changes{append([]Change(nil), changes.Changes...)} + swapped.Changes[0], swapped.Changes[1] = swapped.Changes[1], swapped.Changes[0] + if problems := Check(Encode(generated), encode(swapped), read(t, StatusFile), generated, writers); len(problems) != 1 { + t.Fatal("reordered released records:", problems) + } + if problems := Check(Encode(generated), encode(Changes{changes.Changes[:4]}), read(t, StatusFile), generated, writers); len(problems) == 0 { + t.Fatal("removed released record accepted") + } +} + +type exemplarDisposition struct { + Kind string `json:"kind"` + Handoff struct { + State string `json:"state"` + Routes []string `json:"routes,omitempty"` + } `json:"handoff"` +} +type exemplarLine struct { + Version string `json:"version"` + Token string `json:"token" format:"base64"` + Disposition json.RawMessage `json:"disposition"` + Inputs json.RawMessage `json:"inputs"` +} + +// Lines a builder writes are held to the declared embedded shape and grammars. +func TestConformExemplars(t *testing.T) { + embedded := map[string]reflect.Type{"disposition": reflect.TypeOf(exemplarDisposition{})} + line := func(token, disposition string) []byte { + return []byte(`{"version":"1","token":"` + token + `","disposition":` + disposition + `,"inputs":{"anything":[1]}}`) + } + full := `{"handoff":{"routes":["q"],"state":"requested"},"kind":"unresolved"}` + plain := `{"handoff":{"state":"none"},"kind":"outcome"}` + for _, tc := range []struct { + name string + lines [][]byte + ok bool + }{ + {"valid", [][]byte{line("+/8=", full), line("+/8=", plain)}, true}, + {"undeclared disposition member", [][]byte{line("+/8=", strings.Replace(full, `"kind"`, `"extra":1,"kind"`, 1))}, false}, + {"undeclared nested member", [][]byte{line("+/8=", strings.Replace(full, `"state"`, `"extra":1,"state"`, 1))}, false}, + {"declared member never written", [][]byte{line("+/8=", plain)}, false}, + {"required member absent", [][]byte{line("+/8=", full), line("+/8=", `{"handoff":{"state":"none"}}`)}, false}, + {"wrong kind", [][]byte{line("+/8=", strings.Replace(full, `"unresolved"`, `1`, 1))}, false}, + {"raw URL base64 token", [][]byte{line("-_8", full)}, false}, + {"unpadded token", [][]byte{line("+/8", full)}, false}, + {"undeclared top-level member", [][]byte{[]byte(`{"version":"1","token":"+/8=","disposition":` + full + `,"inputs":null,"extra":true}`)}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + err := Conform(tc.lines, reflect.TypeOf(exemplarLine{}), embedded) + if (err == nil) != tc.ok { + t.Fatal(err) + } + }) + } + // The embedded shape is part of the fingerprint, not an opaque "json". + if got := NormalizeLine(reflect.TypeOf(exemplarLine{}), embedded); got != `{version:string,token:string(base64),disposition:{kind:string,handoff:{state:string,routes?:[string]|null}},inputs:json}` { + t.Fatal(got) + } +} + +// The whole check runs the exemplars: a line format whose encoder writes an +// undeclared member fails it even when its catalog, records and status agree. +func TestCheckFilesRunsExemplars(t *testing.T) { + formats := []audit.LineFormat{{File: "trail.jsonl", Line: "line", Version: "1", Type: reflect.TypeOf(syntheticV1{}), + Exemplars: func() ([][]byte, error) { return [][]byte{[]byte(`{"kind":"k","extra":1}`)}, nil }}} + generated, _ := generate(formats) + changes := `{"changes":[{"file":"trail.jsonl","line":"line","sha256":"` + generated.Lines[0].SHA256 + `","formatVersion":"1","decision":"baseline","reason":"first"}]}` + status := `{"statusVersion":1,"releasedChanges":{"count":0,"sha256":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},"files":[{"file":"trail.jsonl","writes":"1","baseline":"1","released":"1"}]}` + problems := checkFiles(Encode(generated), []byte(changes), []byte(status), formats) + if len(problems) != 1 || !strings.Contains(problems[0].Error(), "extra") { + t.Fatal(problems) + } +} diff --git a/internal/auditformat/catalog.json b/internal/auditformat/catalog.json index de2c1ac..c89bc25 100644 --- a/internal/auditformat/catalog.json +++ b/internal/auditformat/catalog.json @@ -5,8 +5,8 @@ { "file": "evaluations.jsonl", "line": "record", - "type": "{recordVersion:string,trail?:string,sequence?:integer,previous?:string,run:string,at:string,kind:string,surface:string,tool:{name:string,version:string,digest?:string},evaluatorSpecVersion:string,pack?:{id:string,version:string,specVersion:string,digest:string}|null,graph?:{id:string,version:string,formatVersion?:string,digest?:string,node?:string,resultNode?:string}|null,inputs?:{facts:json,evidence:json,evidenceSupplied:boolean}|null,draftPrototype?:{rfc:string,status:string,operators:[string]|null,outcomes?:[string]|null,packValidUnderSpecVersion:boolean,note:string}|null,artifact?:{specVersion:string,bundleDigest:string,provenance:string}|null,reviewed?:boolean|null,reviewedSet?:{lockDigest:string,lockVersion:string,configDigest:string}|null,cites?:[{sessionId:string,callIndex:integer,signature:string}]|null,binding?:{kind:string,digest:string}|null,unknownCauses?:[{path?:string|null,evidenceRequirement?:string,within?:string|null,cause:string,factType?:string}]|null,typeMismatches?:[{path:string,within?:string|null,operator:string,factType:string,operandTypes:[string]|null}]|null,contributors?:[{kind:string,id:string}]|null,disposition:json}", - "sha256": "sha256:3748655634fc2b135437c83631d6739db971516d3fd44d131f988183dd69549d" + "type": "{recordVersion:string,trail?:string,sequence?:integer,previous?:string,run:string,at:string,kind:string,surface:string,tool:{name:string,version:string,digest?:string},evaluatorSpecVersion:string,pack?:{id:string,version:string,specVersion:string,digest:string}|null,graph?:{id:string,version:string,formatVersion?:string,digest?:string,node?:string,resultNode?:string}|null,inputs?:{facts:json,evidence:json,evidenceSupplied:boolean}|null,draftPrototype?:{rfc:string,status:string,operators:[string]|null,outcomes?:[string]|null,packValidUnderSpecVersion:boolean,note:string}|null,artifact?:{specVersion:string,bundleDigest:string,provenance:string}|null,reviewed?:boolean|null,reviewedSet?:{lockDigest:string,lockVersion:string,configDigest:string}|null,cites?:[{sessionId:string,callIndex:integer,signature:string}]|null,binding?:{kind:string,digest:string}|null,unknownCauses?:[{path?:string|null,evidenceRequirement?:string,within?:string|null,cause:string,factType?:string}]|null,typeMismatches?:[{path:string,within?:string|null,operator:string,factType:string,operandTypes:[string]|null}]|null,contributors?:[{kind:string,id:string}]|null,disposition:{handoff:{routes?:[{kind:string,name:string}]|null,state:string,triggeredBy?:[string]|null},kind:string,outcomeId?:string,reasons:[string]|null,value?:{string:any}|null}}", + "sha256": "sha256:16ad4e02ceb65034370837e5aebbf674ee4764aca6e1aa443bc03ad44a4f4d9b" }, { "file": "evaluations.jsonl", @@ -29,8 +29,8 @@ { "file": "stamps.jsonl", "line": "stamp", - "type": "{checkpoint:{checkpointVersion:string,recordDigest:string,sequence:integer,trail:string},stampVersion:string,token:string}", - "sha256": "sha256:316f48ea2f46150a2e28f9583037700125d77ba606ea7c6b2e7f4f097800f5eb" + "type": "{checkpoint:{checkpointVersion:string,recordDigest:string,sequence:integer,trail:string},stampVersion:string,token:string(base64)}", + "sha256": "sha256:227d302ec48dc932b75b29eebebea55fac0876efe45044ae31ec7922e10a57b7" } ] } diff --git a/internal/auditformat/changes.json b/internal/auditformat/changes.json index c17ee6e..e36e7fd 100644 --- a/internal/auditformat/changes.json +++ b/internal/auditformat/changes.json @@ -3,10 +3,10 @@ { "file": "evaluations.jsonl", "line": "record", - "sha256": "sha256:3748655634fc2b135437c83631d6739db971516d3fd44d131f988183dd69549d", + "sha256": "sha256:16ad4e02ceb65034370837e5aebbf674ee4764aca6e1aa443bc03ad44a4f4d9b", "formatVersion": "1", "decision": "baseline", - "reason": "The evaluation and graph-composite record shape as of v0.39.0, written under recordVersion 1." + "reason": "The evaluation and graph-composite record shape as of v0.39.0, written under recordVersion 1, with the canonical disposition members Core 0.2.0-draft, Core 0.3.0 and draft RFC 0016 define." }, { "file": "evaluations.jsonl", @@ -35,10 +35,10 @@ { "file": "stamps.jsonl", "line": "stamp", - "sha256": "sha256:316f48ea2f46150a2e28f9583037700125d77ba606ea7c6b2e7f4f097800f5eb", + "sha256": "sha256:227d302ec48dc932b75b29eebebea55fac0876efe45044ae31ec7922e10a57b7", "formatVersion": "1", "decision": "baseline", - "reason": "The time-stamp line, released since v0.26.0." + "reason": "The time-stamp line, released since v0.26.0; its token is standard padded base64." } ] } diff --git a/internal/auditformat/status.json b/internal/auditformat/status.json index 0064944..02849f1 100644 --- a/internal/auditformat/status.json +++ b/internal/auditformat/status.json @@ -1,5 +1,6 @@ { "statusVersion": 1, + "releasedChanges": {"count": 5, "sha256": "sha256:45b21afd9b94dd20fbac756816159f1cabf54522abc53eba6adb6458a77eb63b"}, "files": [ {"file": "evaluations.jsonl", "writes": "1", "baseline": "1", "released": "1"}, {"file": "signatures.jsonl", "writes": "1", "baseline": "1", "released": "1"}, diff --git a/internal/result/disposition_shape.go b/internal/result/disposition_shape.go new file mode 100644 index 0000000..e234ff6 --- /dev/null +++ b/internal/result/disposition_shape.go @@ -0,0 +1,21 @@ +package result + +// CanonicalDispositionShape declares the members Disposition.CanonicalWith +// writes. CanonicalWith builds the disposition from maps so that RFC 8785 +// orders it, so the persisted shape has no Go type of its own; this is that +// type. It is never encoded. The audit-formats check (#327) fingerprints it as +// the disposition of every evaluations.jsonl record and holds the bytes +// CanonicalWith actually produces to it: a member the builder writes that is +// not declared here, or one declared here that no exemplar shows, fails the +// check. Members are listed in the canonical (code-point) order. +type CanonicalDispositionShape struct { + Handoff struct { + Routes []HandoffTarget `json:"routes,omitempty"` + State string `json:"state"` + TriggeredBy []string `json:"triggeredBy,omitempty"` + } `json:"handoff"` + Kind string `json:"kind"` + OutcomeID string `json:"outcomeId,omitempty"` + Reasons []string `json:"reasons"` + Value map[string]any `json:"value,omitempty"` +} diff --git a/tools/audit-formats/main.go b/tools/audit-formats/main.go index 65e69f2..3222f68 100644 --- a/tools/audit-formats/main.go +++ b/tools/audit-formats/main.go @@ -1,5 +1,6 @@ // Command audit-formats regenerates the audit trail's line-format catalog and -// checks it, the committed change records and the format status (#327). Run it +// checks it, the committed change records, the format status and the writers' +// exemplar lines (#327). Run it // from the repository root. -write rewrites the generated catalog only; change // records and the status are written by hand. package main @@ -22,7 +23,7 @@ func main() { } func run(dir string, write bool) int { - generated, writers := auditformat.Generate() + generated, _ := auditformat.Generate() catalogPath := filepath.Join(dir, auditformat.CatalogFile) if write { current, _ := os.ReadFile(catalogPath) @@ -41,7 +42,7 @@ func run(dir string, write bool) int { } return raw } - problems := auditformat.Check(read(auditformat.CatalogFile), read(auditformat.ChangesFile), read(auditformat.StatusFile), generated, writers) + problems := auditformat.CheckFiles(read(auditformat.CatalogFile), read(auditformat.ChangesFile), read(auditformat.StatusFile)) for _, problem := range problems { fmt.Fprintln(os.Stderr, "audit-formats:", problem) }