Add explicit retained-column Draft document validation

docs/dialect-retained-column-draft.md

@@ -0,0 +1,77 @@
+# Experimental retained-column Draft reader
+
+Runtime has an explicit experimental entry for RFC 0029 at Spec revision
+`ade3914b0b81ef299e5b9a8d6147f37ff3f0976c`. This is a Draft experiment, with no
+normative adoption, preparation verification, source authenticity or execution claim.
+The existing dialect operations retain their default contract and limits.
+
+Select exactly
+`--draft-contract retained-column/1@ade3914b0b81ef299e5b9a8d6147f37ff3f0976c`
+and `--draft-profile base` or `--draft-profile extended` on `dialect contract`,
+`canonicalize`, `digest`, `validate` or `check`. Both flags are required for the
+experiment. Unsupported selection refuses before opening input files. A document
+must declare the selected profile; omission means base. Extended requires explicit
+`"profile":"extended"` in a validated document. No selection is inferred from leaves,
+carrier versions or caller-supplied specification hashes.
+
+`dialect validate <input> --retained-frame` with both selection flags checks a
+standalone retained frame. Otherwise validate checks the entire dialect document,
+including the five converted-copy kinds, evidence availability, ordinary target
+ownership, source ordering and internal references. `dialect check <input> --pack
+<pack>` adds the existing pack validation and bounded declared inventory walk.
+Coverage reports keep `valueTypesChecked:false` and `executionChecked:false`.
+No conversion is executed and no record value is asserted true by these tools.
+
+Canonicalize and digest check encoding only: bounded JSON, canonical numeric domain,
+carrier version and identity spelling. Their success does not validate conversion
+leaves, document structure or profile compatibility. Rules, spellings and source
+pointers remain in canonical bytes and affect the digest. Retained-frame validation
+reports a digest of its exact supplied frame bytes; it does not canonicalize opaque
+retained strings or establish original declaration provenance.
+
+For standalone frame reports, `document` retains the complete exact original JSON
+text and `digest` identifies those bytes. Parse that text to recover every value,
+status, ordered declaration, absent/null distinction and original escape spelling.
+The unpublished `frameSummary` contains only contract version, origin and value,
+declaration and field counts. The full typed inspection remains available to
+internal Go callers; its duplicate raw strings are not repeated on the wire.
+Both MCP report forms contain the same complete report. This keeps the Base frame
+response bounded without reducing accepted input or changing ordinary reports.
+
+The selected contract response reports only role `document-validation` in
+`capabilities.conversionContracts`, with fixed trusted identity:
+
+- Version label: `Draft RFC0029`.
+- Revision: `ade3914b0b81ef299e5b9a8d6147f37ff3f0976c`.
+- Schema: `sha256:996cec17473987eebf1aa553536df0ce5a740971532567f411282ee70036879f`.
+- Corpus: `sha256:e2ac53f56cd5ab779e032d2847b49fa04438ffb9df57d456f75f2ba058dc12d2`.
+
+| Profile | Document UTF-8 bytes | Standalone frame UTF-8 bytes |
+| --- | ---: | ---: |
+| base | 1048576 | 2097152 |
+| extended | 2097152 | 4194304 |
+
+Parser depth, nodes, string budgets and fixed declaration/frame limits remain
+independently bounded. A standalone frame limit grants no combined acquisition,
+producer, transport, preparation or signed record capacity. Init, schema and reference
+remain on the ordinary contract; the experiment advertises no support for those
+operations, generated acquisition, preview or preparation roles.
+
+MCP uses existing `get_dialect_contract`, `dialect_canonicalize`, `dialect_digest`,
+`dialect_validate` and `dialect_check` tools. Supply exact `draftContract` and
+`draftProfile` strings. `retainedFrame:true` is legal only on selected
+`dialect_validate`; the `document` argument then contains exact frame JSON text.
+Unknown/extra/null selectors and duplicate arguments refuse. Tool schemas retain
+conditional legacy and selected document/frame limits.
+
+The unchanged MCP stdio envelope permits a JSON-RPC line below 16777216 bytes,
+including delimiter, field names and JSON string escaping. Decoded artifact limits
+are separate: a valid extended frame of about 3 MiB of literal `<` text can exceed
+that envelope when encoded with HTML escaping. Such a frame is accepted through
+the CLI and refused by the MCP transport before dispatch. Not every 4 MiB frame or
+maximum-size pack/document pair fits every JSON-RPC serialization. No transport
+ceiling is raised by the Draft capability.
+
+Released artifacts, including Runtime 0.39.0-alpha.1, gain no support from this
+experiment. Accepted immutable upstream contracts, reviewed source/artifacts and
+actual Runner/Agent interoperability remain release prerequisites.

internal/cli/dialect.go

@@ -23,6 +23,8 @@ func (a *App) dialectOperation(operation string) *cobra.Command {
 	versionText := "2"
 	formatVersion, artifactID, artifactVersion := "", "", ""
 	packPath := ""
+	draftContract, draftProfile := "", ""
+	retainedFrame := false
 	use := operation
 	args := cobra.NoArgs
 	if operation == "canonicalize" || operation == "digest" || operation == "validate" || operation == "check" {
@@ -35,6 +37,9 @@ func (a *App) dialectOperation(operation string) *cobra.Command {
 		if err := validateFormat(format); err != nil {
 			return a.operational(command, format, result.ExitInvocation, "JPS-INVOCATION-FORMAT", err.Error())
 		}
+		if cmd.Flags().Changed("draft-contract") || cmd.Flags().Changed("draft-profile") || cmd.Flags().Changed("retained-frame") {
+			return a.runRetainedColumnDraft(command, operation, args, format, packPath, draftContract, draftProfile, retainedFrame)
+		}
 		if operation == "check" && (packPath == "" || (packPath == "-" && args[0] == "-")) {
 			return a.operational(command, format, result.ExitInvocation, "JPS-DIALECT-PACK-ARGUMENT", "Check requires --pack; only one operand can read standard input.")
 		}
@@ -145,9 +150,93 @@ func (a *App) dialectOperation(operation string) *cobra.Command {
 	if operation == "check" {
 		c.Flags().StringVar(&packPath, "pack", "", "pack file or - for standard input; required")
 	}
+	if operation == "contract" || operation == "canonicalize" || operation == "digest" || operation == "validate" || operation == "check" {
+		c.Flags().StringVar(&draftContract, "draft-contract", "", "explicit experimental selector: "+dialect.RetainedColumnDraftSelector)
+		c.Flags().StringVar(&draftProfile, "draft-profile", "", "explicit experimental profile: base or extended; requires --draft-contract")
+		if operation == "validate" {
+			c.Flags().BoolVar(&retainedFrame, "retained-frame", false, "validate a standalone retained frame; requires both Draft selection flags")
+		}
+	}
 	return c
 }
 
+// Resolve the trusted experimental selection before opening either artifact.
+func (a *App) runRetainedColumnDraft(command, operation string, args []string, format, packPath, selector, profile string, retainedFrame bool) error {
+	selected, selection, code := dialect.SelectRetainedColumnDraft(command, selector, profile)
+	var output any = selected
+	report := selected.Report
+	if code == result.ExitSuccess {
+		if operation == "check" && (packPath == "" || (packPath == "-" && args[0] == "-")) {
+			return a.operational(command, format, result.ExitInvocation, "JPS-DIALECT-PACK-ARGUMENT", "Check requires --pack; only one operand can read standard input.")
+		}
+		if operation == "contract" {
+			selected, code = dialect.ContractRetainedColumnDraft(command, selection)
+			output, report = selected, selected.Report
+		} else {
+			bound := selected.Limits.DocumentBytes
+			if retainedFrame {
+				bound = selected.Limits.FrameBytes
+			}
+			data, err := a.readInput(args[0], int64(bound), "dialect")
+			if errors.Is(err, fssecure.ErrTooLarge) {
+				data = make([]byte, bound+1)
+			} else if err != nil {
+				return a.operational(command, format, result.ExitIO, "JPS-DIALECT-READ", "The dialect input could not be read as a bounded local regular file or standard input.")
+			}
+			switch operation {
+			case "check":
+				pack, err := a.readInput(packPath, dialect.MaxPackBytes, "pack")
+				if errors.Is(err, fssecure.ErrTooLarge) {
+					pack = make([]byte, dialect.MaxPackBytes+1)
+				} else if err != nil {
+					return a.operational(command, format, result.ExitIO, "JPS-DIALECT-PACK-READ", "The pack input could not be read as a bounded local regular file or standard input.")
+				}
+				selected, code = dialect.CheckRetainedColumnDraft(command, data, pack, a.engine, selection)
+				output, report = selected, selected.Report
+			case "validate":
+				if retainedFrame {
+					frame, frameCode := dialect.ValidateRetainedColumnDraftFrame(command, data, selection)
+					code, output, report = frameCode, frame, frame.Report
+				} else {
+					selected, code = dialect.ValidateRetainedColumnDraft(command, data, selection)
+					output, report = selected, selected.Report
+				}
+			default:
+				selected, code = dialect.EncodeRetainedColumnDraft(command, data, selection)
+				output, report = selected, selected.Report
+			}
+		}
+	}
+	var err error
+	switch {
+	case format == "json":
+		err = a.writeJSON(output)
+	case report.Compatibility != nil:
+		err = renderDialectCompatibility(a.out, report.Compatibility)
+	case code != result.ExitSuccess:
+		for _, diagnostic := range report.Diagnostics {
+			if _, err = fmt.Fprintln(a.errOut, diagnostic.Code+": "+display.Sanitize(diagnostic.Message)); err != nil {
+				break
+			}
+		}
+	case operation == "contract":
+		_, err = fmt.Fprintf(a.out, "Experimental Draft %s · %s\nProfile: %s; role: document-validation\nOperations: contract, canonicalize, digest, validate (document or retained frame), check\nDocument limit: %d UTF-8 bytes; frame limit: %d UTF-8 bytes\nCanonicalize and digest check encoding only. Pack checks cover declared inventory.\n", selected.Contract, selected.Spec.Revision, selected.Profile, selected.Limits.DocumentBytes, selected.Limits.FrameBytes)
+	case operation == "validate":
+		_, err = fmt.Fprintln(a.out, "Experimental Draft checks passed: "+report.ValidationScope+".")
+	case operation == "digest":
+		_, err = fmt.Fprintln(a.out, report.Digest)
+	default:
+		_, err = fmt.Fprint(a.out, report.Document)
+	}
+	if err != nil {
+		return &handledExit{code: result.ExitIO}
+	}
+	if code != result.ExitSuccess {
+		return &handledExit{code: code}
+	}
+	return nil
+}
+
 func renderDialectCompatibility(out io.Writer, c *dialect.Compatibility) error {
 	if _, err := fmt.Fprintf(out, "Declared mapping coverage: %s\nPack: %s · %s\n", c.Status, display.Sanitize(c.Pack.ID), display.Sanitize(c.Pack.Version)); err != nil {
 		return err

internal/cli/dialect_retained_column_test.go

@@ -0,0 +1,147 @@
+package cli
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/Judgment-Pack/judgment-pack-runtime/internal/dialect"
+	"github.com/Judgment-Pack/judgment-pack-runtime/internal/validation"
+)
+
+func retainedEntryDocument(kind, profile string) string {
+	conversion := `{"contractVersion":"1","column":"Column","kind":"` + kind + `"}`
+	if kind == "boolean" {
+		conversion = `{"contractVersion":"1","column":"Column","kind":"boolean","trueValue":"YES","falseValue":"NO"}`
+	}
+	extra := ""
+	if profile == "extended" {
+		extra = `,"profile":"extended"`
+	}
+	return `{"version":3` + extra + `,"case":{"facts":[{"target":"/expense","source":"/retained","conversion":` + conversion + `}],"evidence":[{"requirement":"receipt","source":"/retained","availability":{"contractVersion":"1","column":"Receipt"}},{"requirement":"cost-center","source":"/retained","availability":{"contractVersion":"1","column":"Cost center"}}]}}`
+}
+
+func TestRetainedColumnEntryCLIAndMCP(t *testing.T) {
+	engine, err := validation.NewEngine()
+	if err != nil {
+		t.Fatal(err)
+	}
+	pack, err := os.ReadFile("../artifacts/jps/0.2.0-draft/cases/valid/minimal-expense-approval.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	packFile := filepath.Join(t.TempDir(), "pack.json")
+	if err = os.WriteFile(packFile, pack, 0600); err != nil {
+		t.Fatal(err)
+	}
+	for _, profile := range []string{"base", "extended"} {
+		for _, kind := range []string{"text", "decimal", "number", "boolean", "research-value"} {
+			raw := retainedEntryDocument(kind, profile)
+			for _, operation := range []string{"check", "validate", "canonicalize", "digest"} {
+				t.Run(profile+"/"+kind+"/"+operation, func(t *testing.T) {
+					argv := []string{"dialect", operation, "-", "--draft-contract", dialect.RetainedColumnDraftSelector, "--draft-profile", profile, "--format", "json"}
+					args := map[string]any{"draftContract": dialect.RetainedColumnDraftSelector, "draftProfile": profile, "document": raw}
+					if operation == "check" {
+						argv = append(argv, "--pack", packFile)
+						args["pack"] = string(pack)
+					}
+					code, out, stderr := runTest(t, argv, raw)
+					var cliReport map[string]any
+					if code != 0 || stderr != "" || json.Unmarshal([]byte(out), &cliReport) != nil {
+						t.Fatal("actual CLI failed", code, out, stderr)
+					}
+					mcpReport, isError := dialectMCPResult(t, engine, "dialect_"+operation, args)
+					delete(cliReport, "command")
+					delete(mcpReport, "command")
+					if isError || !reflect.DeepEqual(cliReport, mcpReport) {
+						t.Fatal("actual CLI/MCP Draft mismatch", cliReport, mcpReport)
+					}
+					if cliReport["maturity"] != "experimental-draft" || cliReport["specRevision"] != dialect.RetainedColumnDraftRevision || cliReport["fullDocumentConformance"] != false {
+						t.Fatal("wrong Draft identity/scope", cliReport)
+					}
+					if operation == "check" {
+						coverage := cliReport["compatibility"].(map[string]any)
+						if coverage["status"] != "complete" || coverage["executionChecked"] != false || coverage["valueTypesChecked"] != false || len(coverage["evidence"].([]any)) != 2 {
+							t.Fatal("converted targets bypassed real inventory", coverage)
+						}
+					}
+				})
+			}
+		}
+		code, out, stderr := runTest(t, []string{"dialect", "contract", "--draft-contract", dialect.RetainedColumnDraftSelector, "--draft-profile", profile, "--format", "json"}, "")
+		var cliReport map[string]any
+		if code != 0 || stderr != "" || json.Unmarshal([]byte(out), &cliReport) != nil {
+			t.Fatal(code, out, stderr)
+		}
+		mcpReport, isError := dialectMCPResult(t, engine, "get_dialect_contract", map[string]any{"draftContract": dialect.RetainedColumnDraftSelector, "draftProfile": profile})
+		delete(cliReport, "command")
+		delete(mcpReport, "command")
+		if isError || !reflect.DeepEqual(cliReport, mcpReport) {
+			t.Fatal("contract transport mismatch", cliReport, mcpReport)
+		}
+	}
+}
+
+func TestRetainedColumnEntryCLISelectionBeforeFileRead(t *testing.T) {
+	missing := filepath.Join(t.TempDir(), "missing.json")
+	for _, flags := range [][]string{
+		{"--draft-contract", dialect.RetainedColumnDraftSelector},
+		{"--draft-profile", "base"}, {"--retained-frame"},
+		{"--draft-contract", dialect.RetainedColumnDraftSelector + "foreign", "--draft-profile", "base"},
+		{"--draft-contract", dialect.RetainedColumnDraftSelector, "--draft-profile", "future"},
+	} {
+		argv := append([]string{"dialect", "validate", missing, "--format", "json"}, flags...)
+		code, out, stderr := runTest(t, argv, "")
+		var r map[string]any
+		if code == 0 || code == 4 || stderr != "" || json.Unmarshal([]byte(out), &r) != nil || r["status"] != "refused" || strings.Contains(out, "JPS-DIALECT-READ") {
+			t.Fatal("selection did not precede opening nonexistent input", code, out, stderr)
+		}
+	}
+	raw := retainedEntryDocument("text", "base")
+	code, out, _ := runTest(t, []string{"dialect", "validate", "-", "--format", "json"}, raw)
+	if code == 0 || strings.Contains(out, "conversionContracts") || strings.Contains(out, `"maturity":"experimental-draft"`) {
+		t.Fatal("Draft auto-enabled", code, out)
+	}
+}
+
+func TestRetainedColumnEntryCLIFrameAndByteBoundaries(t *testing.T) {
+	for _, profile := range []string{"base", "extended"} {
+		raw := retainedEntryDocument("text", profile)
+		bound := dialect.RetainedColumnDraftDocumentBytes
+		if profile == "extended" {
+			bound = dialect.RetainedColumnDraftExtendedDocumentBytes
+		}
+		argv := []string{"dialect", "validate", "-", "--format", "json", "--draft-contract", dialect.RetainedColumnDraftSelector, "--draft-profile", profile}
+		for _, extra := range []int{0, 1} {
+			padded := raw + strings.Repeat(" ", bound-len(raw)+extra)
+			code, out, stderr := runTest(t, argv, padded)
+			if stderr != "" || (code == 0) != (extra == 0) || extra == 1 && !strings.Contains(out, "input-over-bound") {
+				t.Fatal("CLI selected byte boundary", profile, extra, code, out, stderr)
+			}
+		}
+		frame := `{"contractVersion":"1","origin":"asserted","values":{"Column":"-0"},"research":null}`
+		frameArgs := append(append([]string{}, argv...), "--retained-frame")
+		code, out, stderr := runTest(t, frameArgs, frame)
+		if code != 0 || stderr != "" || !strings.Contains(out, `"status":"retained-frame-valid"`) {
+			t.Fatal("CLI actual frame entry failed", code, out, stderr)
+		}
+		engine, _ := validation.NewEngine()
+		mcpReport, isError := dialectMCPResult(t, engine, "dialect_validate", map[string]any{"draftContract": dialect.RetainedColumnDraftSelector, "draftProfile": profile, "retainedFrame": true, "document": frame})
+		var cliReport map[string]any
+		json.Unmarshal([]byte(out), &cliReport)
+		delete(cliReport, "command")
+		delete(mcpReport, "command")
+		if isError || !reflect.DeepEqual(cliReport, mcpReport) {
+			t.Fatal("frame transport mismatch", cliReport, mcpReport)
+		}
+	}
+	// A decoded frame may fit CLI yet exceed MCP's envelope with HTML escaping.
+	frame := `{"contractVersion":"1","origin":"generated","values":{},"research":{"recordRecipeRevision":"r","declaration":null,"fields":{"Column":{"status":"supported","valueJson":"` + strings.Repeat("<", 3*1024*1024) + `"}}}}`
+	code, out, stderr := runTest(t, []string{"dialect", "validate", "-", "--retained-frame", "--draft-contract", dialect.RetainedColumnDraftSelector, "--draft-profile", "extended", "--format", "json"}, frame)
+	if len(frame) >= dialect.RetainedColumnDraftExtendedFrameBytes || code != 0 || stderr != "" || !strings.Contains(out, `"status":"retained-frame-valid"`) {
+		t.Fatal("CLI should admit decoded large opaque frame", code, stderr)
+	}
+}