From 783f5fe60c73e01eaaa8041ee086d6a5900dd6ae Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 14:48:04 +0200 Subject: [PATCH 01/15] feat: add a schema for machine-readable provider conformance reports First step of the design discussed in #424: the report format, before any implementation emits one, so four languages do not invent four dialects. Three properties are deliberate and are the parts worth arguing about. Four outcomes, not two. not-declared and not-applicable are distinct, and collapsing them misrepresents a provider: @strict-numeric-typing is unsatisfiable in JavaScript because the language has no integer type, so reporting it as not-declared would show every JS provider as missing something no JS provider can have. Per-scenario results are required, and required to be complete. That is what makes the appendix's own rule -- a skipped scenario is never reported as passed -- checkable rather than aspirational. It is not hypothetical: godog's summary counts capability-gated skips as passes, so the Go runner satisfies the rule only in a separate log line. A runner with that flaw still produces a truthful report here. The spec revision sits inside the tck object rather than standing alone, because it is a property of the artifact that ran rather than an independent assertion, and it travels with a digest of the assets actually executed. The revision is a claim; the digest is the check that catches a locally edited vendored file. There is deliberately no aggregate score. A single percentage would reward declaring fewer capabilities, which inverts the incentive the suite exists to create. Signed-off-by: Simon Schrottner --- specification/assets/provider-tck/README.md | 1 + .../report/conformance-report.schema.json | 171 ++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 specification/assets/provider-tck/report/conformance-report.schema.json diff --git a/specification/assets/provider-tck/README.md b/specification/assets/provider-tck/README.md index 2ce9b246..be575e5a 100644 --- a/specification/assets/provider-tck/README.md +++ b/specification/assets/provider-tck/README.md @@ -16,6 +16,7 @@ These validate a **provider** against a real backend. For assets that validate a | [`gherkin/reason.feature`](./gherkin/reason.feature) | the standard resolution reasons, gated behind `@standard-reasons` | | [`flags/canonical-flags.json`](./flags/canonical-flags.json) | the flag set every scenario assumes | | [`openapi/control-api.yaml`](./openapi/control-api.yaml) | the HTTP surface a backend under test must expose | +| [`report/conformance-report.schema.json`](./report/conformance-report.schema.json) | the shape of a machine-readable conformance report | ## These three travel together diff --git a/specification/assets/provider-tck/report/conformance-report.schema.json b/specification/assets/provider-tck/report/conformance-report.schema.json new file mode 100644 index 00000000..f5a420a6 --- /dev/null +++ b/specification/assets/provider-tck/report/conformance-report.schema.json @@ -0,0 +1,171 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openfeature.dev/schema/provider-tck/conformance-report-1.json", + "title": "OpenFeature provider conformance report", + "description": "The machine-readable result of one run of the provider conformance suite (Appendix F) against one provider in one configuration. Emitted by a TCK implementation; consumed by anything that aggregates or displays conformance. See https://github.com/open-feature/spec/issues/424.", + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "provider", "sdk", "tck", "capabilities", "scenarios"], + "properties": { + "schemaVersion": { + "description": "The version of this schema the report conforms to. Integer-as-string so that a consumer can reject a report it does not understand rather than guessing.", + "type": "string", + "pattern": "^[1-9][0-9]*$" + }, + + "provider": { + "type": "object", + "additionalProperties": false, + "required": ["name", "language"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "version": { "type": "string" }, + "language": { + "enum": ["go", "java", "javascript", "dotnet", "python", "php", "ruby", "kotlin", "swift", "rust", "other"] + }, + "configuration": { + "description": "Which configuration of the provider was tested, when a provider has more than one materially different mode. One provider may therefore produce several reports, and they are not interchangeable: flagd's RPC and in-process resolvers differ in whether they emit PROVIDER_STALE, so a report keyed on provider name alone would have to pick one and misrepresent the other.", + "type": "string" + }, + "repository": { "type": "string", "format": "uri" } + } + }, + + "sdk": { + "type": "object", + "additionalProperties": false, + "required": ["name", "version"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "version": { "type": "string", "minLength": 1 } + } + }, + + "tck": { + "description": "What asked the questions, and which questions. The spec revision is a property of the TCK artifact rather than an independent assertion: a given TCK build consumes exactly one revision of the assets.", + "type": "object", + "additionalProperties": false, + "required": ["implementation", "version", "specRevision"], + "properties": { + "implementation": { + "description": "Which TCK implementation produced this, e.g. 'go-sdk-contrib/tools/provider-tck'.", + "type": "string", + "minLength": 1 + }, + "version": { "type": "string", "minLength": 1 }, + "specRevision": { + "description": "The open-feature/spec commit the executed assets came from.", + "type": "string", + "minLength": 7 + }, + "specRelease": { + "description": "A human-readable spec release tag, when one applies. A commit is precise; a tag is what a person can reason about.", + "type": "string" + }, + "gherkinDigest": { + "description": "Digest over the feature files and canonical flag set actually executed, as 'sha256:'. The revision is an assertion; this is a check. A recorded revision can be wrong if a vendored file was edited locally, and this is what catches that -- which is the drift the whole suite exists to prevent.", + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + } + } + }, + + "backend": { + "description": "What the provider was pointed at. Omitted for a provider with no backend, which is itself informative.", + "type": "object", + "additionalProperties": false, + "properties": { + "description": { "type": "string" }, + "controlApi": { + "description": "How the backend was driven. 'http' means the normative control API; 'in-process' is the narrow allowance for providers with no backend, and a report claiming it for a provider that has one should be treated with suspicion.", + "enum": ["http", "in-process"] + } + } + }, + + "provenance": { + "description": "Where this report came from. A report is a claim, not an audit, and this is what lets a consumer decide how much to trust it.", + "type": "object", + "additionalProperties": false, + "properties": { + "runUrl": { "type": "string", "format": "uri" }, + "commit": { "type": "string" }, + "timestamp": { "type": "string", "format": "date-time" }, + "attestation": { + "description": "An optional signed attestation (e.g. sigstore bundle) covering this document.", + "type": "string" + } + } + }, + + "capabilities": { + "description": "Per-capability outcome, keyed by Gherkin tag including the leading at-sign.", + "type": "object", + "propertyNames": { "pattern": "^@[a-z0-9-]+$" }, + "additionalProperties": { "$ref": "#/$defs/capabilityResult" } + }, + + "scenarios": { + "description": "Per-scenario outcome. Required, and required to be complete: it is what makes 'a skipped scenario is never reported as passed' checkable rather than a promise. A runner whose own summary counts skips as passes still produces a truthful report here.", + "type": "array", + "items": { "$ref": "#/$defs/scenarioResult" } + } + }, + + "$defs": { + "outcome": { + "description": "Four outcomes, not two. 'not-declared' and 'not-applicable' are distinct and collapsing them misrepresents a provider: @strict-numeric-typing is unsatisfiable in JavaScript because the language has no integer type, so reporting it as 'not-declared' would show every JS provider as missing something no JS provider can have.", + "enum": ["passed", "failed", "not-declared", "not-applicable"] + }, + + "capabilityResult": { + "oneOf": [ + { "$ref": "#/$defs/outcome" }, + { + "type": "object", + "additionalProperties": false, + "required": ["state"], + "properties": { + "state": { "$ref": "#/$defs/outcome" }, + "reason": { + "description": "Why, in a form a human reading a comparison page can use. Required in practice for anything other than 'passed' -- 'this provider does not support configuration-change events' is useful information for someone choosing a provider.", + "type": "string" + }, + "knownDeviation": { + "description": "Set when the provider fails a capability the specification does not treat as optional, and the gap is tracked. Distinguishes 'chose not to implement an optional feature' from 'has a known bug', which the tag vocabulary alone cannot express.", + "type": "object", + "additionalProperties": false, + "properties": { + "issue": { "type": "string", "format": "uri" }, + "summary": { "type": "string" } + } + } + } + } + ] + }, + + "scenarioResult": { + "type": "object", + "additionalProperties": false, + "required": ["feature", "name", "outcome"], + "properties": { + "feature": { + "description": "The feature file, without extension, e.g. 'errors'.", + "type": "string" + }, + "name": { "type": "string", "minLength": 1 }, + "tags": { + "type": "array", + "items": { "type": "string", "pattern": "^@[a-z0-9-]+$" } + }, + "outcome": { "$ref": "#/$defs/outcome" }, + "reason": { + "description": "For a skip, why it was skipped. For a failure, what failed.", + "type": "string" + }, + "durationMs": { "type": "number", "minimum": 0 } + } + } + } +} From 6cc62598ec118d40f6db55c518b672110c876dbb Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 16:20:46 +0200 Subject: [PATCH 02/15] feat: identify executed assets by git tree hash rather than a content digest The schema carried a `gherkinDigest`, a SHA-256 over the feature files and the canonical flag set, on the argument that `specRevision` is an assertion and a digest is a check. The check is worth having; computing it ourselves is not. It would have made every implementation carry a normative hashing algorithm -- agreeing on file selection, path form, ordering, framing and line-ending normalisation, the last of which matters because Go embeds committed copies that a Windows checkout converts to CRLF. Four implementations of that is a lot of surface for something git already computes. `git rev-parse :specification/assets/provider-tck` is the same idea with none of the specification. It has two properties the digest was reaching for. It is verifiable: a third party can recompute it from the recorded revision, so a revision recorded wrongly does not pass unnoticed. And it is stable across unrelated spec commits, so two runs that executed identical assets report the same value even when pinned to different commits -- which the commit SHA alone gets wrong, since it changes for every unrelated edit to the specification. The drift the digest was chiefly aimed at -- a vendored copy edited locally while the recorded revision still points at the original -- is in any case already handled where it arises. Java, Python and JavaScript read the assets straight from the submodule, so no second copy exists to diverge. Go has committed copies and a CI check that regenerates them and fails on any difference. It is a build-time problem with a build-time fix, and pushing it into the report format would have every implementation re-detect what its own CI already catches. The cost is that the tree covers the asset directory's documentation too, so a README edit changes it. That errs towards reporting two comparable runs as different rather than the reverse, which is the safe direction. Signed-off-by: Simon Schrottner --- .../provider-tck/report/conformance-report.schema.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/assets/provider-tck/report/conformance-report.schema.json b/specification/assets/provider-tck/report/conformance-report.schema.json index f5a420a6..ea696474 100644 --- a/specification/assets/provider-tck/report/conformance-report.schema.json +++ b/specification/assets/provider-tck/report/conformance-report.schema.json @@ -62,10 +62,10 @@ "description": "A human-readable spec release tag, when one applies. A commit is precise; a tag is what a person can reason about.", "type": "string" }, - "gherkinDigest": { - "description": "Digest over the feature files and canonical flag set actually executed, as 'sha256:'. The revision is an assertion; this is a check. A recorded revision can be wrong if a vendored file was edited locally, and this is what catches that -- which is the drift the whole suite exists to prevent.", + "assetsTree": { + "description": "The git tree object ID of specification/assets/provider-tck at that revision, as 40 lowercase hex characters. Two things make this worth carrying alongside the revision. It is stable across unrelated spec commits, so two runs that executed identical assets report the same value even when pinned to different commits -- which is precisely the comparability question a consumer is asking. And it is checkable rather than merely asserted: `git rev-parse :specification/assets/provider-tck` must reproduce it, so a revision recorded wrongly does not go unnoticed. Note that it covers the whole asset directory including its documentation, so a README edit changes it; that errs towards reporting two comparable runs as different, rather than the reverse.", "type": "string", - "pattern": "^sha256:[0-9a-f]{64}$" + "pattern": "^[0-9a-f]{40}$" } } }, From 197022802d8b725ba8913f7af78e29754f993257 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 17:26:21 +0200 Subject: [PATCH 03/15] feat: enforce reasons, and let a single scenario carry a known deviation Three changes, all of them found by implementing the format rather than by reading it. Writing four emitters against this schema was always going to be the real review, and it was. A reason is now required whenever an outcome is not "passed", at both the capability and the scenario level. The schema previously said it was "required in practice", which is another way of saying not required: nothing enforced it, and a report of bare tags with no explanation would have validated. The reason is most of the value to a person comparing providers -- "does not support configuration-change events" is information, "@configuration-change: not-declared" is a shrug. knownDeviation moves to a shared definition and becomes available on a scenario as well as a capability, because the two are different claims. A capability-level deviation says the provider does not do X. A scenario-level one pins the deviation to the single case that fails, which is what an implementation marking one expected failure has to record -- the Python suite marks exactly one, for open-feature/python-sdk#619, and until now had nowhere to put it but prose in the reason string. The capabilities object now states plainly that it is not a verdict. Scenarios carrying no capability tag are mandatory, roll up into no capability, and are therefore invisible in that summary -- so a provider can fail a mandatory scenario while every entry reads "passed". The Python implementation produced exactly that report. A consumer deciding whether a provider conforms has to read the scenarios, which is the reason those are required and required to be complete, and the schema should say so where someone will read it rather than leaving it to be discovered. All reports emitted by the Go, Python and JavaScript implementations still validate, and the two new constraints were checked to reject a report that omits a reason rather than merely being decorative. Signed-off-by: Simon Schrottner --- .../report/conformance-report.schema.json | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/specification/assets/provider-tck/report/conformance-report.schema.json b/specification/assets/provider-tck/report/conformance-report.schema.json index ea696474..9136dc89 100644 --- a/specification/assets/provider-tck/report/conformance-report.schema.json +++ b/specification/assets/provider-tck/report/conformance-report.schema.json @@ -99,7 +99,7 @@ }, "capabilities": { - "description": "Per-capability outcome, keyed by Gherkin tag including the leading at-sign.", + "description": "Per-capability outcome, keyed by Gherkin tag including the leading at-sign. This is a summary of the OPTIONAL contract only and is not a verdict: scenarios carrying no capability tag are mandatory, roll up into no capability, and are therefore invisible here. A provider can fail a mandatory scenario while every entry in this object reads 'passed', so a consumer deciding whether a provider conforms must read 'scenarios'. That is why per-scenario results are required and required to be complete.", "type": "object", "propertyNames": { "pattern": "^@[a-z0-9-]+$" }, "additionalProperties": { "$ref": "#/$defs/capabilityResult" } @@ -125,21 +125,15 @@ "type": "object", "additionalProperties": false, "required": ["state"], + "if": { "properties": { "state": { "const": "passed" } } }, + "else": { "required": ["reason"] }, "properties": { "state": { "$ref": "#/$defs/outcome" }, "reason": { - "description": "Why, in a form a human reading a comparison page can use. Required in practice for anything other than 'passed' -- 'this provider does not support configuration-change events' is useful information for someone choosing a provider.", + "description": "Why, in a form a human reading a comparison page can use. Required for anything other than 'passed', and enforced by this schema rather than left to convention -- 'this provider does not support configuration-change events' is useful information for someone choosing a provider, and a bare tag with no explanation is not.", "type": "string" }, - "knownDeviation": { - "description": "Set when the provider fails a capability the specification does not treat as optional, and the gap is tracked. Distinguishes 'chose not to implement an optional feature' from 'has a known bug', which the tag vocabulary alone cannot express.", - "type": "object", - "additionalProperties": false, - "properties": { - "issue": { "type": "string", "format": "uri" }, - "summary": { "type": "string" } - } - } + "knownDeviation": { "$ref": "#/$defs/knownDeviation" } } } ] @@ -149,6 +143,8 @@ "type": "object", "additionalProperties": false, "required": ["feature", "name", "outcome"], + "if": { "properties": { "outcome": { "const": "passed" } } }, + "else": { "required": ["reason"] }, "properties": { "feature": { "description": "The feature file, without extension, e.g. 'errors'.", @@ -161,10 +157,21 @@ }, "outcome": { "$ref": "#/$defs/outcome" }, "reason": { - "description": "For a skip, why it was skipped. For a failure, what failed.", + "description": "For a skip, why it was skipped. For a failure, what failed. Required whenever the outcome is not 'passed', and enforced rather than merely expected: 'not-declared' without a reason is the difference between a report a person can act on and a wall of tags.", "type": "string" }, - "durationMs": { "type": "number", "minimum": 0 } + "durationMs": { "type": "number", "minimum": 0 }, + "knownDeviation": { "$ref": "#/$defs/knownDeviation" } + } + }, + + "knownDeviation": { + "description": "Set when a provider fails something the specification does not treat as optional, and the gap is tracked. It distinguishes 'chose not to implement an optional feature' from 'has a known bug', which the tag vocabulary alone cannot express. It applies at both levels because the two are not the same claim: a capability-level deviation says the provider does not do X, while a scenario-level one pins the deviation to the single case that fails, which is what an implementation marking one expected failure has to record.", + "type": "object", + "additionalProperties": false, + "properties": { + "issue": { "type": "string", "format": "uri" }, + "summary": { "type": "string" } } } } From a372701497ae076d39dfe1a1a2234e431e4a5a54 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 21:09:11 +0200 Subject: [PATCH 04/15] feat: identify a Scenario Outline row by its parameters A scenario entry was identified by feature and name, and every row of a Scenario Outline shares one name. The type-mismatch matrix is eleven rows, so a report carried eleven entries distinguished by nothing but a duration measurement. When one row fails and ten pass -- which is what the Python implementation actually produced -- the report cannot say which failed, and a consumer keying on feature and name keeps whichever row it happened to see last. An entry now carries the Examples row it came from, as the row's parameters keyed by column header, and identity is feature, name and example together. The parameters are the identity, and they come from the feature file rather than from any runner, which is why this is a field rather than a naming convention. Mandating a mangled name would have put a separator, an ordering and an escaping rule for values containing the separator into normative text that four implementations must reproduce byte-for-byte, where drift stays invisible until two reports quietly fail to line up. That was not a hypothetical risk. Before this field existed the three implementations had already diverged on exactly this point, each reasonably and none compatibly: Go emitted the bare scenario name for all eleven rows, Python appended pytest's example id, JavaScript used jest-cucumber's expanded title. The same row of the same feature file had three identities, which defeats the comparison the format exists to make possible. Values are the cell contents verbatim as strings, because Gherkin has no types. Coercing "1" to a number here would make the report disagree with the table it came from. Signed-off-by: Simon Schrottner --- .../provider-tck/report/conformance-report.schema.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/specification/assets/provider-tck/report/conformance-report.schema.json b/specification/assets/provider-tck/report/conformance-report.schema.json index 9136dc89..d081e6c0 100644 --- a/specification/assets/provider-tck/report/conformance-report.schema.json +++ b/specification/assets/provider-tck/report/conformance-report.schema.json @@ -106,7 +106,7 @@ }, "scenarios": { - "description": "Per-scenario outcome. Required, and required to be complete: it is what makes 'a skipped scenario is never reported as passed' checkable rather than a promise. A runner whose own summary counts skips as passes still produces a truthful report here.", + "description": "Per-scenario outcome. Required, and required to be complete: it is what makes 'a skipped scenario is never reported as passed' checkable rather than a promise. A runner whose own summary counts skips as passes still produces a truthful report here. An entry is identified by feature, name and example together -- not by feature and name, which are shared by every row of a Scenario Outline.", "type": "array", "items": { "$ref": "#/$defs/scenarioResult" } } @@ -151,6 +151,12 @@ "type": "string" }, "name": { "type": "string", "minLength": 1 }, + "example": { + "description": "The Examples row this entry came from, as the row's parameters keyed by column header. Required for a scenario originating from a Scenario Outline, omitted otherwise.\n\nWithout it a report is ambiguous exactly where it matters most. Every row of an outline shares one scenario name, so eleven rows of a type-mismatch matrix produce eleven entries with identical feature and name; if one fails and ten pass, the report cannot say which failed, and a consumer keying on feature and name keeps whichever row it saw last.\n\nIt is a field rather than a naming convention because the parameters are the identity, and they come from the feature file rather than from any runner. Mandating a mangled name instead would put a separator, an ordering and an escaping rule into normative text that every implementation must reproduce byte-for-byte, and drift there is invisible until two reports silently fail to line up. Implementations were observed diverging on precisely this point before the field existed: one emitted the bare scenario name for all eleven rows, another appended its runner's example id, a third its runner's expanded title.\n\nValues are the cell contents verbatim, as strings, because Gherkin has no types -- no coercion, no trimming beyond what the parser does, so the report says what the table said.", + "type": "object", + "minProperties": 1, + "additionalProperties": { "type": "string" } + }, "tags": { "type": "array", "items": { "type": "string", "pattern": "^@[a-z0-9-]+$" } From 2442f5f945e1c0e02c3005a5b74211ed606261fe Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 21:56:41 +0200 Subject: [PATCH 05/15] docs: state that example key order is not significant Two implementations asked independently whether the four languages should agree on the key order within an example object. They currently do not: Java emits Examples column order, Go sorts alphabetically because encoding/json sorts map keys when marshalling, and neither is wrong. A JSON object is unordered, so the answer is that order carries no meaning and implementations need not agree. Saying so explicitly is worth a sentence, because the alternative is three more implementations each deciding privately and someone eventually byte-comparing two reports and concluding they disagree when they do not. If byte-level reproducibility is ever required -- an attestation over the document is the obvious case -- that is a canonicalisation problem for the whole report, not something to solve inside one field. Signed-off-by: Simon Schrottner --- .../assets/provider-tck/report/conformance-report.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/assets/provider-tck/report/conformance-report.schema.json b/specification/assets/provider-tck/report/conformance-report.schema.json index d081e6c0..60c2ae7c 100644 --- a/specification/assets/provider-tck/report/conformance-report.schema.json +++ b/specification/assets/provider-tck/report/conformance-report.schema.json @@ -152,7 +152,7 @@ }, "name": { "type": "string", "minLength": 1 }, "example": { - "description": "The Examples row this entry came from, as the row's parameters keyed by column header. Required for a scenario originating from a Scenario Outline, omitted otherwise.\n\nWithout it a report is ambiguous exactly where it matters most. Every row of an outline shares one scenario name, so eleven rows of a type-mismatch matrix produce eleven entries with identical feature and name; if one fails and ten pass, the report cannot say which failed, and a consumer keying on feature and name keeps whichever row it saw last.\n\nIt is a field rather than a naming convention because the parameters are the identity, and they come from the feature file rather than from any runner. Mandating a mangled name instead would put a separator, an ordering and an escaping rule into normative text that every implementation must reproduce byte-for-byte, and drift there is invisible until two reports silently fail to line up. Implementations were observed diverging on precisely this point before the field existed: one emitted the bare scenario name for all eleven rows, another appended its runner's example id, a third its runner's expanded title.\n\nValues are the cell contents verbatim, as strings, because Gherkin has no types -- no coercion, no trimming beyond what the parser does, so the report says what the table said.", + "description": "The Examples row this entry came from, as the row's parameters keyed by column header. Required for a scenario originating from a Scenario Outline, omitted otherwise.\n\nWithout it a report is ambiguous exactly where it matters most. Every row of an outline shares one scenario name, so eleven rows of a type-mismatch matrix produce eleven entries with identical feature and name; if one fails and ten pass, the report cannot say which failed, and a consumer keying on feature and name keeps whichever row it saw last.\n\nIt is a field rather than a naming convention because the parameters are the identity, and they come from the feature file rather than from any runner. Mandating a mangled name instead would put a separator, an ordering and an escaping rule into normative text that every implementation must reproduce byte-for-byte, and drift there is invisible until two reports silently fail to line up. Implementations were observed diverging on precisely this point before the field existed: one emitted the bare scenario name for all eleven rows, another appended its runner's example id, a third its runner's expanded title.\n\nValues are the cell contents verbatim, as strings, because Gherkin has no types -- no coercion, no trimming beyond what the parser does, so the report says what the table said.\n\nKey order is not significant and implementations are not required to agree on it: a JSON object is unordered, and implementations observed so far differ (one emits Examples column order, another sorts alphabetically because its language sorts map keys when marshalling). Compare these by parsing, not by comparing bytes. If byte-level reproducibility is ever needed -- for an attestation over the document, say -- that is a canonicalisation concern for the whole report rather than something to solve in this one field.", "type": "object", "minProperties": 1, "additionalProperties": { "type": "string" } From 722c822a0bef8aaa63979fea46fe85f69be869ab Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Thu, 10 Sep 2026 19:15:01 +0200 Subject: [PATCH 06/15] feat: carry results in Cucumber Messages instead of defining our own The schema defined its own per-scenario result list, a four-value outcome enum, and a field identifying which Scenario Outline row an entry came from. All three already exist in Cucumber Messages, which is maintained, cross-language and schema'd, and which cucumber-jvm emits natively. The example field is the clearest case for dropping ours. It was added days ago to make outline rows identifiable, and four implementations each reinvented it independently -- one of them by reverse-engineering how its runner maps a pickle back to a table row. Messages carries that identity as AST node ids and has all along. Messages is also better than what it replaces. It carries the executed feature source, so a consumer can verify which questions were asked rather than trusting a recorded revision, which is stronger than the tree hash this schema had been using for the same purpose. Two things Messages cannot carry, so they stay here. The declaration -- which capabilities the provider claims -- is an input to reading the results rather than a summary of them. A skipped scenario says the question was not put to this provider; only the declaration says whether that is because the provider declines the capability. Given the declaration and a scenario's tags, the reason for a skip follows, so it no longer needs transporting per scenario. The tested subject and its provenance. No standard results format has a slot for "the provider under test": Messages records the runtime and OS, not the subject. The results are referenced rather than inlined, because a Messages stream carries the feature sources and is far larger than this envelope, and a consumer deciding whether it cares about a report should not have to fetch a whole run to find out. Two lossier formats are admitted for runners that cannot emit Messages yet, with what each loses stated. Not addressed here, and still open on #424: whether a report should be published inside the provider's released artifact, and how strongly the tested subject revision must be identified. Signed-off-by: Simon Schrottner --- .../report/conformance-report.schema.json | 241 +++++++++++------- 1 file changed, 149 insertions(+), 92 deletions(-) diff --git a/specification/assets/provider-tck/report/conformance-report.schema.json b/specification/assets/provider-tck/report/conformance-report.schema.json index 60c2ae7c..d8319bf7 100644 --- a/specification/assets/provider-tck/report/conformance-report.schema.json +++ b/specification/assets/provider-tck/report/conformance-report.schema.json @@ -2,57 +2,100 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://openfeature.dev/schema/provider-tck/conformance-report-1.json", "title": "OpenFeature provider conformance report", - "description": "The machine-readable result of one run of the provider conformance suite (Appendix F) against one provider in one configuration. Emitted by a TCK implementation; consumed by anything that aggregates or displays conformance. See https://github.com/open-feature/spec/issues/424.", + "description": "The machine-readable result of one run of the provider conformance suite (Appendix F) against one provider in one configuration. This document is an envelope: it identifies what was tested, what the provider claims, and where the executed results are. The results themselves are a standard format (Cucumber Messages), not something this specification defines. See https://github.com/open-feature/spec/issues/424.", "type": "object", "additionalProperties": false, - "required": ["schemaVersion", "provider", "sdk", "tck", "capabilities", "scenarios"], + "required": [ + "schemaVersion", + "provider", + "sdk", + "tck", + "declaration", + "results" + ], "properties": { "schemaVersion": { "description": "The version of this schema the report conforms to. Integer-as-string so that a consumer can reject a report it does not understand rather than guessing.", "type": "string", "pattern": "^[1-9][0-9]*$" }, - "provider": { "type": "object", "additionalProperties": false, - "required": ["name", "language"], + "required": [ + "name", + "language" + ], "properties": { - "name": { "type": "string", "minLength": 1 }, - "version": { "type": "string" }, + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string" + }, "language": { - "enum": ["go", "java", "javascript", "dotnet", "python", "php", "ruby", "kotlin", "swift", "rust", "other"] + "enum": [ + "go", + "java", + "javascript", + "dotnet", + "python", + "php", + "ruby", + "kotlin", + "swift", + "rust", + "other" + ] }, "configuration": { "description": "Which configuration of the provider was tested, when a provider has more than one materially different mode. One provider may therefore produce several reports, and they are not interchangeable: flagd's RPC and in-process resolvers differ in whether they emit PROVIDER_STALE, so a report keyed on provider name alone would have to pick one and misrepresent the other.", "type": "string" }, - "repository": { "type": "string", "format": "uri" } + "repository": { + "type": "string", + "format": "uri" + } } }, - "sdk": { "type": "object", "additionalProperties": false, - "required": ["name", "version"], + "required": [ + "name", + "version" + ], "properties": { - "name": { "type": "string", "minLength": 1 }, - "version": { "type": "string", "minLength": 1 } + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + } } }, - "tck": { "description": "What asked the questions, and which questions. The spec revision is a property of the TCK artifact rather than an independent assertion: a given TCK build consumes exactly one revision of the assets.", "type": "object", "additionalProperties": false, - "required": ["implementation", "version", "specRevision"], + "required": [ + "implementation", + "version", + "specRevision" + ], "properties": { "implementation": { "description": "Which TCK implementation produced this, e.g. 'go-sdk-contrib/tools/provider-tck'.", "type": "string", "minLength": 1 }, - "version": { "type": "string", "minLength": 1 }, + "version": { + "type": "string", + "minLength": 1 + }, "specRevision": { "description": "The open-feature/spec commit the executed assets came from.", "type": "string", @@ -69,115 +112,129 @@ } } }, - "backend": { "description": "What the provider was pointed at. Omitted for a provider with no backend, which is itself informative.", "type": "object", "additionalProperties": false, "properties": { - "description": { "type": "string" }, + "description": { + "type": "string" + }, "controlApi": { "description": "How the backend was driven. 'http' means the normative control API; 'in-process' is the narrow allowance for providers with no backend, and a report claiming it for a provider that has one should be treated with suspicion.", - "enum": ["http", "in-process"] + "enum": [ + "http", + "in-process" + ] } } }, - - "provenance": { - "description": "Where this report came from. A report is a claim, not an audit, and this is what lets a consumer decide how much to trust it.", + "declaration": { + "description": "The capability set this provider claims, as Gherkin tags including the leading at-sign. This is an INPUT to reading the results, not a summary of them, which is why it cannot be derived from the results payload and has to be stated here. A skipped scenario in the payload says the question was not put to this provider; only the declaration says whether that is because the provider declines the capability. Given the declaration and a scenario's tags, the reason for a skip follows without needing to be transported per scenario.", "type": "object", "additionalProperties": false, + "required": [ + "declared" + ], "properties": { - "runUrl": { "type": "string", "format": "uri" }, - "commit": { "type": "string" }, - "timestamp": { "type": "string", "format": "date-time" }, - "attestation": { - "description": "An optional signed attestation (e.g. sigstore bundle) covering this document.", - "type": "string" + "declared": { + "description": "Capabilities the provider declares. A scenario tagged with anything absent from this list is expected to be skipped in the results payload.", + "type": "array", + "items": { + "type": "string", + "pattern": "^@[a-z0-9-]+$" + } + }, + "notApplicable": { + "description": "Capabilities that cannot hold for this provider rather than merely being undeclared, each with a reason. Kept separate because collapsing the two misrepresents whole languages: @strict-numeric-typing is unsatisfiable in JavaScript because the language has no integer type, and reporting that as an undeclared choice would show every JavaScript provider as missing something none of them can have. Where the impossibility is a property of the language rather than of the provider it is better recorded once in the capability documentation than repeated in every report, so this is for provider-specific cases.", + "type": "object", + "propertyNames": { + "pattern": "^@[a-z0-9-]+$" + }, + "additionalProperties": { + "type": "string", + "minLength": 1 + } } } }, - - "capabilities": { - "description": "Per-capability outcome, keyed by Gherkin tag including the leading at-sign. This is a summary of the OPTIONAL contract only and is not a verdict: scenarios carrying no capability tag are mandatory, roll up into no capability, and are therefore invisible here. A provider can fail a mandatory scenario while every entry in this object reads 'passed', so a consumer deciding whether a provider conforms must read 'scenarios'. That is why per-scenario results are required and required to be complete.", + "results": { + "description": "Where the executed results live, and in what format. The results themselves are deliberately NOT defined by this schema: per-scenario outcomes, tags, Scenario Outline row identity and the executed feature source are all already specified by Cucumber Messages, and defining them again here would create a second format to maintain and version and two places for the same fact to disagree.\n\nIt is referenced rather than inlined because a Messages stream carries the feature sources and so is far larger than this envelope, and because a consumer deciding whether it cares about a report should not have to download the whole run to find out.", "type": "object", - "propertyNames": { "pattern": "^@[a-z0-9-]+$" }, - "additionalProperties": { "$ref": "#/$defs/capabilityResult" } + "additionalProperties": false, + "required": [ + "format", + "location" + ], + "properties": { + "format": { + "description": "The results format. 'cucumber-messages' is the ndjson protocol at https://github.com/cucumber/messages -- the expected value, and the only one that carries the executed feature source and an exact Scenario Outline row identity. The others are admitted because a runner may not emit Messages yet, and are lossier: 'cucumber-json' identifies outline rows only by line number, and 'junit-xml' carries no tags at all, which leaves the capability that gated a skip unrecoverable.", + "enum": [ + "cucumber-messages", + "cucumber-json", + "junit-xml" + ] + }, + "location": { + "description": "Where to fetch the results: a URI, or a path relative to this document.", + "type": "string", + "minLength": 1 + }, + "digest": { + "description": "Digest over the results payload as 'sha256:', so a consumer can tell that what it fetched is what this envelope describes.", + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + } + } }, - - "scenarios": { - "description": "Per-scenario outcome. Required, and required to be complete: it is what makes 'a skipped scenario is never reported as passed' checkable rather than a promise. A runner whose own summary counts skips as passes still produces a truthful report here. An entry is identified by feature, name and example together -- not by feature and name, which are shared by every row of a Scenario Outline.", + "knownDeviations": { + "description": "Deviations the provider acknowledges. Optional, and an empty list is not the same as an absent one: stating none is a claim, omitting the field is silence.", "type": "array", - "items": { "$ref": "#/$defs/scenarioResult" } - } - }, - - "$defs": { - "outcome": { - "description": "Four outcomes, not two. 'not-declared' and 'not-applicable' are distinct and collapsing them misrepresents a provider: @strict-numeric-typing is unsatisfiable in JavaScript because the language has no integer type, so reporting it as 'not-declared' would show every JS provider as missing something no JS provider can have.", - "enum": ["passed", "failed", "not-declared", "not-applicable"] - }, - - "capabilityResult": { - "oneOf": [ - { "$ref": "#/$defs/outcome" }, - { - "type": "object", - "additionalProperties": false, - "required": ["state"], - "if": { "properties": { "state": { "const": "passed" } } }, - "else": { "required": ["reason"] }, - "properties": { - "state": { "$ref": "#/$defs/outcome" }, - "reason": { - "description": "Why, in a form a human reading a comparison page can use. Required for anything other than 'passed', and enforced by this schema rather than left to convention -- 'this provider does not support configuration-change events' is useful information for someone choosing a provider, and a bare tag with no explanation is not.", - "type": "string" - }, - "knownDeviation": { "$ref": "#/$defs/knownDeviation" } - } - } - ] + "items": { + "$ref": "#/$defs/knownDeviation" + } }, - - "scenarioResult": { + "provenance": { + "description": "Where this report came from. A report is a claim, not an audit, and this is what lets a consumer decide how much to trust it.", "type": "object", "additionalProperties": false, - "required": ["feature", "name", "outcome"], - "if": { "properties": { "outcome": { "const": "passed" } } }, - "else": { "required": ["reason"] }, "properties": { - "feature": { - "description": "The feature file, without extension, e.g. 'errors'.", - "type": "string" + "runUrl": { + "type": "string", + "format": "uri" }, - "name": { "type": "string", "minLength": 1 }, - "example": { - "description": "The Examples row this entry came from, as the row's parameters keyed by column header. Required for a scenario originating from a Scenario Outline, omitted otherwise.\n\nWithout it a report is ambiguous exactly where it matters most. Every row of an outline shares one scenario name, so eleven rows of a type-mismatch matrix produce eleven entries with identical feature and name; if one fails and ten pass, the report cannot say which failed, and a consumer keying on feature and name keeps whichever row it saw last.\n\nIt is a field rather than a naming convention because the parameters are the identity, and they come from the feature file rather than from any runner. Mandating a mangled name instead would put a separator, an ordering and an escaping rule into normative text that every implementation must reproduce byte-for-byte, and drift there is invisible until two reports silently fail to line up. Implementations were observed diverging on precisely this point before the field existed: one emitted the bare scenario name for all eleven rows, another appended its runner's example id, a third its runner's expanded title.\n\nValues are the cell contents verbatim, as strings, because Gherkin has no types -- no coercion, no trimming beyond what the parser does, so the report says what the table said.\n\nKey order is not significant and implementations are not required to agree on it: a JSON object is unordered, and implementations observed so far differ (one emits Examples column order, another sorts alphabetically because its language sorts map keys when marshalling). Compare these by parsing, not by comparing bytes. If byte-level reproducibility is ever needed -- for an attestation over the document, say -- that is a canonicalisation concern for the whole report rather than something to solve in this one field.", - "type": "object", - "minProperties": 1, - "additionalProperties": { "type": "string" } + "commit": { + "type": "string" }, - "tags": { - "type": "array", - "items": { "type": "string", "pattern": "^@[a-z0-9-]+$" } + "timestamp": { + "type": "string", + "format": "date-time" }, - "outcome": { "$ref": "#/$defs/outcome" }, - "reason": { - "description": "For a skip, why it was skipped. For a failure, what failed. Required whenever the outcome is not 'passed', and enforced rather than merely expected: 'not-declared' without a reason is the difference between a report a person can act on and a wall of tags.", + "attestation": { + "description": "An optional signed attestation (e.g. sigstore bundle) covering this document.", "type": "string" - }, - "durationMs": { "type": "number", "minimum": 0 }, - "knownDeviation": { "$ref": "#/$defs/knownDeviation" } + } } - }, - + } + }, + "$defs": { "knownDeviation": { - "description": "Set when a provider fails something the specification does not treat as optional, and the gap is tracked. It distinguishes 'chose not to implement an optional feature' from 'has a known bug', which the tag vocabulary alone cannot express. It applies at both levels because the two are not the same claim: a capability-level deviation says the provider does not do X, while a scenario-level one pins the deviation to the single case that fails, which is what an implementation marking one expected failure has to record.", + "description": "A gap the provider is known to have against something the specification does not treat as optional, with the gap tracked somewhere. It is distinct from an undeclared capability, which is a choice, and from a not-applicable one, which is impossible -- this is a defect that is acknowledged rather than hidden. Three of the four implementations needed it before it existed: flagd narrows floats to integers in two languages, and one Python scenario is marked as an expected failure against an SDK bug.", "type": "object", "additionalProperties": false, "properties": { - "issue": { "type": "string", "format": "uri" }, - "summary": { "type": "string" } + "capability": { + "description": "The capability the deviation concerns, if it maps to one. Omitted when the deviation is against a mandatory scenario, which belongs to no capability.", + "type": "string", + "pattern": "^@[a-z0-9-]+$" + }, + "issue": { + "type": "string", + "format": "uri" + }, + "summary": { + "type": "string" + } } } } From 785d203f677a65c4a49b0839d174e8d911b89ee1 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Thu, 10 Sep 2026 19:48:39 +0200 Subject: [PATCH 07/15] feat: record which version of the results format was produced Two corrections, both found by implementing this in Go. assetsTree is gone. It identified the executed artifacts so that a recorded spec revision could be checked rather than trusted. Cucumber Messages carries the executed feature source itself, so the check is now the source, and keeping a hash of the same thing alongside it is a second place for the answer to be wrong. The previous commit dropped it from the emitters and neglected to drop it here. formatVersion is new, and its absence was a real hole. Cucumber Messages is versioned, and implementations pin different releases: the Go TCK builds against messages v21, while cucumber-jvm ships a considerably later one and the npm package is later still. The message types differ between releases. Without recording what the producer built against, a consumer validating a stream has to guess which schema to validate against -- and guessing wrong is worse than not validating at all, because a later schema accepts messages an earlier producer could not have emitted, while an earlier one rejects messages that are perfectly valid. That hazard is not hypothetical. During this work a schema copy of the wrong version was left in a shared temporary directory, and validating a v21 stream against it would have reported success or failure for reasons unrelated to the stream. Signed-off-by: Simon Schrottner --- .../provider-tck/report/conformance-report.schema.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specification/assets/provider-tck/report/conformance-report.schema.json b/specification/assets/provider-tck/report/conformance-report.schema.json index d8319bf7..37ec349f 100644 --- a/specification/assets/provider-tck/report/conformance-report.schema.json +++ b/specification/assets/provider-tck/report/conformance-report.schema.json @@ -104,11 +104,6 @@ "specRelease": { "description": "A human-readable spec release tag, when one applies. A commit is precise; a tag is what a person can reason about.", "type": "string" - }, - "assetsTree": { - "description": "The git tree object ID of specification/assets/provider-tck at that revision, as 40 lowercase hex characters. Two things make this worth carrying alongside the revision. It is stable across unrelated spec commits, so two runs that executed identical assets report the same value even when pinned to different commits -- which is precisely the comparability question a consumer is asking. And it is checkable rather than merely asserted: `git rev-parse :specification/assets/provider-tck` must reproduce it, so a revision recorded wrongly does not go unnoticed. Note that it covers the whole asset directory including its documentation, so a README edit changes it; that errs towards reporting two comparable runs as different, rather than the reverse.", - "type": "string", - "pattern": "^[0-9a-f]{40}$" } } }, @@ -175,6 +170,11 @@ "junit-xml" ] }, + "formatVersion": { + "description": "The version of the results format, for a format that has versions. Cucumber Messages does: implementations pin different releases -- one TCK builds against messages v21 while another's runner ships a much later one -- and the message types differ between them. Without this a consumer validating a stream has to guess which schema to validate against, and guessing wrong is worse than not checking, because a later schema accepts messages an earlier producer could not have emitted and an earlier one rejects messages that are perfectly valid. Record what the producer actually built against.", + "type": "string", + "minLength": 1 + }, "location": { "description": "Where to fetch the results: a URI, or a path relative to this document.", "type": "string", From 5fa8916a9181e600ec753c1c610998e61c69eb41 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Thu, 10 Sep 2026 20:04:52 +0200 Subject: [PATCH 08/15] docs: correct what specRevision alone has to carry, and how to read a stream Two corrections from implementing this in Java. The claim that a Messages stream makes the asset tree hash redundant was wrong. The stream carries the executed feature source, so the features are corroborated rather than asserted -- but it says nothing about the canonical flag set or the control API description, and a scenario asserting that integer-flag resolves to 10 depends entirely on the flag set it ran against. specRevision now carries those two alone, and the schema says so instead of implying the stream covers everything. The tree hash stays dropped, on the same reasoning as before: every implementation either reads the assets from the submodule or regenerates and diffs its vendored copy in CI, so drift is caught where it happens. Second, a testCaseFinished message carries no status. A scenario's outcome is the most severe testStepResult among its steps, hooks included. That ordering is what makes a capability skip visible at all, since a gate aborting in a before-hook produces a SKIPPED hook result that outranks the PASSED steps it prevented -- and a consumer reading only testCaseFinished sees no outcome whatsoever. Adopting a standard format means inheriting its sharp edges, and this is the one most likely to mislead someone reading a report for the first time. Signed-off-by: Simon Schrottner --- .../assets/provider-tck/report/conformance-report.schema.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/assets/provider-tck/report/conformance-report.schema.json b/specification/assets/provider-tck/report/conformance-report.schema.json index 37ec349f..5f255d74 100644 --- a/specification/assets/provider-tck/report/conformance-report.schema.json +++ b/specification/assets/provider-tck/report/conformance-report.schema.json @@ -97,7 +97,7 @@ "minLength": 1 }, "specRevision": { - "description": "The open-feature/spec commit the executed assets came from.", + "description": "The open-feature/spec commit the executed assets came from.\n\nFor the feature files this is corroborated rather than merely asserted, because a Cucumber Messages stream carries the executed source and a consumer can compare it. For the canonical flag set and the control API description it stands alone: the stream says nothing about either, and a scenario asserting that integer-flag resolves to 10 depends entirely on the flag set it was run against. An earlier draft carried a hash of the whole asset directory to close that gap; it was dropped because every TCK implementation either reads the assets from the submodule directly or regenerates and diffs its vendored copy in CI, so drift is caught where it happens rather than re-detected in every report.", "type": "string", "minLength": 7 }, @@ -163,7 +163,7 @@ ], "properties": { "format": { - "description": "The results format. 'cucumber-messages' is the ndjson protocol at https://github.com/cucumber/messages -- the expected value, and the only one that carries the executed feature source and an exact Scenario Outline row identity. The others are admitted because a runner may not emit Messages yet, and are lossier: 'cucumber-json' identifies outline rows only by line number, and 'junit-xml' carries no tags at all, which leaves the capability that gated a skip unrecoverable.", + "description": "The results format. 'cucumber-messages' is the ndjson protocol at https://github.com/cucumber/messages -- the expected value, and the only one that carries the executed feature source and an exact Scenario Outline row identity. The others are admitted because a runner may not emit Messages yet, and are lossier: 'cucumber-json' identifies outline rows only by line number, and 'junit-xml' carries no tags at all, which leaves the capability that gated a skip unrecoverable.\n\nOne thing to know before reading a Messages stream: a testCaseFinished message carries no status. A scenario's outcome is the most severe testStepResult among its steps, hooks included, on the ordering UNKNOWN < PASSED < SKIPPED < PENDING < UNDEFINED < AMBIGUOUS < FAILED. That ordering is what makes a capability skip visible -- a gate that aborts in a before-hook produces a SKIPPED hook result outranking the PASSED steps it prevented -- and a consumer reading only testCaseFinished will see no outcome at all.", "enum": [ "cucumber-messages", "cucumber-json", From ca759c990750733699c875b45861cb0bbc37d5a2 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Thu, 10 Sep 2026 21:07:21 +0200 Subject: [PATCH 09/15] feat: a capability the suite does not test must not be declared The declaration replaced the per-capability rollup this schema used to carry, and inherited none of its discipline. The rollup omitted a capability no scenario exercised, so an untested capability never reached a report; the declaration lists whatever the provider claims, so a real report now asserts @targeting and @caching -- reserved tags with no scenarios behind them. That is the third route to one vacuous claim. It was closed once when the rollup reported such a capability as passed, once when a capability whose every scenario had been skipped still read passed, and now again through a field that looked like a plain transcription of the provider's own configuration. So the field now says what belongs in it: capabilities the provider declares and the executed suite gates on. A capability nothing tests cannot produce a skip, so it plays no part in interpreting the results, and listing it invites a reader to believe it was verified. It also says where the complaint goes. An implementation finding a declared capability the suite does not test should warn or fail, so the adopter hears about it -- silence in the report and silence to the adopter are different things, and this got in because an adopter declaring "everything except X" picks up every reserved tag on the way past. Signed-off-by: Simon Schrottner --- .../assets/provider-tck/report/conformance-report.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/assets/provider-tck/report/conformance-report.schema.json b/specification/assets/provider-tck/report/conformance-report.schema.json index 5f255d74..ce92f4e6 100644 --- a/specification/assets/provider-tck/report/conformance-report.schema.json +++ b/specification/assets/provider-tck/report/conformance-report.schema.json @@ -133,7 +133,7 @@ ], "properties": { "declared": { - "description": "Capabilities the provider declares. A scenario tagged with anything absent from this list is expected to be skipped in the results payload.", + "description": "Capabilities the provider declares AND that the executed suite gates on. A scenario tagged with anything absent from this list is expected to be skipped in the results payload.\n\nA capability that no executed scenario carries MUST NOT appear here, even when the provider genuinely supports it. Such a capability cannot produce a skip, so it plays no part in interpreting the results -- and listing it invites a reader to believe it was verified when nothing examined it. That is the vacuous conformance claim the capability vocabulary exists to prevent, and it is easy to reintroduce: an adopter declaring 'everything except X' picks up every reserved tag in the vocabulary along the way.\n\nAn implementation that finds a declared capability the suite does not test should say so where the adopter will see it -- a warning or a failed run -- rather than quietly passing it through to the report. Silence in the report and silence to the adopter are not the same thing.", "type": "array", "items": { "type": "string", From e954cac2c3997c94eeb185a20783ea4f293cc01a Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Fri, 11 Sep 2026 09:54:33 +0200 Subject: [PATCH 10/15] docs: rename the capability in the notApplicable illustration The field's description illustrates a not-applicable capability with @strict-numeric-typing, which was renamed to @numeric-coercion. Prose only, so nothing failed validation -- which is exactly why it survived the rename: a tag inside a description is invisible to every check that would otherwise have caught it. Found by an implementation updating its own references and noticing the schema it validates against still used the old name. Signed-off-by: Simon Schrottner --- .../assets/provider-tck/report/conformance-report.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/assets/provider-tck/report/conformance-report.schema.json b/specification/assets/provider-tck/report/conformance-report.schema.json index ce92f4e6..4aa91e66 100644 --- a/specification/assets/provider-tck/report/conformance-report.schema.json +++ b/specification/assets/provider-tck/report/conformance-report.schema.json @@ -141,7 +141,7 @@ } }, "notApplicable": { - "description": "Capabilities that cannot hold for this provider rather than merely being undeclared, each with a reason. Kept separate because collapsing the two misrepresents whole languages: @strict-numeric-typing is unsatisfiable in JavaScript because the language has no integer type, and reporting that as an undeclared choice would show every JavaScript provider as missing something none of them can have. Where the impossibility is a property of the language rather than of the provider it is better recorded once in the capability documentation than repeated in every report, so this is for provider-specific cases.", + "description": "Capabilities that cannot hold for this provider rather than merely being undeclared, each with a reason. Kept separate because collapsing the two misrepresents whole languages: @numeric-coercion is unsatisfiable in JavaScript because the language has no integer type, and reporting that as an undeclared choice would show every JavaScript provider as missing something none of them can have. Where the impossibility is a property of the language rather than of the provider it is better recorded once in the capability documentation than repeated in every report, so this is for provider-specific cases.", "type": "object", "propertyNames": { "pattern": "^@[a-z0-9-]+$" From e70d21ee9cc69c3ccafab0bf616147c3a47ca5d8 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Fri, 11 Sep 2026 18:08:34 +0200 Subject: [PATCH 11/15] feat: drop declaration.notApplicable, one skip carries its own reason "Not declared" and "not applicable" are both skips. Giving them separate representations asks every implementation to agree on more vocabulary without telling a reader anything the skip's reason does not already say: the scenario's tags say what was asked, `declared` says whether it was claimed, and the reason says why it was skipped. The field's own description made the argument for removing it. It reserved itself for provider-specific impossibility, on the grounds that "where the impossibility is a property of the language rather than of the provider it is better recorded once in the capability documentation than repeated in every report" -- and both motivating cases are exactly that. @numeric-coercion cannot hold where the language has a single numeric type; @large-integers cannot hold on a 32-bit accessor. Neither is a fact about a provider, and both are now stated in Appendix F. Four implementations built the field. No adoption in any of them populates it, so the provider-specific case it was kept for has not arisen in four languages. The declaration's description now says where language-level impossibility lives, so a reader who looks for this field finds the answer rather than its absence. Signed-off-by: Simon Schrottner --- .../report/conformance-report.schema.json | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/specification/assets/provider-tck/report/conformance-report.schema.json b/specification/assets/provider-tck/report/conformance-report.schema.json index 4aa91e66..b29f8093 100644 --- a/specification/assets/provider-tck/report/conformance-report.schema.json +++ b/specification/assets/provider-tck/report/conformance-report.schema.json @@ -125,7 +125,7 @@ } }, "declaration": { - "description": "The capability set this provider claims, as Gherkin tags including the leading at-sign. This is an INPUT to reading the results, not a summary of them, which is why it cannot be derived from the results payload and has to be stated here. A skipped scenario in the payload says the question was not put to this provider; only the declaration says whether that is because the provider declines the capability. Given the declaration and a scenario's tags, the reason for a skip follows without needing to be transported per scenario.", + "description": "The capability set this provider claims, as Gherkin tags including the leading at-sign. This is an INPUT to reading the results, not a summary of them, which is why it cannot be derived from the results payload and has to be stated here. A skipped scenario in the payload says the question was not put to this provider; only the declaration says whether that is because the provider declines the capability. Given the declaration and a scenario's tags, the reason for a skip follows without needing to be transported per scenario. A capability that cannot hold in a language at all -- @numeric-coercion where the language has a single numeric type, @large-integers on a 32-bit accessor -- is a property of the SDK rather than of the provider, and is recorded in Appendix F rather than restated in every report; in a run it is simply undeclared, and the skip carries the reason.", "type": "object", "additionalProperties": false, "required": [ @@ -139,17 +139,6 @@ "type": "string", "pattern": "^@[a-z0-9-]+$" } - }, - "notApplicable": { - "description": "Capabilities that cannot hold for this provider rather than merely being undeclared, each with a reason. Kept separate because collapsing the two misrepresents whole languages: @numeric-coercion is unsatisfiable in JavaScript because the language has no integer type, and reporting that as an undeclared choice would show every JavaScript provider as missing something none of them can have. Where the impossibility is a property of the language rather than of the provider it is better recorded once in the capability documentation than repeated in every report, so this is for provider-specific cases.", - "type": "object", - "propertyNames": { - "pattern": "^@[a-z0-9-]+$" - }, - "additionalProperties": { - "type": "string", - "minLength": 1 - } } } }, From fc87782249be8329e443603912ce8c23eef88458 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Fri, 11 Sep 2026 22:13:59 +0200 Subject: [PATCH 12/15] docs: say what the conformance report is, and what provenance is for The schema landed without the appendix ever describing the document it describes. Appendix F named reports twenty-three times, defined none of them, and said nothing at all about provenance -- which is the shape of gap the appendix exists to close, since a field four implementations can read two ways is one they will implement two ways. Provenance is optional and nobody emits it, and both of those are now stated rather than left to be inferred from silence. It is optional because its contents are properties of the environment that ran the suite: outside CI there is no run URL, and a working tree with uncommitted changes has no commit that describes what was executed. A report that filled the field in regardless would be worse than one that omits it, because the value of the field is precisely that a consumer can check it. That is also the argument for keeping it where the same argument removed declaration.notApplicable. That field was a second way to say what a skip's reason already said. This one carries facts nothing else in the document carries, and an absent optional object is unambiguous in a way a missing not-applicable entry was not. The section also writes down why declaration.declared and knownDeviations sit where they do, because both are read alongside the results and neither can be derived from them. Signed-off-by: Simon Schrottner --- .../appendix-f-provider-conformance.md | 55 +++++++++++++++++++ specification/assets/provider-tck/README.md | 4 +- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/specification/appendix-f-provider-conformance.md b/specification/appendix-f-provider-conformance.md index 25c9edd4..5077b8be 100644 --- a/specification/appendix-f-provider-conformance.md +++ b/specification/appendix-f-provider-conformance.md @@ -643,6 +643,61 @@ provider on a 32-bit accessor leaves undeclared. Nothing above 2^53 − 1 is ask cannot represent it, and what a provider owes a value that does not fit the requested accessor is the open question in [open-feature/spec#430](https://github.com/open-feature/spec/issues/430). +## The conformance report + +A run may emit a **conformance report**: a machine-readable document saying what was tested, what the +provider claimed, and where the results are. Its shape is fixed by +[`assets/provider-tck/report/conformance-report.schema.json`](./assets/provider-tck/report/conformance-report.schema.json), +and it is deliberately a transcription of that schema in every language rather than whatever each +language finds convenient, because the point of the format is that four implementations emit the +same thing. + +Emitting one is optional. A developer running the suite locally wants a pass or a failure, not a +document; CI publishing a claim wants the document. Nothing about a provider's conformance depends +on whether a report was written. + +The report is an **envelope**. It identifies the run and points at the results; it does not contain +them. The results are Cucumber Messages written alongside it, because per-scenario outcomes, tags, +Scenario Outline row identity and the executed feature source are all already specified there. +Restating them in a second format would create two places for the same fact to disagree. + +Two fields carry the interpretation the results cannot: + +- **`declaration.declared`** is every capability the configuration claims, as tags. It is an input to + reading the results rather than a summary of them: a skipped scenario says only that the question + was not put to this provider, and it is the declaration that says whether that is because the + provider declines the capability. Given the declaration and a scenario's tags, the reason for a + skip follows without being transported per scenario. +- **`knownDeviations`** are the gaps the provider author acknowledges. It sits beside the declaration + rather than inside it because it is not a claim about capabilities: an entry may concern a declared + capability, or a mandatory scenario belonging to no capability at all. What it qualifies is which + of the declaration's absences were decisions and which were defects — a distinction the results + cannot carry, because a skip looks the same either way. Omitted when empty, which is silence rather + than a claim of having no known gaps. + +### Provenance + +`provenance` is optional and records where a report came from: `runUrl`, `commit`, `timestamp` and an +optional signed `attestation`. + +It exists because **a report is a claim, not an audit**. Everything else in the document is what the +provider says about itself, and a consumer comparing two providers' reports — or deciding whether to +publish a badge from one — has no way to tell a report produced by a CI run against a tagged commit +from one produced by hand on a laptop. `provenance` is what lets that consumer decide how much to +trust the rest. + +It is optional rather than required because the facts in it are properties of the environment that +ran the suite, not of the provider or the TCK. A TCK implementation cannot invent them: outside CI +there is no run URL, and a working tree with uncommitted changes has no commit that describes what +was actually executed. A report that filled the field in anyway would be worse than one that omits +it, because the whole value of the field is that its contents can be checked. + +**No TCK implementation is required to emit it, and at the time of writing none does.** It is +specified here so that the first one to need it does not invent a second shape for the same facts, +and so that a reader who finds the field absent knows that means "not stated" rather than "this +implementation calls it something else". Where a suite runs in CI and the information is available, +populating it is encouraged; a consumer must treat its absence as unremarkable. + ## Implementing the suite in a language A TCK implementation is the language-specific harness around these three artifacts. Each obligation diff --git a/specification/assets/provider-tck/README.md b/specification/assets/provider-tck/README.md index be575e5a..b8347f5f 100644 --- a/specification/assets/provider-tck/README.md +++ b/specification/assets/provider-tck/README.md @@ -18,10 +18,12 @@ These validate a **provider** against a real backend. For assets that validate a | [`openapi/control-api.yaml`](./openapi/control-api.yaml) | the HTTP surface a backend under test must expose | | [`report/conformance-report.schema.json`](./report/conformance-report.schema.json) | the shape of a machine-readable conformance report | -## These three travel together +## The three inputs travel together A feature file that evaluates `boolean-flag` is meaningless without the flag definition, and a disconnect scenario is meaningless without the control endpoint that produces the disconnect. Changing one without the others breaks the suite in every language at once. +The report schema is not one of the three: it describes what a run *emits*, not what a run needs, and a suite that never writes a report is no less conformant for it. + ## Five properties that are load-bearing - **`missing-flag` must not exist** in the flag set. Its absence is what the `FLAG_NOT_FOUND` scenario tests. Seeding it turns that scenario green for the wrong reason. From 4978d929fe8d7907a909ffddaf653e0015b9913e Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sat, 12 Sep 2026 13:41:29 +0200 Subject: [PATCH 13/15] feat: require a deviation's summary, and stop implying its issue is mandatory The knownDeviation definition had no required array, so a deviation carrying neither a summary nor an issue validated. summary is now required and non-empty: one that does not say what the gap is leaves a reader worse off than the bare skip or failure it accompanies, which is the thing it exists to explain. issue stays optional and now says so. The description claimed the gap was "tracked somewhere" while the schema required nothing of the kind, and one implementation read that as normative -- Python made issue a mandatory field with the docstring "a URI, because the schema requires one", leaving it the only one of four that cannot record an untracked defect. Naming an untracked defect is precisely what distinguishes it from a capability the provider chose to withhold, so the schema should not have been read as forbidding it and now cannot be. Appendix F carries which of the two legitimate shapes to prefer; the description points at it rather than restating it. Signed-off-by: Simon Schrottner --- .../provider-tck/report/conformance-report.schema.json | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/specification/assets/provider-tck/report/conformance-report.schema.json b/specification/assets/provider-tck/report/conformance-report.schema.json index b29f8093..0f478467 100644 --- a/specification/assets/provider-tck/report/conformance-report.schema.json +++ b/specification/assets/provider-tck/report/conformance-report.schema.json @@ -208,9 +208,12 @@ }, "$defs": { "knownDeviation": { - "description": "A gap the provider is known to have against something the specification does not treat as optional, with the gap tracked somewhere. It is distinct from an undeclared capability, which is a choice, and from a not-applicable one, which is impossible -- this is a defect that is acknowledged rather than hidden. Three of the four implementations needed it before it existed: flagd narrows floats to integers in two languages, and one Python scenario is marked as an expected failure against an SDK bug.", + "description": "A gap the provider is known to have against something the specification does not treat as optional. It is distinct from an undeclared capability, which is a choice, and from a not-applicable one, which is impossible -- this is a defect that is acknowledged rather than hidden. Three of the four implementations needed it before it existed: flagd narrows floats to integers in two languages, and one Python scenario is marked as an expected failure against an SDK bug. Appendix F states which of the two legitimate shapes to prefer: declaring the capability and letting the scenario fail, over withholding the capability so its scenarios skip.", "type": "object", "additionalProperties": false, + "required": [ + "summary" + ], "properties": { "capability": { "description": "The capability the deviation concerns, if it maps to one. Omitted when the deviation is against a mandatory scenario, which belongs to no capability.", @@ -218,11 +221,14 @@ "pattern": "^@[a-z0-9-]+$" }, "issue": { + "description": "Where the gap is tracked. Optional: an untracked deviation is still worth declaring, because naming the defect is what distinguishes it from a capability the provider chose to withhold. Prefer a tracked one as soon as there is somewhere to point at.", "type": "string", "format": "uri" }, "summary": { - "type": "string" + "description": "What the gap is, for someone comparing providers. Required, because a deviation with no summary records that something is wrong without saying what, leaving a reader worse off than the bare skip or failure it accompanies.", + "type": "string", + "minLength": 1 } } } From 61c7de22a1012a89803e6efbbc392a906d470c26 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sat, 12 Sep 2026 14:00:13 +0200 Subject: [PATCH 14/15] feat: require backend.controlApi, which decides what the rest of the report is worth The field said how the backend was driven and nothing required it, so four implementations arrived at four answers: a default method returning "in-process" in Java, an optional interface field in JavaScript, a comment-documented duck-typed property in Python, and in Go an unexported optional interface on the report side with a doc comment conceding that "silence here is a small lie by omission". Each was locally reasonable and together they mean a consumer cannot rely on the field being there. It is not a detail. The same scenarios passing over the normative control API and passing through in-process manipulation of a provider that has a real backend are not the same claim, and this is the only field that separates them. So it is required, and required to come from the control: nothing outside the control can tell which path a run used, and a harness that infers it from the control's concrete type is right about its own built-ins and quietly wrong about an adopter's custom control -- the one case where the answer matters. backend becomes required with it. It was documented as "omitted for a provider with no backend", which contradicted the controlApi enum, whose "in-process" member exists for exactly that provider: the one value most worth knowing could never legally appear. Every run manipulated flag state somehow, so every report can say how. Signed-off-by: Simon Schrottner --- .../provider-tck/report/conformance-report.schema.json | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/specification/assets/provider-tck/report/conformance-report.schema.json b/specification/assets/provider-tck/report/conformance-report.schema.json index 0f478467..970c42e2 100644 --- a/specification/assets/provider-tck/report/conformance-report.schema.json +++ b/specification/assets/provider-tck/report/conformance-report.schema.json @@ -10,6 +10,7 @@ "provider", "sdk", "tck", + "backend", "declaration", "results" ], @@ -108,15 +109,19 @@ } }, "backend": { - "description": "What the provider was pointed at. Omitted for a provider with no backend, which is itself informative.", + "description": "What the provider was pointed at, and how the suite drove it. Present for every run: a provider with no backend still had its flag state manipulated somehow, and which of the two ways that was is the single most important thing a reader needs in order to know what the results are worth.", "type": "object", "additionalProperties": false, + "required": [ + "controlApi" + ], "properties": { "description": { + "description": "A short description of what was being controlled, as the control itself reports it. Free text, for a person: a container image and tag, an in-memory store, a named environment.", "type": "string" }, "controlApi": { - "description": "How the backend was driven. 'http' means the normative control API; 'in-process' is the narrow allowance for providers with no backend, and a report claiming it for a provider that has one should be treated with suspicion.", + "description": "How the backend was driven. 'http' means the normative control API, the contract that makes a conformance claim portable between languages; 'in-process' is the narrow allowance for providers with no backend, where the flag state is a data structure in the test process. A report claiming 'in-process' for a provider that does have a backend should be treated with suspicion -- the path it exercised is not the path the contract describes. Required, and required to come from the control rather than be inferred: nothing outside the control can tell which of the two it was, so a harness that guesses will eventually guess wrong about a custom control, and an absent value is not neutral -- every run is one or the other, so silence here reads as a claim that cannot be checked.", "enum": [ "http", "in-process" From ec1d4edf8a846fe6b830157111ddff7534fb52c2 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sun, 13 Sep 2026 16:54:33 +0200 Subject: [PATCH 15/15] fix: the declaration no longer explains every skip, and said it did The declaration's description claimed that "given the declaration and a scenario's tags, the reason for a skip follows without needing to be transported per scenario", and then two sentences later conceded that an inexpressible capability's skip "carries the reason". Both cannot be true, and the flat claim came first, so a consumer would build on it. Appendix F now requires an implementation to refuse a capability its SDK cannot express rather than leaving adopters to withhold it, which makes the gap permanent and uniform: @large-integers is absent from every Java declaration and @numeric-coercion from every JavaScript one, in both cases for a reason that says nothing about the provider. A reader who infers "the provider declined" from an absent capability is now wrong in the direction that matters -- it reads as a deliberate limitation of a specific provider when it is a property of the language. So the description says what is actually derivable and what is not: most skips follow from the declaration, the inexpressible ones do not and cannot, Appendix F records which capabilities that applies to per language, and the results payload carries the distinction per skip. Found by the Java implementation while moving @large-integers into the new refusal. Signed-off-by: Simon Schrottner --- .../assets/provider-tck/report/conformance-report.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/assets/provider-tck/report/conformance-report.schema.json b/specification/assets/provider-tck/report/conformance-report.schema.json index 970c42e2..b1a1f6d5 100644 --- a/specification/assets/provider-tck/report/conformance-report.schema.json +++ b/specification/assets/provider-tck/report/conformance-report.schema.json @@ -130,7 +130,7 @@ } }, "declaration": { - "description": "The capability set this provider claims, as Gherkin tags including the leading at-sign. This is an INPUT to reading the results, not a summary of them, which is why it cannot be derived from the results payload and has to be stated here. A skipped scenario in the payload says the question was not put to this provider; only the declaration says whether that is because the provider declines the capability. Given the declaration and a scenario's tags, the reason for a skip follows without needing to be transported per scenario. A capability that cannot hold in a language at all -- @numeric-coercion where the language has a single numeric type, @large-integers on a 32-bit accessor -- is a property of the SDK rather than of the provider, and is recorded in Appendix F rather than restated in every report; in a run it is simply undeclared, and the skip carries the reason.", + "description": "The capability set this provider claims, as Gherkin tags including the leading at-sign. This is an INPUT to reading the results, not a summary of them, which is why it cannot be derived from the results payload and has to be stated here. A skipped scenario in the payload says the question was not put to this provider; the declaration is what says why. Given the declaration and a scenario's tags, most skips follow without needing a per-scenario explanation. NOT ALL OF THEM, and the exception is worth reading before building on this field: a capability may be absent because the provider declined it, or because the language's SDK cannot express it at all -- @numeric-coercion where the language has a single numeric type, @large-integers on a 32-bit accessor. Only the first says anything about the provider. The two are indistinguishable from the declaration alone, because an inexpressible capability is absent from every declaration in that language and no implementation permits declaring it. Appendix F records which capabilities that applies to per language, and the results payload carries the distinction per skip: a skip for an inexpressible capability says the SDK cannot ask the question, not that the provider declined. A consumer comparing providers across languages should read the skip reason rather than inferring from the declaration.", "type": "object", "additionalProperties": false, "required": [