diff --git a/tools/openfeature-provider-tck/.gitignore b/tools/openfeature-provider-tck/.gitignore index 06664622..04ba5649 100644 --- a/tools/openfeature-provider-tck/.gitignore +++ b/tools/openfeature-provider-tck/.gitignore @@ -5,3 +5,6 @@ src/openfeature/contrib/tools/provider_tck/features/ src/openfeature/contrib/tools/provider_tck/flag_data/ src/openfeature/contrib/tools/provider_tck/control-api.yaml +# Generated alongside them, from the submodule pin, so a conformance report can +# name the spec revision it ran against. +src/openfeature/contrib/tools/provider_tck/spec_revision.json diff --git a/tools/openfeature-provider-tck/README.md b/tools/openfeature-provider-tck/README.md index 040141f7..d15992d3 100644 --- a/tools/openfeature-provider-tck/README.md +++ b/tools/openfeature-provider-tck/README.md @@ -32,7 +32,7 @@ from pytest_bdd import scenarios from openfeature.contrib.tools.provider_tck import ( Capability, TckConfig, - features_path, + feature_paths, ) @@ -47,7 +47,7 @@ def tck_config(): ) -scenarios(features_path()) +scenarios(*feature_paths()) ``` There is **no `conftest.py` to write and nothing to import for the steps**. The step definitions @@ -70,6 +70,74 @@ different timescales — a streaming provider sees a configuration change in mil polls every 30 seconds may need most of a poll interval. Set it to comfortably exceed your worst-case detection latency, or the suite reports timeouts that are really just impatience. +## Adding your own scenarios + +A provider is rarely only a provider. flagd has `fractional` targeting, another vendor has a +proprietary rollout rule, and the behaviour of those is as worth pinning as the contract they sit on +top of. Verifying them used to mean a second harness: a second backend lifecycle, a second set of +fixtures, a second thing to keep working. + +Put them in the same run instead. Create a directory named `tck-extensions` beside the module that +calls `scenarios()`, and write step definitions for whatever is new in a `conftest.py` beside it: + +``` +tests/ +├── conftest.py # your step definitions +├── test_conformance.py # the fixture and the one call, unchanged +└── tck-extensions/ + └── fractional.feature +``` + +```python +# conftest.py +from pytest_bdd import then + +from openfeature.contrib.tools.provider_tck import TckState + + +@then("the fractional rule splits the population") +def fractional_splits(tck_state: TckState) -> None: + ... +``` + +That is the whole of it — **no registration, no option and no new argument**. pytest collects +`conftest.py` on its own, pytest-bdd resolves steps through the fixture system, and the canonical +step vocabulary is in scope in your feature file beside your own steps. `tck_state` is the same +per-scenario state the canonical steps use, so your scenario runs against the provider the suite +registered, in the same backend lifecycle, with the same reset between scenarios. + +The one thing pytest cannot find by itself is the feature files, because the canonical ones are +inside the installed distribution rather than in your repository. `feature_paths()` returns both: + +```python +scenarios(*feature_paths()) +``` + +That line does not change when you add an extension, and it is the only difference from the older +`scenarios(features_path())` — which still works and still sees only the canonical set. An adopter +with no `tck-extensions` directory runs exactly what they ran before: same scenarios, same count, +same report. + +### Your scenarios cannot stand in for ours + +In the report, canonical scenarios are the ones under the `features/` uri prefix and yours are under +`extensions/` — the prefix Go and JavaScript mount theirs under too, so a consumer holding reports +from several languages applies one rule. The prefix is derived from where a file *is*, not from what +the runner called it, and two cases are refused outright rather than documented: + +- **A feature file of yours under the reserved `features/` prefix.** Handing `scenarios()` a + directory of your own named `features` is the one route left to a canonical-looking uri. No report + is written and the run fails. +- **Two feature files that would share one uri.** A Cucumber Messages stream carries one source per + uri, so the second file's scenarios would be reported against the first file's source. + +This is not hypothetical. Java's suite found that a same-named feature file in a second classpath +root *replaced* the canonical one, and the run went green having asked the adopter's questions +instead of the specification's — the worst outcome available to a conformance suite. The Python +route to the same place is narrower and just as quiet: pytest-bdd names a feature file by its parent +directory joined to its own name, so `tck-extensions/features/errors.feature` arrives under the uri +the canonical `errors.feature` already occupies. + ## Capabilities Not every provider implements every optional part of the contract. Each scenario exercising an @@ -94,8 +162,8 @@ SKIPPED provider does not declare capability @stale. | `Capability.OBJECT` | `@object` | supports structured flag values | | `Capability.UNAVAILABLE_INIT` | `@unavailable` | reports an error state instead of hanging against a dead backend | | `Capability.NUMERIC_COERCION` | `@numeric-coercion` | coerces between integer and float only when lossless, else `TYPE_MISMATCH` | -| `Capability.TARGETING` | `@targeting` | reserved; no scenarios yet | -| `Capability.CACHING` | `@caching` | reserved; no scenarios yet | +| `Capability.TARGETING` | `@targeting` | reserved; **not declarable** — no scenarios yet | +| `Capability.CACHING` | `@caching` | reserved; **not declarable** — no scenarios yet | `@lifecycle` and `@events` are deliberately separate, and the split matters in both directions. An SDK dispatches `PROVIDER_READY` around `initialize` for *any* provider, so a provider declaring only @@ -104,9 +172,18 @@ identically. Meanwhile a stateless provider has a real initialisation to verify of its own to declare `@events` for, and gating on `@events` shut it out of a scenario it should be held to. -Untagged scenarios are mandatory and always run. `capabilities` defaults to everything — narrow it -rather than widening it: start from the default, run the suite, and remove only what your provider -genuinely cannot do. +Untagged scenarios are mandatory and always run. `capabilities` defaults to every *declarable* +capability — `DECLARABLE_CAPABILITIES` — and you should narrow it rather than widen it: start from +the default, run the suite, and remove only what your provider genuinely cannot do. + +A reserved capability is documented so the vocabulary has a place for it once scenarios exist, and +until then it **must not be declared**. Nothing carries the tag, so declaring it cannot be verified, +cannot produce a skip, and tells a reader of a conformance report only that something was claimed and +nothing examined. `TckConfig` raises if you name one in `capabilities` or in `not_applicable`, and +`DECLARABLE_CAPABILITIES` excludes them — which is the case that matters, because "every capability +except X" is how a reserved tag reaches a report by accident rather than by decision. One +implementation's published report asserts `@targeting` and `@caching` as declared for exactly that +reason. `@numeric-coercion` deserves a note, because it is the one capability here that **the specification does not define**. OpenFeature has a single numeric type on purpose — `number` is "a numeric value of @@ -128,6 +205,11 @@ of, so a provider that wrongly rejects `10.0` as an integer still passes; adding set for every language at once. Appendix F records that as an open gap, together with a second one: the width of a language's integer accessor — 64-bit against 32-bit — is not modelled at all. +For a capability that *cannot* hold rather than one you chose not to declare, use +`not_applicable={Capability.X: "why"}`. The suite treats it identically — the scenarios are skipped +either way — but the report keeps the two apart, because collapsing them misrepresents a provider: +declining an optional feature is a choice, and an impossibility is not. + ## Controlling the backend `BackendControl` is the single seam between the scenarios and whatever manipulates the backend. Step @@ -177,8 +259,10 @@ This is **Python-specific** — the identical scenario passes in every other lan is a fair advertisement for having more than one implementation. Tracked as [open-feature/python-sdk#619](https://github.com/open-feature/python-sdk/issues/619). -The self-test marks that one row `xfail(strict=True)` with a pointer to the issue, so it stays -visible in the report and un-hides itself automatically once the SDK is fixed. +The self-test marks that one row `xfail(strict=True)` with a pointer to the issue, so the run +un-hides itself automatically once the SDK is fixed, and declares it in +`TckConfig.known_deviations`, so the report acknowledges it. The results payload still reports the +scenario as `FAILED`: the acknowledgement records the gap, it does not soften it. ### 2. The in-memory provider cannot update its flag set @@ -218,6 +302,131 @@ is recorded by the pin and nowhere else, so the two cannot drift apart unnoticed This mirrors what `openfeature-flagd-api-testkit` already does for the flagd test harness. +## Conformance reports + +Set `PROVIDER_TCK_REPORT_DIR` and each suite writes **two** files: an envelope at `/.json`, +conforming to the [report schema][report-schema] in the specification, and the results it points at +at `/.ndjson`, which is a [Cucumber Messages][messages] stream. + +```console +$ PROVIDER_TCK_REPORT_DIR=./reports pytest +provider-tck [in-memory]: report written to reports/in-memory.json with results in in-memory.ndjson (1 failed, 23 passed, 5 skipped) + +$ jq -c .results reports/in-memory.json +{"format":"cucumber-messages","location":"in-memory.ndjson","digest":"sha256:c7e12a…"} + +$ jq -r 'select(.testStepFinished) | .testStepFinished.testStepResult.status' \ + reports/in-memory.ndjson | sort | uniq -c + 1 FAILED + 220 PASSED + 45 SKIPPED +``` + +Statuses are per step, not per scenario. Of the 45 skipped, 42 belong to the five scenarios the +capability gate stopped — their before-hooks included, which is where the reason is — and three are +the steps of the failing scenario that were never reached. + +It is an environment variable rather than a `TckConfig` field so that emitting a report is a property +of the *run* and not of the code: CI sets it, a developer running the suite locally does not, and no +adopter changes a line to publish one. Unset means no report, which is not an error. Several suites +in one pytest session each write their own pair, so flagd's two resolvers would not collide. + +### A partial run is not a conformance run + +The canonical scenario set is fixed by the specification, and a run that executed less of it cannot +support a conformance claim. `-k`, `-m`, `--deselect`, or a test module that stopped calling +`scenarios()` on the canonical path each run fewer scenarios, and none of them is an error to +pytest. Go measured the consequence: `-run` on a single scenario passed green and emitted a +well-formed report covering 1 of 29 canonical scenarios, with nothing in the document saying so. + +So every run is checked against the scenarios this distribution ships, and a suite that did not +execute all of them writes no report: + +```console +$ PROVIDER_TCK_REPORT_DIR=./reports pytest -k "unknown_flag_key" +provider-tck [in-memory]: 28 of 29 canonical scenarios did not run, so this run cannot support a +conformance claim and no report is written for it. … + - features/errors.feature: A float flag is not silently narrowed to an integer + - features/errors.feature: Requesting the wrong type returns the code default [key=float-flag requested=Boolean default=false] + … and 18 more +``` + +A scenario the capability gate skipped **has** run: it was asked, and the report accounts for it +with its reason, so declining a capability never trips this. Your own scenarios are yours — they are +not counted towards the canonical set and cannot close a gap in it. + +Set `PROVIDER_TCK_PARTIAL=1` to work on a single scenario without the guard failing the run. It buys +a green run and nothing else: no report is written for an incomplete suite either way. Java's TCK +spells the same escape hatch the same way. + +### Why the results are not our format + +Per-scenario outcomes, tags, Scenario Outline row identity and the executed feature source are all +already specified by Cucumber Messages, which is maintained, cross-language, schema'd, and emitted +natively by cucumber-jvm. Defining them again in the report schema created a second format to +maintain and version, and two places for the same fact to disagree. So the envelope says what was +tested and what the provider claims; the payload says what happened. + +The results are referenced rather than inlined because the stream carries the feature sources and is +far larger than the envelope, and a consumer deciding whether it cares about a report should not have +to fetch a whole run to find out. `results.digest` is a SHA-256 over the exact bytes written, so a +consumer can tell that what it fetched is what the envelope described. + +Two things Messages cannot carry, so they stay in the envelope. `declaration` is an *input* to +reading the results rather than a summary of them: a skipped scenario says the question was not put +to this provider, and only the declaration says whether that is because the provider declines the +capability. And no standard results format has a slot for the tested subject — Messages records the +runtime and the OS, not what was being asked about. + +### Reading the payload + +Appendix F requires that a scenario skipped for an undeclared capability is reported as skipped +**with the reason** and never as passed. A consumer cannot check that against a summary line, so the +stream carries every scenario the run collected, including the ones the capability gate skipped +before their first step, and Cucumber's own `SKIPPED` is what it reports them as. + +Each scenario is a `TestCase` referring to a `Pickle`, and a test case is as bad as its worst step, +which is Cucumber's rule. Every test case carries two hook steps as well as its Gherkin steps: pytest +runs a scenario in three phases and only the middle one executes steps, so the before-hook is where a +capability skip's reason lands and the after-hook is where a teardown failure does. + +Given a scenario's tags — in its pickle — and the envelope's `declaration`, the capability +responsible for a skip follows, which is why it is no longer transported once per scenario. + +Which also means the payload is not a transcription of pytest's summary. The run above finishes +green: the one scenario the Python SDK cannot satisfy is marked `xfail` (finding 1), so pytest counts +it as expected and exits zero. The provider still did not satisfy it, and the stream says `FAILED`. +The acknowledgement goes in the envelope's `knownDeviations` instead — an expected failure is a +recorded deviation, not an excused one — which an adoption declares with `TckConfig.known_deviations`. + +### Which row of a Scenario Outline + +A pickle's `astNodeIds` are `[scenario id, table row id]`, and the row id resolves in the +`GherkinDocument` to exactly the cells the feature file wrote. That is what tells the eleven rows of +the type-mismatch matrix apart — one of which differs in outcome from its ten siblings — and it is +exact rather than a naming convention every implementation has to reproduce byte-for-byte. + +### What identifies a report + +`tck.specRevision` comes from `spec_revision.json`, which `hatch_build_sync.py` generates from the +submodule alongside the copied assets. It has to be captured at build time: the submodule is not in +the wheel, so an installed copy has nothing left to ask. A build that cannot reach git — an unpacked +sdist, say — warns and records `unknown` rather than inventing a commit. + +No asset tree hash. It was carried so a consumer could tell whether two runs executed the same +questions; the payload's `Source` messages carry the executed feature files verbatim, which answers +that directly rather than by proxy. + +`provider.name` is what the provider reports through its own metadata, not `TckConfig.name`. +`TckConfig.name` is chosen to read well in a failure message — `flagd-rpc` — which makes it the +*configuration*, and it is reported as such. One provider with two materially different modes +produces two reports that are not interchangeable. + +`backend.controlApi` is read off an optional `control_api` property on your `BackendControl`, +returning `"http"` or `"in-process"`. It is not a member of the protocol: adding one would make every +existing control incomplete for the sake of one string, and a control that stays quiet simply omits +the field. + ## The self-tests | Suite | Subject | Why | @@ -225,12 +434,18 @@ This mirrors what `openfeature-flagd-api-testkit` already does for the flagd tes | `test_in_memory_conformance` | the SDK's `InMemoryProvider` | reference adoption for a backend-less provider | | `test_controllable_conformance` | `ControllableInMemoryProvider` | the only suite that exercises the configuration-change path — see finding 2 | | `test_in_process_control` | `InProcessControl` | pins what the Gherkin cannot assert about itself | +| `test_report` | the conformance report | checks the two properties a consumer is entitled to assume, against the emitted Messages stream | +| `test_extensions` | an adopter's own scenarios | an extension runs inside the canonical suite, changes nothing for an adopter who has none, and cannot stand in for a canonical scenario | +| `test_canonical_set` | the canonical-set guard | a run that executed less than the canonical set fails and publishes nothing | ``` -54 passed, 9 skipped, 2 xfailed +132 passed, 9 skipped, 2 xfailed ``` -No Docker, no network, under a second. +No Docker and no network. The conformance suites take under a second; `test_report`, +`test_extensions` and `test_canonical_set` take most of the time, because the properties they check +are properties of a whole pytest session and they run generated adoptions in subprocesses to check +them. Neither in-memory suite declares `@lifecycle`, so the three lifecycle scenarios are skipped in both. That is the point: with no backend to reach, they would pass without testing anything — which is @@ -242,7 +457,15 @@ what they did while the feature was gated on `@events`. cannot assert one *reached* the backend. That needs an echo operation on the control API. - **No HTTP control client yet.** It arrives with the first containerised adopter. - **Caching, hooks and flag metadata** are not covered. - +- **The results payload is assembled here.** pytest-bdd emits no Cucumber Messages — it ships the + legacy Cucumber JSON format and nothing for the ndjson protocol — so `messages.py` builds the + stream from the official types and re-parses the feature files to get the AST node ids a pickle + refers to. If pytest-bdd ever emits Messages itself, that module should shrink to a shim. Whether + a report belongs inside a provider's released artifact is still open on + [open-feature/spec#424](https://github.com/open-feature/spec/issues/424). + +[report-schema]: https://github.com/open-feature/spec/blob/main/specification/assets/provider-tck/report/conformance-report.schema.json +[messages]: https://github.com/cucumber/messages [appendix-a]: https://github.com/open-feature/spec/blob/main/specification/appendix-a-included-utilities.md [appendix-f]: https://github.com/open-feature/spec/blob/main/specification/appendix-f-provider-conformance.md [spec]: https://github.com/open-feature/spec diff --git a/tools/openfeature-provider-tck/hatch_build.py b/tools/openfeature-provider-tck/hatch_build.py index 4b6f1d84..3e187984 100644 --- a/tools/openfeature-provider-tck/hatch_build.py +++ b/tools/openfeature-provider-tck/hatch_build.py @@ -18,7 +18,14 @@ # the single definition of what gets copied where -- would not be importable. sys.path.insert(0, str(Path(__file__).parent)) -from hatch_build_sync import FILES, PACKAGE_REL, SPEC_ASSETS, TREES, sync +from hatch_build_sync import ( + FILES, + PACKAGE_REL, + REVISION_FILE, + SPEC_ASSETS, + TREES, + sync, +) class SpecAssetsCopyHook(BuildHookInterface): @@ -26,7 +33,13 @@ class SpecAssetsCopyHook(BuildHookInterface): def initialize(self, version: str, build_data: dict) -> None: root = Path(self.root) - copies = [root / PACKAGE_REL / dest for _, dest in TREES + FILES] + # The generated revision file travels with the assets it describes. It + # has to be built here rather than read at run time, because the + # submodule that knows the answer is not in the wheel and a conformance + # report has to name the revision it ran against. + copies = [root / PACKAGE_REL / dest for _, dest in TREES + FILES] + [ + root / PACKAGE_REL / REVISION_FILE + ] # Building from a checkout: refresh from the submodule, so what ships is # always the revision the pin names. Building from an sdist: there is no diff --git a/tools/openfeature-provider-tck/hatch_build_sync.py b/tools/openfeature-provider-tck/hatch_build_sync.py index f31bc55b..8e0a5270 100644 --- a/tools/openfeature-provider-tck/hatch_build_sync.py +++ b/tools/openfeature-provider-tck/hatch_build_sync.py @@ -10,11 +10,16 @@ needs no submodule: the copies are inside the distribution. """ +import json import shutil +import subprocess +import warnings from pathlib import Path ROOT = Path(__file__).parent -SPEC_ASSETS = (ROOT / "spec/specification/assets/provider-tck").resolve() +SPEC_ROOT = (ROOT / "spec").resolve() +ASSETS_PATH_IN_SPEC = "specification/assets/provider-tck" +SPEC_ASSETS = (SPEC_ROOT / ASSETS_PATH_IN_SPEC).resolve() PACKAGE_REL = Path("src/openfeature/contrib/tools/provider_tck") DEST_BASE = ROOT / PACKAGE_REL @@ -28,6 +33,27 @@ TREES = [("gherkin", "features"), ("flags", "flag_data")] FILES = [("openapi/control-api.yaml", "control-api.yaml")] +REVISION_FILE = "spec_revision.json" +"""Which revision of the specification the copied assets came from. + +Recorded at build time because the answer is only available at build time: the +submodule that holds it is not in the wheel, and a conformance report that cannot +name the revision it ran against cannot be compared with another. It is generated +by the same command that copies the assets, which is what keeps the two from +disagreeing. + +Not committed, for the same reason the assets are not: the submodule pin is the +single record of which revision this package targets. +""" + +UNKNOWN_REVISION = "unknown" +"""Seven characters, the minimum the report schema accepts. + +A build that cannot reach git says it does not know rather than inventing a +commit, and still produces a document that validates. Which happens for real: +building from a source tarball has no ``.git`` to ask. +""" + def sync() -> None: if not SPEC_ASSETS.exists(): @@ -51,6 +77,48 @@ def sync() -> None: dest.unlink() shutil.copy2(SPEC_ASSETS / src_name, dest) + write_revision() + + +def write_revision() -> None: + """Record the spec commit these copies came from. + + The asset tree hash that used to accompany it is gone. It was carried so a + consumer could tell whether two runs executed the same questions; the + conformance report's results are now a Cucumber Messages stream, which + carries the executed feature source itself and answers that directly rather + than by proxy. + """ + commit = _git("rev-parse", "HEAD") or UNKNOWN_REVISION + (DEST_BASE / REVISION_FILE).write_text( + json.dumps({"specRevision": commit}, indent=2) + "\n", + encoding="utf-8", + ) + + +def _git(*args: str) -> str: + """Run git inside the submodule, returning its output or an empty string. + + A build must not hard-fail because git is absent or the checkout is not a + repository -- both are ordinary when building from an unpacked sdist. The + failure is reported as a warning and the identity degrades to ``unknown``, + which is legible in the resulting report rather than silently wrong. + """ + command = ["git", "-C", str(SPEC_ROOT), *args] + try: + completed = subprocess.run( # noqa: S603 + command, capture_output=True, check=True, text=True + ) + except (OSError, subprocess.CalledProcessError) as error: + warnings.warn( + f"could not determine the spec revision ({' '.join(command)}: {error}); " + f"conformance reports from this build will not name the revision they " + f"ran against", + stacklevel=2, + ) + return "" + return completed.stdout.strip() + if __name__ == "__main__": sync() diff --git a/tools/openfeature-provider-tck/pyproject.toml b/tools/openfeature-provider-tck/pyproject.toml index ff0cbe43..d8b98dc1 100644 --- a/tools/openfeature-provider-tck/pyproject.toml +++ b/tools/openfeature-provider-tck/pyproject.toml @@ -22,6 +22,21 @@ dependencies = [ # Same runner the flagd provider and the flagd testkit already use, so an # adopting module gains no new test framework. "pytest-bdd>=8.1.0,<9.0.0", + # The conformance report's results are a Cucumber Messages stream rather + # than a format this package defines. These two are the reference + # implementations of the halves of that protocol: cucumber-messages is the + # official Python types, published from the same repository as the protocol + # itself, and gherkin-official is the parser that produces the + # gherkinDocument and pickle messages. pytest-bdd already depends on + # gherkin-official, so only the first is genuinely new -- and it has no + # dependencies of its own. + # + # pytest-bdd ships no Messages emitter (its cucumber_json.py is the legacy + # JSON format), so the stream is assembled here; assembling it from typed + # messages rather than hand-written dicts is what keeps it from drifting + # away from the protocol. + "cucumber-messages>=34.0.0,<35.0.0", + "gherkin-official>=29.0.0", ] requires-python = ">=3.10" @@ -58,6 +73,10 @@ artifacts = [ "src/openfeature/contrib/tools/provider_tck/features/", "src/openfeature/contrib/tools/provider_tck/flag_data/", "src/openfeature/contrib/tools/provider_tck/control-api.yaml", + # Which spec revision those assets came from, generated beside them. The + # submodule is not in the wheel, so a conformance report emitted by an + # installed copy has no other way to name the revision it ran against. + "src/openfeature/contrib/tools/provider_tck/spec_revision.json", ] [tool.hatch.build.hooks.custom] @@ -74,6 +93,21 @@ fixed_format_cache = true pretty = true strict = true disallow_any_generics = false +# cucumber-messages and gherkin-official ship no py.typed. Both are annotated +# internally, so following them gives real types for the messages this package +# builds rather than the Any a plain `ignore_missing_imports` would hand back -- +# which is the point of using the typed library at all. +follow_untyped_imports = true + +[[tool.mypy.overrides]] +# gherkin-official has no annotations at all, so following it turns every call +# into a `no-untyped-call` error rather than into a type. pytest-bdd silences +# the same import the same way. cucumber-messages is the opposite case -- fully +# annotated, only missing py.typed -- and is followed, which is where the value +# of using it rather than hand-written dicts actually lands. +module = ["gherkin.*"] +follow_untyped_imports = false +ignore_missing_imports = true [tool.coverage.run] omit = ["tests/**"] diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py index 8b615296..299e07fb 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py @@ -16,7 +16,7 @@ Capability, InProcessControl, TckConfig, - features_path, + feature_paths, ) @pytest.fixture(scope="session") @@ -29,11 +29,14 @@ def tck_config(): capabilities={Capability.EVENTS, Capability.OBJECT}, ) - scenarios(features_path()) + scenarios(*feature_paths()) ``scenarios()`` is pytest-bdd's own, called directly rather than wrapped: it injects the generated tests into the *calling module* by walking the stack, so a convenience wrapper around it would deposit them inside this package instead. +:func:`~.extensions.feature_paths` is the canonical assets plus a +``tck-extensions`` directory beside the calling module, if there is one -- see +:mod:`~.extensions`. The step definitions arrive through this package's pytest plugin, so there is nothing to import for them and no ``conftest.py`` to write. Everything else -- @@ -49,33 +52,51 @@ def tck_config(): import importlib.resources -from .capability import ALL_CAPABILITIES, Capability -from .config import TckConfig +from .canonical import PARTIAL_ENV +from .capability import DECLARABLE_CAPABILITIES, RESERVED_CAPABILITIES, Capability +from .config import KnownDeviation, TckConfig from .control import ( BackendControl, ConnectionControl, UnsupportedControlError, ) +from .extensions import ( + EXTENSIONS_DIRECTORY, + feature_paths, + features_path, +) from .inprocess import InProcessControl +from .messages import MESSAGES_FORMAT from .provider import ( CHANGING_FLAG_KEY, ControllableInMemoryProvider, canonical_flag_set, ) +from .report import REPORT_DIR_ENV, SCHEMA_VERSION +from .state import TckState __all__ = [ - "ALL_CAPABILITIES", "CHANGING_FLAG_KEY", + "DECLARABLE_CAPABILITIES", + "EXTENSIONS_DIRECTORY", + "MESSAGES_FORMAT", + "PARTIAL_ENV", + "REPORT_DIR_ENV", + "RESERVED_CAPABILITIES", + "SCHEMA_VERSION", "BackendControl", "Capability", "ConnectionControl", "ControllableInMemoryProvider", "InProcessControl", + "KnownDeviation", "TckConfig", + "TckState", "UnsupportedControlError", "canonical_flag_set", "canonical_flags_json", "control_api_spec", + "feature_paths", "features_path", ] @@ -101,21 +122,6 @@ def tck_config(): _PACKAGE = "openfeature.contrib.tools.provider_tck" -def features_path() -> str: - """Return the directory holding the canonical feature files. - - Packaged with this distribution, so a consumer needs no submodule and no - particular directory layout. Hand it to pytest-bdd's ``scenarios()``, which - accepts an absolute path:: - - scenarios(features_path()) - - pytest-bdd generates one test per scenario -- and one per row of a Scenario - Outline -- so failures name a scenario and ``-k`` selects one as usual. - """ - return str(importlib.resources.files(_PACKAGE) / "features") - - def canonical_flags_json() -> str: """Return the canonical flag set as raw JSON, in the flagd flag-definition format. diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/canonical.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/canonical.py new file mode 100644 index 00000000..0009b424 --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/canonical.py @@ -0,0 +1,116 @@ +"""The canonical scenario set, and whether a run actually executed it. + +A conformance suite that goes green on scenarios it did not run is worse than no +suite. The capability gate already rules out the loud version of that -- an +undeclared capability is reported as skipped, with its reason, never as passed -- +but it says nothing about the quiet version, where the scenarios were never asked +for in the first place. A ``-k`` expression, a ``-m`` filter, a ``--deselect``, a +test module that stopped calling ``scenarios()`` on the canonical path: each of +those runs less of the suite and none of them is an error. The Go implementation +measured it. ``-run`` on a single scenario passed green and emitted a well-formed +report covering one of twenty-nine canonical scenarios, and nothing in the +document said so. + +So the run is checked against what this distribution *ships* rather than against +what it was asked to run. The expectation is compiled from the packaged feature +files with the same Gherkin compiler that produces the results payload, which +means it is one entry per Scenario Outline row -- the granularity the runner +generates and therefore the only one a comparison can be made at. + +Two properties this has to have, and both are about what may close a gap. + +**A skip counts; an absence does not.** A scenario the capability gate skipped +did run: it was asked, and the report accounts for it with a reason. A scenario +that was never collected is missing, and no declaration makes it otherwise. + +**An extension cannot close a gap.** An adopter's own scenarios are matched by +neither uri nor path against the packaged set, and the executed side of the +comparison is filtered to files that are genuinely inside this distribution -- +not merely to files reported under the canonical prefix, so that the check does +not rest on the same derivation it is meant to corroborate. +""" + +from __future__ import annotations + +import functools +import os +import typing + +from .extensions import canonical_root, is_canonical, uri_for +from .messages import FeatureCatalog, ScenarioKey, ScenarioRun + +__all__ = [ + "PARTIAL_ENV", + "canonical_scenarios", + "describe", + "missing_canonical", + "partial_run_allowed", +] + +PARTIAL_ENV = "PROVIDER_TCK_PARTIAL" +"""Set to acknowledge that a run is deliberately not a conformance run. + +For working on one scenario with ``-k`` without the guard failing the run. It +never makes a partial run publishable: no report is written for a suite that did +not execute the canonical set, with or without it. The Java TCK spells the same +escape hatch the same way, so the two are one thing to know rather than two. +""" + +_TRUTHY = {"1", "true", "yes", "on"} + + +def partial_run_allowed(environment: typing.Mapping[str, str] | None = None) -> bool: + """Whether the run has declared itself partial.""" + source = os.environ if environment is None else environment + return source.get(PARTIAL_ENV, "").strip().lower() in _TRUTHY + + +@functools.cache +def canonical_scenarios() -> frozenset[ScenarioKey]: + """Every scenario the packaged feature files define, row by row. + + Empty when the assets are not reachable as files -- an installation from a + zipimport, say. Everything built on this then degrades to "cannot tell", + which is the honest answer and never a false accusation. + + Cached because it is the same answer for the whole process and parsing it is + the same work the results payload already does. + """ + root = canonical_root() + if root is None or not root.is_dir(): + return frozenset() + + catalog = FeatureCatalog() + for path in sorted(root.rglob("*.feature")): + uri = uri_for(path) + if uri is not None: + catalog.load_file(uri, path) + return catalog.scenario_keys + + +def missing_canonical(runs: typing.Iterable[ScenarioRun]) -> tuple[ScenarioKey, ...]: + """The canonical scenarios this suite did not execute, in reporting order. + + ``runs`` is everything the suite accounted for, extensions included; only + the ones whose feature file is genuinely one of the packaged assets are + counted, so an adopter's scenario can neither fill a gap nor be blamed for + one. + """ + expected = canonical_scenarios() + if not expected: + return () + executed = { + (run.identity.uri, run.identity.name, run.identity.example) + for run in runs + if is_canonical(run.identity.path) + } + return tuple(sorted(expected - executed)) + + +def describe(key: ScenarioKey) -> str: + """One missing scenario, named the way a failure message should name it.""" + uri, name, row = key + if not row: + return f"{uri}: {name}" + cells = " ".join(f"{header}={cell}" for header, cell in row) + return f"{uri}: {name} [{cells}]" diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py index 6021b461..8be2b240 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py @@ -97,29 +97,62 @@ class Capability(str, Enum): """ TARGETING = "targeting" - """Reserved. No scenario carries this tag: targeting is backend evaluation logic.""" + """Reserved, and **not declarable**. No scenario carries this tag: targeting + is backend evaluation logic.""" CACHING = "caching" - """Reserved; no scenario carries this tag yet.""" + """Reserved, and **not declarable**. No scenario carries this tag yet.""" @property def tag(self) -> str: """Return the Gherkin tag, with its leading at-sign, that gates this capability.""" return f"@{self.value}" + @property + def reserved(self) -> bool: + """Whether this capability exists in the vocabulary but gates no scenario.""" + return self in RESERVED_CAPABILITIES + def __str__(self) -> str: return self.tag -ALL_CAPABILITIES: frozenset[Capability] = frozenset(Capability) -"""Every capability the TCK recognises. +RESERVED_CAPABILITIES: frozenset[Capability] = frozenset( + {Capability.TARGETING, Capability.CACHING} +) +"""Capabilities that exist in the vocabulary and gate no scenario. + +They are documented so the vocabulary has a place for them when scenarios exist, +and until then they **must not be declared** and must not appear in a conformance +report's declaration. Nothing carries the tag, so declaring it cannot be +verified, cannot produce a skip, and tells a reader of the report only that +something was claimed and nothing examined. + +Listed once, here, and read everywhere else -- by +:data:`DECLARABLE_CAPABILITIES`, by :attr:`Capability.reserved` and by the +validation in :class:`~.config.TckConfig` -- so that the set and the rule cannot +drift apart. +""" + +DECLARABLE_CAPABILITIES: frozenset[Capability] = ( + frozenset(Capability) - RESERVED_CAPABILITIES +) +"""Every capability an adoption may declare: the vocabulary minus the reserved tags. A reasonable starting point for a new adoption: declare everything, run the -suite, and remove only what the provider genuinely cannot do. Narrowing from the -full set surfaces gaps; widening towards it hides them. +suite, and remove only what the provider genuinely cannot do. Narrowing from this +set surfaces gaps; widening towards it hides them. + +It excludes the reserved capabilities rather than spanning the whole enum, and it +is named for what it is rather than for "all", because the declare-everything +convenience is exactly how a reserved tag reaches a report by accident: an +adopter writing "every capability except X" picks up every reserved tag on the +way past, which is how one implementation came to report ``@targeting`` and +``@caching`` as declared without anyone deciding to claim them. """ _BY_MARKER: dict[str, Capability] = {c.value: c for c in Capability} +_BY_TAG: dict[str, Capability] = {c.tag: c for c in Capability} def capability_for_marker(name: str) -> Capability | None: @@ -129,3 +162,14 @@ def capability_for_marker(name: str) -> Capability | None: the canonical feature files carry organisational tags freely. """ return _BY_MARKER.get(name) + + +def capability_for_tag(tag: str) -> Capability | None: + """Map a Gherkin tag, leading at-sign included, onto the capability it gates. + + The tag form rather than the marker form because that is what the + conformance report carries: the report records a scenario's tags as the + feature files spell them, and deciding whether a failure counts against a + capability means reading them back. + """ + return _BY_TAG.get(tag) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py index 77b783cd..38470266 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py @@ -2,15 +2,16 @@ from __future__ import annotations -from collections.abc import Callable, Collection, Iterable +import typing +from collections.abc import Callable, Collection, Iterable, Mapping, Sequence from dataclasses import dataclass, field from openfeature.provider import FeatureProvider -from .capability import ALL_CAPABILITIES, Capability +from .capability import DECLARABLE_CAPABILITIES, Capability from .control import BackendControl -__all__ = ["ProviderFactory", "TckConfig"] +__all__ = ["KnownDeviation", "ProviderFactory", "TckConfig"] ProviderFactory = Callable[[], FeatureProvider] """Creates the provider under test. @@ -24,6 +25,46 @@ DEFAULT_READY_TIMEOUT = 30.0 +@dataclass(frozen=True) +class KnownDeviation: + """A gap the provider is known to have, acknowledged rather than hidden. + + Distinct from an undeclared capability, which is a choice, and from a + not-applicable one, which is impossible: this is a defect against something + the specification does not treat as optional, with the gap tracked + somewhere. + + It changes nothing about how the suite runs. The scenario still fails, and + the results payload still reports it as failed -- a report that softened a + failure into a footnote would hide exactly what the acknowledgement exists to + keep visible. What this adds is the acknowledgement itself, in the envelope, + so that a consumer can tell a known and tracked gap from a surprise. + """ + + issue: str + """Where the gap is tracked. A URI, because the schema requires one.""" + + summary: str + """What is wrong, for a person reading a comparison page.""" + + capability: Capability | None = None + """The capability the deviation concerns, when it maps to one. + + Left out for a deviation against a mandatory scenario, which belongs to no + capability -- which is the common case, since a capability a provider fails + is usually one it should not have declared. + """ + + def as_json(self) -> dict[str, typing.Any]: + document: dict[str, typing.Any] = { + "issue": self.issue, + "summary": self.summary, + } + if self.capability is not None: + document["capability"] = self.capability.tag + return document + + @dataclass(frozen=True) class TckConfig: """Everything the TCK needs to test one provider. @@ -79,7 +120,7 @@ class TckConfig: skipped with the reason reported. """ - capabilities: Collection[Capability] = field(default=ALL_CAPABILITIES) + capabilities: Collection[Capability] = field(default=DECLARABLE_CAPABILITIES) """Which optional parts of the provider contract this provider supports. Typed as a ``Collection`` rather than a ``frozenset`` so that the obvious @@ -88,8 +129,39 @@ class TckConfig: construction, so a list, a set or a generator all behave identically. Scenarios tagged with an undeclared capability are reported as skipped with - the reason, never as passed. Defaults to everything; narrow it rather than - widening it. + the reason, never as passed. Defaults to every *declarable* capability -- + :data:`~.capability.DECLARABLE_CAPABILITIES`, which excludes the reserved + tags no scenario carries -- and narrowing it surfaces gaps where widening + towards it hides them. + + Naming a reserved capability here is rejected at construction rather than + passed into a report. See :data:`~.capability.RESERVED_CAPABILITIES`. + """ + + not_applicable: Mapping[Capability, str] = field(default_factory=dict) + """Capabilities that cannot hold for this provider, each with a reason. + + Kept apart from simply leaving a capability out of :attr:`capabilities`, + because the two are different claims and collapsing them misrepresents whole + languages: ``@numeric-coercion`` is unsatisfiable in JavaScript because + the language has no integer type, and reporting that as a choice would show + every JavaScript provider as missing something none of them can have. + + Scenarios behind a not-applicable capability are skipped exactly as an + undeclared one's are -- the gate makes no distinction, and neither does the + results payload. The difference is recorded once, here, and reaches the + report's declaration. + + Where the impossibility is a property of the language rather than of the + provider it belongs in the capability documentation rather than in every + report, so this is for provider-specific cases. + """ + + known_deviations: Sequence[KnownDeviation] = () + """Gaps this provider is known to have, with each one tracked somewhere. + + An acknowledgement, not an excuse: the scenarios still fail and the results + payload still says so. See :class:`KnownDeviation`. """ event_timeout: float = DEFAULT_EVENT_TIMEOUT @@ -138,6 +210,47 @@ def __post_init__(self) -> None: f"the Capability enum" ) + # Normalised the same way, so a dict literal keyed by Capability is what + # an adopter writes and a plain mapping is what everything else reads. + object.__setattr__(self, "not_applicable", dict(self.not_applicable)) + object.__setattr__(self, "known_deviations", tuple(self.known_deviations)) + + stray = [c for c in self.not_applicable if not isinstance(c, Capability)] + if stray: + problems.append( + f"unknown capabilities {stray!r} in not_applicable: capabilities are " + f"the members of the Capability enum" + ) + + both = sorted( + capability.tag + for capability in self.not_applicable + if isinstance(capability, Capability) and capability in self.capabilities + ) + if both: + problems.append( + f"capabilities and not_applicable both claim {' '.join(both)}: a " + f"capability is either declared or impossible, and a report saying " + f"both leaves a consumer to guess which" + ) + + problems.extend( + reserved_problems(self.capabilities, self.not_applicable.keys()) + ) + + unreasoned = sorted( + capability.tag + for capability, reason in self.not_applicable.items() + if isinstance(capability, Capability) + and (not isinstance(reason, str) or not reason.strip()) + ) + if unreasoned: + problems.append( + f"not_applicable gives no reason for {' '.join(unreasoned)}: " + f"'impossible for this provider' is only useful to a reader who is " + f"told why, and the report schema requires the reason" + ) + if ( Capability.UNAVAILABLE_INIT in self.capabilities and self.new_unavailable_provider is None @@ -174,6 +287,41 @@ def sorted_capabilities(self) -> list[str]: return sorted(c.tag for c in self.capabilities) +def reserved_problems(*named: Iterable[Capability]) -> list[str]: + """Refuse a reserved capability named anywhere in a configuration. + + A reserved capability gates no scenario, so naming it cannot be verified + either way: declaring it claims something nothing examined, and calling it + not-applicable records an impossibility about a question that was never + asked. Either would reach the report's declaration, which the schema + forbids. + + Refused rather than dropped quietly. The adopter wrote it down and meant + something by it, so a configuration silently different from the one they + wrote is worse than one that will not build -- and construction is where + their own code is still on the stack to say which line to fix. The + alternative, a warning, is a line of CI output nobody reads while an + untested capability goes on being asserted in a published report, which is + how this got into one in the first place. + """ + reserved = sorted( + capability.tag + for group in named + for capability in group + if isinstance(capability, Capability) and capability.reserved + ) + if not reserved: + return [] + declarable = " ".join(sorted(c.tag for c in DECLARABLE_CAPABILITIES)) + return [ + f"reserved capabilities {' '.join(sorted(set(reserved)))} cannot be declared " + f"or called not-applicable: no scenario carries them, so the claim cannot be " + f"verified, cannot produce a skip, and would tell a reader of the report only " + f"that something was claimed and nothing examined. The declarable " + f"capabilities, which is what DECLARABLE_CAPABILITIES holds, are {declarable}" + ] + + def capabilities_of(values: Iterable[Capability]) -> frozenset[Capability]: """Convenience for building a capability set from any iterable.""" return frozenset(values) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control.py index 0e83e5bd..e92dd18d 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control.py @@ -73,6 +73,20 @@ def change_flag(self) -> None: def description(self) -> str: """A short description of what is being controlled, for messages a human reads.""" + # OPTIONAL: ``control_api`` + # + # A control may also offer a ``control_api`` property returning ``"http"`` + # for the normative HTTP control API, or ``"in-process"`` for the narrow + # allowance made for providers with no backend. The conformance report + # records it, so that a claim of in-process control by a provider that does + # have a backend can be treated with the suspicion it deserves. + # + # It is deliberately not a member of this protocol. Adding one would make + # every existing control incomplete for the sake of one string, and there is + # nothing useful the TCK can do with a control that has not said: it cannot + # tell from the outside whether a control spoke HTTP or reached into the + # process, so the field is simply omitted. See ``report.control_api_of``. + @typing.runtime_checkable class ConnectionControl(typing.Protocol): diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py new file mode 100644 index 00000000..85e5c49c --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py @@ -0,0 +1,664 @@ +"""The pytest half of the conformance report: turning a run into the two documents. + +Kept apart from :mod:`report`, which knows what an envelope *is*, and from +:mod:`messages`, which knows what a Cucumber Messages stream is; neither knows +anything about pytest. Everything here is translation -- a pytest node into a +scenario, a :class:`pytest.TestReport` into a step status, the end of a session +into a pair of files on disk. + +Two translations matter. + +**Skips.** pytest reports a skip honestly, unlike some runners, and Cucumber's +``SKIPPED`` says the same thing, so a capability-gated scenario reaches the +stream as skipped without anything having to be decided. What the stream does not +say is *why* -- and it does not need to, because the envelope carries the +provider's declaration and the stream carries the scenario's tags, so the reason +for the skip follows from the two. The skip message is carried anyway, on the +setup hook's result, because a person reading the stream should not have to +perform that derivation. + +**Expected failures.** A scenario marked ``xfail`` is one pytest reports as +skipped and finishes green on. The provider still did not satisfy it, so the +stream reports it as failed. The acknowledgement belongs in the envelope's +``knownDeviations``, where it is a claim about the provider rather than a +softening of the result. +""" + +from __future__ import annotations + +import os +import time +import typing +from pathlib import Path + +import pytest + +from .canonical import ( + PARTIAL_ENV, + canonical_scenarios, + describe, + missing_canonical, + partial_run_allowed, +) +from .config import TckConfig +from .extensions import ( + collision_problem, + reserved_prefix_problem, + uri_collisions, + uri_for, +) +from .messages import ( + FeatureCatalog, + ScenarioIdentity, + ScenarioRun, + Status, + StepRun, + feature_uri, + worse, + write_stream, +) +from .report import ( + REPORT_DIR_ENV, + TCK_DISTRIBUTION, + TCK_IMPLEMENTATION, + PhaseOutcome, + ReportCollector, + Results, + SuiteReport, + control_api_gap, + distribution_version, + envelope_file_name, + normalise_tags, + stream_file_name, + write_envelope, +) + +__all__ = [ + "COLLECTOR_KEY", + "ReportEmitter", + "classify_phase", + "scenario_identity", + "scenario_run", +] + +COLLECTOR_KEY = pytest.StashKey[ReportCollector]() +"""Where the session's collector lives, so a fixture can reach it from a request.""" + +_EXAMPLE_PARAM = "_pytest_bdd_example" +"""The parameter pytest-bdd renders a Scenario Outline over. + +An implementation detail of pytest-bdd, named here rather than spelled inline so +that a version bump that renames it fails in one place. The alternative -- asking +the scenario template for its examples -- would have to work out which row *this* +node is, which is the question the callspec already answers. +""" + +_MAX_REASON = 500 +"""How much of a failure message the stream carries. + +A message is for a person reading a comparison page, not for debugging: whoever +ran the suite has the traceback. Whole tracebacks in a published document also +leak local paths. +""" + +_SKIPPED = pytest.skip.Exception +"""What ``pytest.skip`` raises, named so a step hook can recognise it.""" + +_MAX_MISSING = 10 +"""How many missing canonical scenarios a failure names before summarising. + +Enough to act on and not so many that the reason is lost above them. A run with +one scenario selected is missing twenty-eight, and listing all of them says +nothing the count did not. +""" + + +def scenario_identity(node: pytest.Item) -> ScenarioIdentity | None: + """Describe a pytest node as a Gherkin scenario, or return ``None``. + + ``__scenario__`` is what pytest-bdd hangs on the function it generates, so + its presence is also the test for "is this a TCK scenario at all" -- and it + is readable at collection, without running a single fixture, which is what + lets a scenario skipped before its first step still be accounted for. + """ + scenario = getattr(getattr(node, "function", None), "__scenario__", None) + if scenario is None: + return None + + feature = getattr(scenario, "feature", None) + tags: set[str] = set(getattr(scenario, "tags", None) or ()) + tags |= set(getattr(feature, "tags", None) or ()) + rule = getattr(scenario, "rule", None) + if rule is not None: + tags |= set(getattr(rule, "tags", None) or ()) + tags |= _examples_tags(node, scenario) + + filename = str(getattr(feature, "filename", "")) + path = Path(filename) + relative = str(getattr(feature, "rel_filename", "") or path.name) + + return ScenarioIdentity( + # Derived from where the file is, falling back to what pytest-bdd called + # it. pytest-bdd names a feature by its parent directory joined to its + # own name, which two files can share: an extension at + # tck-extensions/features/errors.feature arrives under exactly the uri + # the canonical errors.feature already occupies, and the payload carries + # one source per uri. + uri=uri_for(path) or feature_uri(relative), + path=path, + name=str(getattr(scenario, "name", "")), + example=_example_of(node), + tags=normalise_tags(tags), + ) + + +def _examples_tags(node: pytest.Item, scenario: object) -> set[str]: + """The tags of the Examples block *this row* came from. + + Gherkin allows an Examples block to carry its own tags, so two rows of one + Scenario Outline can differ in which capability gates them. Those tags are + not on the scenario, the feature or the rule, so a stream built from those + three alone would show a row the capability gate skipped as carrying no + capability at all -- and the envelope's declaration would then not explain + the skip, which is the one derivation the format asks a consumer to make. + + Resolved by intersecting the tags the scenario's Examples blocks declare with + the markers pytest actually put on this node: pytest-bdd attaches an Examples + block's tags as marks on that block's parameter sets, so the intersection + names this row's blocks without having to work out which block a row came + from, and admits nothing that is not a Gherkin tag of this scenario. + + No canonical feature file uses per-Examples tags today, so this is latent -- + but it is latent in the direction of under-reporting a skip, which is the one + failure mode the format exists to rule out. + """ + declared: set[str] = set() + for examples in getattr(scenario, "examples", None) or (): + declared |= set(getattr(examples, "tags", None) or ()) + if not declared: + return set() + return declared & {marker.name for marker in node.iter_markers()} + + +def _example_of(node: pytest.Item) -> tuple[tuple[str, str], ...]: + """The Examples row this node came from, keyed by column header. + + No longer reported -- Cucumber Messages identifies an outline row by the AST + node id of the table row a pickle was compiled from, which is exact and which + every runner that emits Messages already carries. This survives as the *join + key*: it is the one description of a row that both a pytest-bdd node and a + Gherkin pickle can produce independently, so it is how a node is matched to + its pickle. Matching on the pickle's name would not work, because the + compiler interpolates the row's parameters into it and pytest-bdd does not. + + pytest-bdd renders an outline by parametrizing the generated test over one + dict per row, keyed by the Examples column header, and pytest hangs it on the + node's callspec. A scenario that is not an outline is not parametrized and + has no callspec at all, which is why the empty tuple is the answer for one -- + and it matches the empty row of a pickle with a single AST node id. + + Values are passed through as the parser produced them: Gherkin cells are + strings, and both sides of the join have to agree on ``"1"`` rather than one + of them guessing it was meant as a number. + """ + params = getattr(getattr(node, "callspec", None), "params", None) + if not isinstance(params, dict): + return () + row = params.get(_EXAMPLE_PARAM) + if not isinstance(row, dict): + return () + # Column order, as the feature file wrote it, because dicts preserve + # insertion order and pytest-bdd builds this one from the header row. + return tuple((str(header), str(cell)) for header, cell in row.items()) + + +def _group_of(node: pytest.Item) -> str: + """Which module a scenario was generated into. + + pytest-bdd's ``scenarios()`` injects its tests into the module that called + it, and a module resolves one ``tck_config``, so the module is what says + which suite a scenario belongs to. Two modules sharing a ``tck_config`` from + a conftest are two groups pointing at one suite, which is exactly right. + """ + return node.nodeid.partition("::")[0] + + +class ReportEmitter: + """Collects outcomes for the session and writes one report per suite. + + A plugin object rather than module-level hook functions because + ``pytest_runtest_logreport`` is handed a report and nothing else: the state + it has to reach has to come from somewhere, and an instance is a less + surprising somewhere than a module global. + """ + + def __init__(self, config: pytest.Config) -> None: + self.collector = ReportCollector() + self._step_started: dict[str, int] = {} + config.stash[COLLECTOR_KEY] = self.collector + + @pytest.hookimpl(trylast=True) + def pytest_collection_modifyitems(self, items: list[pytest.Item]) -> None: + """Enumerate every TCK scenario the session will run. + + At collection rather than as each runs, so that the stream accounts for + scenarios that never got as far as running a fixture. + + ``trylast`` so that pytest's own deselection -- ``-k``, ``-m``, + ``--deselect`` -- has already removed what it is going to remove. + Enumerating before it does would make a filtered run report every + deselected scenario as collected but never run, which is a true + statement about a list nobody asked for and drowns the one message that + matters: which canonical scenarios are missing. + """ + for item in items: + identity = scenario_identity(item) + if identity is not None: + self.collector.collect(item.nodeid, _group_of(item), identity) + + def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: + self.collector.observe(report.nodeid, _phase_outcome(report)) + + # -- what each Gherkin step did ------------------------------------------ + # + # pytest reports a scenario, not its steps. Cucumber Messages records a + # result per step, and inventing one -- marking all eight steps failed + # because the scenario failed -- would be saying something untrue about the + # seven that passed and the ones that were never reached. pytest-bdd's step + # hooks are the only place the truth is available. + + def pytest_bdd_before_step( + self, request: pytest.FixtureRequest, step: object + ) -> None: + self._step_started[request.node.nodeid] = time.time_ns() + + def pytest_bdd_after_step(self, request: pytest.FixtureRequest) -> None: + self._finish_step(request, Status.passed) + + def pytest_bdd_step_error( + self, request: pytest.FixtureRequest, exception: BaseException + ) -> None: + # A step that calls ``pytest.skip`` raises through the same hook as one + # that failed, and the two are not the same result. Told apart by the + # exception type rather than by the message, which is prose. + if isinstance(exception, _SKIPPED): + self._finish_step(request, Status.skipped, exception) + return + self._finish_step(request, Status.failed, exception) + + def pytest_bdd_step_func_lookup_error( + self, request: pytest.FixtureRequest, exception: BaseException + ) -> None: + # UNDEFINED rather than FAILED: the step was never run, because nothing + # claimed to know how to run it. That is a defect in an adoption rather + # than a finding about the provider, and the stream says which. + self._step_started.setdefault(request.node.nodeid, time.time_ns()) + self._finish_step(request, Status.undefined, exception) + + def _finish_step( + self, + request: pytest.FixtureRequest, + status: Status, + exception: BaseException | None = None, + ) -> None: + node_id = request.node.nodeid + finished = time.time_ns() + self.collector.observe_step( + node_id, + StepRun( + status=status, + message=_reason(str(exception)) if exception is not None else "", + exception_type=type(exception).__name__ + if exception is not None + else "", + started_ns=self._step_started.pop(node_id, finished), + finished_ns=finished, + ), + ) + + def pytest_sessionfinish(self, session: pytest.Session) -> None: + if session.config.getoption("collectonly", False): + # Nothing ran, and nothing was meant to. Every check below asks what + # a run executed, and the answer "nothing" is not a finding here. + return + + problems = self.collector.resolve(scenario_run) + + # Whether a suite may be published is a property of the run rather than + # of the report, so it is established whether or not one was asked for. + # Both checks are made on every suite rather than short-circuited, so a + # suite with two faults hears about both. + unpublishable: set[int] = set() + for suite in self.collector.suites: + sound = self._identities_are_sound(session, suite) + complete = self._canonical_set_ran(session, suite) + if not sound or not complete: + unpublishable.add(id(suite)) + + directory = os.environ.get(REPORT_DIR_ENV, "").strip() + if not directory: + return + for problem in problems: + self._fail(session, f"provider-tck: {problem}") + self.write(session, Path(directory), unpublishable) + + def write( + self, + session: pytest.Session, + directory: Path, + unpublishable: typing.AbstractSet[int] = frozenset(), + ) -> None: + """Write every suite's pair of files, failing the session if one cannot be. + + A run that asked for a report and silently did not get one is how a + publishing pipeline ends up serving a stale result forever, so both a + write failure and an incomplete document are loud and change the exit + status rather than being logged and forgotten. + """ + written: dict[str, str] = {} + for suite in self.collector.suites: + if id(suite) in unpublishable: + continue + name = suite.config.name + file_name = envelope_file_name(name) + if written.get(file_name, name) != name: + self._fail( + session, + f"provider-tck: suites {written[file_name]!r} and {name!r} both " + f"write {file_name}; give them names that do not collide", + ) + continue + written[file_name] = name + self._write_suite(session, directory, suite) + + def _canonical_set_ran(self, session: pytest.Session, suite: SuiteReport) -> bool: + """Whether this suite executed every scenario the TCK ships. + + The check a conformance claim rests on that no amount of reading the + report can supply: the results payload says what happened to the + scenarios that ran, and says nothing at all about the ones that did not. + + A capability-gated skip counts -- it was asked, and the report accounts + for it with a reason. An extension scenario does not count and cannot + close a gap. :data:`~.canonical.PARTIAL_ENV` downgrades the failure to a + note for someone working on a single scenario; it does not make the run + publishable, because the report is withheld either way. + """ + missing = missing_canonical(suite.runs.values()) + if not missing: + return True + + name = suite.config.name + total = len(canonical_scenarios()) + headline = ( + f"provider-tck [{name}]: {len(missing)} of {total} canonical scenarios " + f"did not run, so this run cannot support a conformance claim and no " + f"report is written for it. The canonical set is fixed by the " + f"specification; running less of it is not a configuration. Decline " + f"capabilities your provider does not have through TckConfig instead, " + f"which reports the scenarios as skipped with their reason" + ) + if partial_run_allowed(): + self._say( + session, + f"{headline}. {PARTIAL_ENV} is set, so the run is not failed for it", + ) + else: + self._fail( + session, + f"{headline}. To filter anyway while working on one scenario, set " + f"{PARTIAL_ENV}=1 and accept that the run is not a conformance run", + ) + for key in missing[:_MAX_MISSING]: + self._say(session, f" - {describe(key)}") + if len(missing) > _MAX_MISSING: + self._say(session, f" ... and {len(missing) - _MAX_MISSING} more") + return False + + def _write_suite( + self, session: pytest.Session, directory: Path, suite: SuiteReport + ) -> None: + name = suite.config.name + runs = suite.sorted_runs + + catalog = FeatureCatalog() + try: + for run in runs: + catalog.load(run.identity) + except OSError as error: + self._fail( + session, + f"provider-tck [{name}]: could not read the feature files the run " + f"executed, so the results payload cannot name them: {error}", + ) + return + + unmatched = [ + run.identity for run in runs if catalog.pickle_for(run.identity) is None + ] + for identity in unmatched: + # The one failure mode this format exists to rule out: a scenario + # that ran and is missing from the results. Reported per scenario + # rather than as a count, because which one it is is the whole point. + self._fail( + session, + f"provider-tck [{name}]: {identity.uri} scenario " + f"{identity.name!r}{_row(identity)} matched no Gherkin pickle, so " + f"the results payload does not account for it", + ) + + stream_path = directory / stream_file_name(name) + try: + digest = write_stream( + stream_path, + catalog, + runs, + implementation=TCK_IMPLEMENTATION, + implementation_version=distribution_version(TCK_DISTRIBUTION), + ) + envelope = suite.build(Results(location=stream_path.name, digest=digest)) + path = write_envelope(directory, name, envelope) + except OSError as error: + self._fail( + session, + f"provider-tck [{name}]: could not write the conformance report " + f"to {directory}: {error}", + ) + return + + counts = ", ".join( + f"{count} {status}" for status, count in sorted(suite.counts().items()) + ) + self._say( + session, + f"provider-tck [{name}]: report written to {path} with results in " + f"{stream_path.name} ({counts})", + ) + + gap = control_api_gap(suite.config.control) + if gap: + self._say(session, f"provider-tck [{name}]: {gap}") + + def _identities_are_sound( + self, session: pytest.Session, suite: SuiteReport + ) -> bool: + """Whether every feature file this suite ran is named in the payload as itself. + + Two ways it might not be, and both are silent without this. A file that + is not one of the packaged assets must not be reported under their uri + prefix, or a consumer reading ``features/errors.feature`` in the stream + has no way to tell that the specification did not write it. And two + files must not share a uri, or the stream carries one source for both + and the second file's scenarios are reported against the first's. + + A suite that fails either writes no report. Refusing is the point: a + document that presents an adopter's feature file as the specification's + is worse than no document, because it is the one thing a consumer cannot + check. + """ + name = suite.config.name + runs = suite.sorted_runs + + reserved = [ + problem + for run in runs + if (problem := reserved_prefix_problem(run.identity.uri, run.identity.path)) + ] + for problem in dict.fromkeys(reserved): + self._fail(session, f"provider-tck [{name}]: {problem}") + + collisions = uri_collisions( + (run.identity.uri, run.identity.path) for run in runs + ) + for uri, paths in sorted(collisions.items()): + self._fail( + session, f"provider-tck [{name}]: {collision_problem(uri, paths)}" + ) + + return not reserved and not collisions + + def _say(self, session: pytest.Session, message: str) -> None: + reporter = session.config.pluginmanager.get_plugin("terminalreporter") + if reporter is not None: + reporter.write_line(message) + + def _fail(self, session: pytest.Session, message: str) -> None: + self._say(session, message) + session.exitstatus = pytest.ExitCode.INTERNAL_ERROR + + +def _row(identity: ScenarioIdentity) -> str: + if not identity.example: + return "" + row = " ".join(f"{header}={cell}" for header, cell in identity.example) + return f" [{row}]" + + +def _phase_outcome(report: pytest.TestReport) -> PhaseOutcome: + """Reduce a pytest phase report to what the conformance report needs.""" + xfail_reason: str | None = getattr(report, "wasxfail", None) + message = _skip_reason(report) if report.skipped else _failure_reason(report) + return PhaseOutcome( + when=report.when or "", + outcome=report.outcome, + xfail_reason=xfail_reason, + message=message, + start=getattr(report, "start", 0.0), + stop=getattr(report, "stop", 0.0), + ) + + +def scenario_run( + identity: ScenarioIdentity, + phases: list[PhaseOutcome], + steps: list[StepRun], +) -> ScenarioRun: + """Assemble one scenario's execution from what pytest reported about it. + + The scenario's own status is the most serious of its phases', so a scenario + whose steps passed and whose teardown then blew up is a failed scenario: the + phase that reports last must not be the one that decides. + """ + starts = [phase.start for phase in phases if phase.start] + stops = [phase.stop for phase in phases if phase.stop] + run = ScenarioRun( + identity=identity, + steps=list(steps), + started_ns=int(min(starts, default=0.0) * 1_000_000_000), + finished_ns=int(max(stops, default=0.0) * 1_000_000_000), + ) + for phase in phases: + status, message = classify_phase(phase) + result = StepRun( + status=status, + message=message, + started_ns=int(phase.start * 1_000_000_000), + finished_ns=int(phase.stop * 1_000_000_000), + ) + if phase.when == "setup": + run.setup = result + elif phase.when == "teardown": + run.teardown = result + upgraded = worse(run.status, status) + if upgraded is not run.status: + # The message belongs to whichever phase decided the verdict, so a + # teardown failure does not inherit the reason a passing call gave. + run.message = message + run.status = upgraded + return run + + +def classify_phase(phase: PhaseOutcome) -> tuple[Status, str]: + """Map one pytest phase onto a Cucumber status. + + The one decision that is not a rename: an expected failure is still a + failure. pytest reports an ``xfail`` as skipped and exits zero; the provider + did not satisfy the scenario, and a stream calling it anything else would + hide exactly the deviation the marker was added to keep visible. The + acknowledgement goes in the envelope's ``knownDeviations`` instead, which is + where a claim about the provider belongs. + """ + if phase.outcome == "skipped" and phase.xfail_reason is not None: + return Status.failed, _reason(f"expected failure: {phase.xfail_reason}") + if phase.outcome == "failed": + return Status.failed, phase.message or "failed" + if phase.outcome == "skipped": + return Status.skipped, phase.message or "skipped" + return Status.passed, "" + + +def _skip_reason(report: pytest.TestReport) -> str: + longrepr = report.longrepr + if isinstance(longrepr, tuple) and len(longrepr) == 3: + return _reason(str(longrepr[2]).removeprefix("Skipped: ")) + return _reason(str(longrepr)) if longrepr else "" + + +def _failure_reason(report: pytest.TestReport) -> str: + message = getattr(getattr(report.longrepr, "reprcrash", None), "message", "") + if not message: + message = str(report.longrepr) if report.longrepr else "" + return _reason(message) + + +def _reason(message: str) -> str: + collapsed = " ".join(message.split()) + if len(collapsed) <= _MAX_REASON: + return collapsed + return collapsed[: _MAX_REASON - 1].rstrip() + "…" + + +def observe_provider_name( + config: pytest.Config, tck_config: TckConfig, provider_name: str | None +) -> None: + """Record what the provider called itself, for the suite the run is in. + + The provider's own metadata name rather than the suite name, because the two + answer different questions: the suite name is chosen to read well in a + failure message, which makes it the configuration and it is reported as one. + """ + collector: ReportCollector | None = config.stash.get(COLLECTOR_KEY, None) + if collector is not None and provider_name: + collector.suite_for(tck_config).observe_provider_name(provider_name) + + +def bind_scenario(request: pytest.FixtureRequest) -> None: + """Tell the collector which suite this scenario's module is testing. + + Called from an autouse fixture that the capability gate depends on, so that a + scenario the gate stops has still contributed its suite. Only one scenario of + a module has to get this far, but the gate skips whole capabilities at a + time, and a module all of whose scenarios were skipped would otherwise have + no report to be written to. + """ + collector: ReportCollector | None = request.config.stash.get(COLLECTOR_KEY, None) + if collector is None or scenario_identity(request.node) is None: + # Checked before asking for the config so that a test which is not a TCK + # scenario instantiates nothing, which is the same bargain the capability + # gate makes. + return + try: + tck_config = typing.cast(TckConfig, request.getfixturevalue("tck_config")) + except pytest.FixtureLookupError: + return + collector.bind(request.node.nodeid, tck_config) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/extensions.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/extensions.py new file mode 100644 index 00000000..b1a667cd --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/extensions.py @@ -0,0 +1,286 @@ +"""Where the scenarios come from: the canonical set, plus whatever an adopter adds. + +A provider is rarely only a provider. flagd has ``fractional`` targeting, another +vendor has a proprietary rollout rule, and the behaviour of those is as worth +pinning as the contract they sit on top of. Verifying them used to mean standing +up a second harness: a second backend lifecycle, a second set of fixtures, a +second thing to keep working. The canonical suite ran, then something else ran, +and nothing tied the two together. + +So an adopter's own scenarios run **inside** the canonical suite instead -- +against the same provider instance, in the same backend lifecycle, with the same +step vocabulary available. Almost nothing is needed to make that happen, because +pytest already scans. It collects ``conftest.py`` on its own and pytest-bdd +resolves step definitions through the fixture system, so a step defined in the +adopter's ``conftest.py`` -- or in the test module itself -- is in scope for the +scenarios ``scenarios()`` generates there. The only thing pytest cannot find by +itself is the feature files, which is what this module finds: a directory named +``tck-extensions`` beside the adopter's test module. + +That leaves one line, and it is the same line whether or not there are +extensions:: + + scenarios(*feature_paths()) + +**An extension can never stand in for a canonical scenario.** The two are told +apart by the uri each feature file reaches the results payload under, and this +module derives that uri from where the file *is* rather than taking what the +runner offers: + +* ``features/…`` is the packaged canonical assets, and nothing else; +* ``extensions/…`` is a discovered extension, whatever the adopter's own + directory layout under ``tck-extensions`` looks like. + +The derivation is not decoration. pytest-bdd names a feature file by its parent +directory joined to its own name, so ``tck-extensions/features/errors.feature`` +arrives as ``features/errors.feature`` -- the same uri as a canonical file, and +the payload can carry only one source per uri. The canonical source is parsed +first, the extension's is never read, and its scenarios are reported against the +canonical file's pickles or against none at all. Java hit the same thing by a +different route: a same-named feature file in a second classpath root replaced +the canonical one outright and the suite went green having run the adopter's +version. +""" + +from __future__ import annotations + +import importlib.resources +import inspect +import typing +from pathlib import Path + +from .messages import feature_uri + +__all__ = [ + "CANONICAL_DIRECTORY", + "EXTENSIONS_DIRECTORY", + "EXTENSIONS_URI_PREFIX", + "canonical_root", + "collision_problem", + "extension_root", + "feature_paths", + "features_path", + "is_canonical", + "is_canonical_uri", + "reserved_prefix_problem", + "uri_collisions", + "uri_for", +] + +_PACKAGE = "openfeature.contrib.tools.provider_tck" + +CANONICAL_DIRECTORY = "features" +"""The packaged directory the canonical feature files live in. + +Also the uri prefix they carry in the results payload, which is why it is +reserved: a consumer reading ``features/errors.feature`` in a stream is entitled +to assume it is reading the specification's file rather than a local one that +happened to land in a directory of that name. +""" + +EXTENSIONS_DIRECTORY = "tck-extensions" +"""Where an adopter puts feature files of their own, beside their test module. + +Deliberately not ``features``: a directory sharing the canonical name is how an +extension comes to occupy a canonical file's identity, and a convention that +cannot collide is worth more than one that reads slightly better. The name is +the one Java's TCK scans for on the classpath, so an adopter who ships a provider +in both languages puts the same directory in both repositories. +""" + +EXTENSIONS_URI_PREFIX = "extensions" +"""The uri prefix an extension's scenarios reach the results payload under. + +The Go and JavaScript suites mount extensions under the same prefix, so a +consumer reading reports from several languages applies one rule to tell an +adopter's scenario from the specification's. +""" + + +def features_path() -> str: + """Return the directory holding the canonical feature files. + + Packaged with this distribution, so a consumer needs no submodule and no + particular directory layout. This is the canonical set on its own; prefer + :func:`feature_paths`, which also picks up an adopter's own scenarios. + """ + return str(importlib.resources.files(_PACKAGE) / CANONICAL_DIRECTORY) + + +def feature_paths() -> tuple[str, ...]: + """Return every feature directory this adoption should run. + + The canonical set, always, and a ``tck-extensions`` directory beside the + calling module if there is one. Hand the result to pytest-bdd's + ``scenarios()``:: + + scenarios(*feature_paths()) + + That line does not change when an adopter adds an extension, which is what + makes adding one a matter of creating a directory rather than of configuring + anything. + + The calling module is located from the caller's frame, which is how + pytest-bdd locates it for ``scenarios()`` itself, so the two agree about + which module is adopting the suite. Call it from the test module rather than + from a helper: a helper's directory is what a helper would find. A caller + with no ``__file__`` -- an interactive session, an exec'd string -- gets the + canonical set alone. + """ + paths = [features_path()] + directory = _caller_directory() + if directory is not None: + extensions = extension_root(directory) + if extensions is not None: + paths.append(str(extensions)) + return tuple(paths) + + +def extension_root(module_directory: Path) -> Path | None: + """The extension directory beside a test module, or ``None`` if there is none. + + ``None`` rather than a path that contributes nothing, so that an adopter + without extensions hands ``scenarios()`` exactly what they handed it before: + same scenarios, same count, same report. + """ + candidate = module_directory / EXTENSIONS_DIRECTORY + return candidate if candidate.is_dir() else None + + +def canonical_root() -> Path | None: + """The packaged canonical features directory, as a real path. + + ``None`` if the assets are not on the filesystem -- an installation from a + zipimport, say. Everything built on this degrades to "cannot tell", which is + the honest answer and never a false accusation. + """ + try: + return _resolve(Path(features_path())) + except (OSError, TypeError): # pragma: no cover - assets outside a filesystem + return None + + +def is_canonical(path: Path) -> bool: + """Whether a feature file is one of the packaged canonical ones.""" + canonical = canonical_root() + return canonical is not None and _resolve(path).is_relative_to(canonical) + + +def is_canonical_uri(uri: str) -> bool: + """Whether a uri names a canonical feature file. + + The discriminator between a canonical scenario and an extension one wherever + it matters -- the report, and a consumer reading the stream. Derived from the + uri rather than carried beside it, so there is no second fact to disagree + with the first. + """ + return uri.startswith(f"{CANONICAL_DIRECTORY}/") + + +def uri_for(path: Path) -> str | None: + """The uri a feature file should reach the results payload under. + + ``None`` when the file is neither canonical nor under an extension + directory, in which case the caller falls back to what pytest-bdd named it. + + Derived from the file's location rather than from pytest-bdd's + ``rel_filename``, which is the parent directory's name joined to the file's + own. That is what let ``tck-extensions/features/errors.feature`` present + itself as ``features/errors.feature``: the same uri as a canonical file, and + the payload can carry only one source per uri. + """ + resolved = _resolve(path) + + canonical = canonical_root() + if canonical is not None and resolved.is_relative_to(canonical): + return feature_uri( + str(Path(CANONICAL_DIRECTORY) / resolved.relative_to(canonical)) + ) + + for parent in resolved.parents: + if parent.name == EXTENSIONS_DIRECTORY: + return feature_uri( + str(Path(EXTENSIONS_URI_PREFIX) / resolved.relative_to(parent)) + ) + return None + + +def reserved_prefix_problem(uri: str, path: Path) -> str | None: + """Report a feature file claiming the canonical uri prefix without being canonical. + + The one thing the naming convention cannot rule out on its own: an adopter + who hands ``scenarios()`` a directory of their own named ``features``. The + file is then named exactly as a canonical one would be, and a consumer + reading the payload has no way to tell that the specification did not write + it. Caught where the report is assembled rather than shipped. + """ + if not is_canonical_uri(uri) or is_canonical(path): + return None + return ( + f"{uri} is not a canonical feature file -- it is {path} -- but it would be " + f"reported under the {CANONICAL_DIRECTORY}/ prefix, which is reserved for " + f"the packaged conformance assets. Move it into a directory named " + f"{EXTENSIONS_DIRECTORY} beside the test module, which feature_paths() " + f"finds on its own" + ) + + +def uri_collisions( + identified: typing.Iterable[tuple[str, Path]], +) -> dict[str, tuple[Path, ...]]: + """Feature files that would reach the payload under one uri, keyed by that uri. + + Deriving the uri from the file's location removes the collision an adopter + is actually likely to hit, but it does not make one impossible. Two + extension roots contributing the same relative path to a single suite -- two + test modules sharing one ``tck_config`` from a conftest, each with a + ``tck-extensions/vendor.feature`` -- still land on ``extensions/vendor.feature`` + twice, and so does a ``tck-extensions`` directory nested inside another one. + + That has to be refused rather than resolved. A Messages stream carries one + ``Source`` per uri, so the second file's source is never read: its scenarios + are reported against the first file's pickles where the names happen to + match, and go missing where they do not. The first is the silent form of + exactly the failure Java measured, and it is the one a consumer cannot + detect from the outside. + + Compared by resolved path, so the same file reached by two routes is one + file rather than a collision. + """ + files: dict[str, dict[Path, None]] = {} + for uri, path in identified: + files.setdefault(uri, {})[_resolve(path)] = None + return {uri: tuple(paths) for uri, paths in files.items() if len(paths) > 1} + + +def collision_problem(uri: str, paths: typing.Sequence[Path]) -> str: + """Say which files collided and what to do about it.""" + listed = ", ".join(str(path) for path in sorted(paths)) + return ( + f"{uri} is the uri of {len(paths)} different feature files -- {listed} -- " + f"and the results payload carries one source per uri, so one of them " + f"would be reported against the other's source. Give them paths that " + f"differ below their {EXTENSIONS_DIRECTORY} directory" + ) + + +def _caller_directory() -> Path | None: + """The directory of the module two frames up, if it has a file.""" + frame = inspect.currentframe() + for _ in range(2): + if frame is None: # pragma: no cover - no Python frames to walk + return None + frame = frame.f_back + if frame is None: # pragma: no cover - called with no caller above + return None + file_name: typing.Any = frame.f_globals.get("__file__") + if not isinstance(file_name, str) or not file_name: + return None + return _resolve(Path(file_name)).parent + + +def _resolve(path: Path) -> Path: + try: + return path.resolve() + except OSError: # pragma: no cover - a path that cannot be resolved at all + return path diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/inprocess.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/inprocess.py index 1d69254c..11586e0f 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/inprocess.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/inprocess.py @@ -63,6 +63,16 @@ def __init__(self) -> None: def description(self) -> str: return "in-process control of an in-memory provider" + @property + def control_api(self) -> str: + """Report how this backend was driven, for the conformance report. + + ``in-process`` is the narrow allowance for providers with no backend, + which is exactly what this control exists for. A provider that does have + a backend and reports this is claiming something it should not. + """ + return "in-process" + def new_provider(self) -> FeatureProvider: """Create the provider for the scenario about to run, at the baseline. diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/messages.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/messages.py new file mode 100644 index 00000000..6d8199ce --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/messages.py @@ -0,0 +1,667 @@ +"""The results payload: one run of the suite written as Cucumber Messages. + +The conformance report used to define its own per-scenario result list -- an +outcome enum, a tag list, and a field naming which Scenario Outline row an entry +came from. All three already exist in `Cucumber Messages`_, the ndjson protocol +Cucumber itself emits: it is maintained, schema'd, cross-language, and it carries +things a bespoke format would have had to invent, including the executed feature +source. So the report no longer describes results. It points at a stream of them. + +Nothing here knows about pytest. It takes a set of scenario outcomes and a set of +feature files and produces the stream; :mod:`.emitter` is what turns a pytest +session into the former. + +**Where the messages come from.** Two libraries, each doing the half it owns. + +``gherkin-official`` -- the reference Gherkin parser, already a dependency of +pytest-bdd -- produces the ``gherkinDocument`` and ``pickle`` payloads. Those +payloads *are* Messages: emitting Messages ndjson is what that library exists +for, so its output is used as it comes rather than round-tripped through +another representation that could quietly drop a field it does not model. + +``cucumber-messages`` -- the official Python types, from the same repository as +the protocol -- builds the execution half: ``Meta``, ``TestCase``, +``TestCaseStarted``, ``TestStepFinished`` and the rest. Hand-writing those dicts +would work until the protocol moved. + +**Why the feature files are parsed again.** pytest-bdd parses them with +``gherkin-official`` too, but converts the result into dataclasses of its own +that do not carry the AST node ids. Those ids are what a pickle refers to, and +what makes one Scenario Outline row distinguishable from another, so the stream +needs a parse whose ids it owns. Four small files, parsed once per session. + +.. _Cucumber Messages: https://github.com/cucumber/messages +""" + +from __future__ import annotations + +import hashlib +import importlib.metadata +import json +import platform +import typing +from dataclasses import dataclass, field +from pathlib import Path, PurePath + +import cucumber_messages as cucumber +from gherkin.ast_builder import AstBuilder +from gherkin.parser import Parser +from gherkin.pickles.compiler import Compiler +from gherkin.stream.id_generator import IdGenerator + +from .capability import Capability, capability_for_tag + +__all__ = [ + "MESSAGES_FORMAT", + "FeatureCatalog", + "ScenarioIdentity", + "ScenarioKey", + "ScenarioRun", + "Status", + "StepRun", + "worse", + "write_stream", +] + +ScenarioKey = tuple[str, str, tuple[tuple[str, str], ...]] +"""What names one scenario: its feature file's uri, its name, and its Examples row. + +The one description of a scenario that a pytest-bdd node and a Gherkin pickle +can each produce without consulting the other, which is what makes it usable +both as the join key inside :class:`FeatureCatalog` and as the currency of a +comparison between the scenarios that ran and the ones that were shipped. +""" + +MESSAGES_FORMAT = "cucumber-messages" +"""The ``results.format`` value the envelope carries for this payload.""" + +Status = cucumber.TestStepResultStatus +"""The protocol's own status vocabulary, used rather than one of ours. + +Cucumber's seven statuses already draw the distinctions a conformance run needs, +and the four-value outcome enum this replaced drew a different set: it split +"did not run" into a capability the provider did not declare and one that cannot +apply to it, and merged "failed" with "the step was never reached". The first +distinction is not a property of the run at all -- it follows from the report's +declaration and the scenario's tags -- so it belongs in the envelope, once, and +not in every scenario. +""" + +_SEVERITY = { + Status.unknown: 0, + Status.passed: 1, + Status.skipped: 2, + Status.pending: 3, + Status.undefined: 4, + Status.ambiguous: 5, + Status.failed: 6, +} +"""How Cucumber orders its statuses, which is how a test case takes one. + +A test case is as bad as its worst step -- that is the rule a consumer applies to +derive a scenario's outcome from the stream, and this module applies the same one +so that what the stream says and what this package believes cannot diverge. +""" + + +def worse(left: Status, right: Status) -> Status: + """The more serious of two statuses.""" + return right if _SEVERITY[right] > _SEVERITY[left] else left + + +_MEDIA_TYPE = cucumber.SourceMediaType.text_x_cucumber_gherkin_plain + +_SETUP_HOOK_ID = "provider-tck-setup" +_TEARDOWN_HOOK_ID = "provider-tck-teardown" +"""The two hooks every test case carries, as the protocol models them. + +pytest runs a scenario in three phases and only the middle one executes Gherkin +steps: the capability gate skips during setup, and a provider that fails to shut +down fails during teardown. Neither has a pickle step to attach a result to, so +without hooks a gated skip would have to borrow the first step's result and a +teardown failure would be invisible behind a row of passed steps. Cucumber +represents exactly this with a ``Hook`` and a ``TestStep`` that references it, +which is what these are. +""" + +_MESSAGES_DISTRIBUTION = "cucumber-messages" +_UNKNOWN_VERSION = "unknown" + + +@dataclass(frozen=True) +class ScenarioIdentity: + """What a scenario is, independent of how it turned out. + + Established at collection, from the pytest node alone, so that a scenario + skipped before a single step ran is identified exactly as fully as one that + passed. That is what lets the stream account for every scenario rather than + only for the ones that got far enough to be interesting. + """ + + uri: str + """The feature file as Cucumber names it, e.g. ``features/errors.feature``. + + Slash-separated on every platform, and the same string in ``Source``, + ``GherkinDocument`` and ``Pickle``, which is what ties the three together. + """ + + path: Path + """Where that file actually is, so its source can be read and parsed.""" + + name: str + """The scenario name as the feature file spells it. + + For a Scenario Outline this is the template name, shared by every row -- + which is why it is not on its own an identity. + """ + + tags: tuple[str, ...] + + example: tuple[tuple[str, str], ...] = () + """The Examples row, as header/cell pairs, for a scenario from an outline. + + Not reported: Messages carries row identity as the pickle's AST node ids, + which is where four independent implementations of a bespoke ``example`` + field were each converging by hand. It survives here only as the join key + that matches a pytest node to its pickle -- pytest-bdd parametrises the + generated test over the row, and the row is the one thing both sides of that + join can see. + """ + + def capabilities(self) -> tuple[Capability, ...]: + """The capabilities this scenario's tags gate it behind.""" + gated = (capability_for_tag(tag) for tag in self.tags) + return tuple(capability for capability in gated if capability is not None) + + +@dataclass(frozen=True) +class StepRun: + """What happened to one step, as the protocol records it.""" + + status: Status + message: str = "" + exception_type: str = "" + started_ns: int = 0 + finished_ns: int = 0 + + +@dataclass +class ScenarioRun: + """One scenario's execution: its verdict, and what each phase did.""" + + identity: ScenarioIdentity + status: Status = Status.unknown + message: str = "" + setup: StepRun | None = None + teardown: StepRun | None = None + steps: list[StepRun] = field(default_factory=list) + """Step results in execution order, as far as execution got. + + Shorter than the pickle's step list whenever a scenario stopped early, which + is the normal case for a failure and the whole list for a skip. The stream + pads the difference with ``SKIPPED``, which is what Cucumber means by it. + """ + + started_ns: int = 0 + finished_ns: int = 0 + + +@dataclass(frozen=True) +class _Pickle: + """One compiled pickle, reduced to what the stream needs to refer to it.""" + + id: str + step_ids: tuple[str, ...] + payload: dict[str, typing.Any] + + +class FeatureCatalog: + """The feature files a run executed, parsed into Messages and indexed. + + Indexed by what both sides of the join can see: the file, the scenario name + as the feature file spells it, and the Examples row. A pytest-bdd node knows + those three; a pickle can be made to yield them by following its AST node ids + back to the scenario and the table row it was compiled from. Matching on the + pickle's own name would not do, because the compiler interpolates outline + parameters into it and pytest-bdd does not. + """ + + def __init__(self) -> None: + # One generator across the whole session, so ids are unique across + # feature files rather than only within one -- a stream is a single id + # space and two files numbering from zero would collide. + self._ids = IdGenerator() + self._sources: dict[str, str] = {} + self._documents: dict[str, dict[str, typing.Any]] = {} + self._pickles: dict[str, list[_Pickle]] = {} + self._index: dict[ScenarioKey, _Pickle] = {} + + @property + def uris(self) -> list[str]: + return sorted(self._documents) + + @property + def scenario_keys(self) -> frozenset[ScenarioKey]: + """Every scenario the loaded feature files define, row by row. + + What a feature file *asks*, as opposed to what a run executed. Taken + from the compiled pickles rather than from the AST so that a Scenario + Outline contributes one entry per row, which is what the runner + generates and therefore what a comparison has to be made in. + """ + return frozenset(self._index) + + def load(self, identity: ScenarioIdentity) -> None: + """Parse the feature file this scenario came from, once.""" + self.load_file(identity.uri, identity.path) + + def load_file(self, uri: str, path: Path) -> None: + """Parse one feature file under the uri it will be reported by, once. + + Separate from :meth:`load` because the canonical set has to be parsed + with no run in hand: the question "did every shipped scenario execute" + is asked of feature files, not of outcomes. + """ + if uri in self._documents: + return + source = path.read_text(encoding="utf-8") + document: dict[str, typing.Any] = Parser( + ast_builder=AstBuilder(self._ids) + ).parse(source) + document["uri"] = uri + pickles: list[dict[str, typing.Any]] = Compiler(self._ids).compile(document) + + self._sources[uri] = source + self._documents[uri] = document + self._pickles[uri] = [ + _Pickle( + id=str(pickle["id"]), + step_ids=tuple(str(step["id"]) for step in pickle.get("steps") or ()), + payload=pickle, + ) + for pickle in pickles + ] + self._index_pickles(uri, document, pickles) + + def _index_pickles( + self, + uri: str, + document: dict[str, typing.Any], + pickles: list[dict[str, typing.Any]], + ) -> None: + names, rows = _ast_index(document) + for pickle, entry in zip(pickles, self._pickles[uri], strict=True): + ast_node_ids = [str(node) for node in pickle["astNodeIds"]] + name = names.get(ast_node_ids[0], str(pickle["name"])) + row = rows.get(ast_node_ids[1], ()) if len(ast_node_ids) > 1 else () + self._index.setdefault((uri, name, row), entry) + + def pickle_for(self, identity: ScenarioIdentity) -> _Pickle | None: + """The pickle this scenario was compiled from, or ``None`` if unmatched. + + ``None`` is a defect rather than a possibility to tolerate: a scenario + that ran and has no pickle cannot appear in the stream, which is the one + failure mode the report exists to rule out. The caller fails the run. + """ + return self._index.get((identity.uri, identity.name, identity.example)) + + def source_envelopes(self) -> typing.Iterator[dict[str, typing.Any]]: + """The ``Source``, ``GherkinDocument`` and ``Pickle`` messages, in order. + + The source comes first because everything after it refers to it, and it + is the reason this format beats recording an asset revision: a consumer + can read the questions that were actually asked rather than trusting a + commit hash to stand for them. + """ + for uri in self.uris: + yield _envelope( + cucumber.Envelope( + source=cucumber.Source( + data=self._sources[uri], media_type=_MEDIA_TYPE, uri=uri + ) + ) + ) + yield {"gherkinDocument": self._documents[uri]} + for entry in self._pickles[uri]: + yield {"pickle": entry.payload} + + +def _ast_index( + document: dict[str, typing.Any], +) -> tuple[dict[str, str], dict[str, tuple[tuple[str, str], ...]]]: + """Map AST node ids onto scenario names and Examples rows. + + Walks the parsed document rather than the pickles, because the pickle is + where the outline has already been expanded: the scenario name it carries has + the row's parameters substituted into it, and the row itself has become a + list of interpolated step texts. The AST still has both separately, which is + what a pytest-bdd node can be compared against. + """ + names: dict[str, str] = {} + rows: dict[str, tuple[tuple[str, str], ...]] = {} + + def visit(children: typing.Iterable[dict[str, typing.Any]]) -> None: + for child in children: + if "rule" in child: + visit(child["rule"].get("children") or ()) + continue + scenario = child.get("scenario") + if scenario is None: + continue + names[str(scenario["id"])] = str(scenario["name"]) + for examples in scenario.get("examples") or (): + header = examples.get("tableHeader") + if header is None: + continue + headers = [str(cell["value"]) for cell in header["cells"]] + for row in examples.get("tableBody") or (): + cells = [str(cell["value"]) for cell in row["cells"]] + rows[str(row["id"])] = tuple(zip(headers, cells, strict=False)) + + feature = document.get("feature") + if feature is not None: + visit(feature.get("children") or ()) + return names, rows + + +def write_stream( + path: Path, + catalog: FeatureCatalog, + runs: typing.Sequence[ScenarioRun], + implementation: str, + implementation_version: str, +) -> str: + """Write the stream and return its digest as ``sha256:``. + + The digest is returned rather than recomputed by the caller so that what is + hashed is exactly the bytes that were written, which is the only version of + that claim worth putting in the envelope. + """ + digest = hashlib.sha256() + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8", newline="\n") as stream: + for envelope in _stream(catalog, runs, implementation, implementation_version): + line = ( + json.dumps(envelope, ensure_ascii=False, separators=(",", ":")) + "\n" + ) + stream.write(line) + digest.update(line.encode("utf-8")) + return f"sha256:{digest.hexdigest()}" + + +def _stream( + catalog: FeatureCatalog, + runs: typing.Sequence[ScenarioRun], + implementation: str, + implementation_version: str, +) -> typing.Iterator[dict[str, typing.Any]]: + started = min((run.started_ns for run in runs), default=0) + finished = max((run.finished_ns for run in runs), default=started) + + yield _envelope( + cucumber.Envelope(meta=_meta(implementation, implementation_version)) + ) + yield _envelope( + cucumber.Envelope( + test_run_started=cucumber.TestRunStarted( + id=_RUN_ID, timestamp=_timestamp(started) + ) + ) + ) + yield from _hook_envelopes() + yield from catalog.source_envelopes() + + for index, run in enumerate(runs): + entry = catalog.pickle_for(run.identity) + if entry is None: + # Ruled out by the caller before it gets here; skipping rather than + # raising keeps a defect in the report from destroying the run's own + # exit status, which says something the report cannot. + continue + yield from _test_case_envelopes(f"test-case-{index}", entry, run) + + yield _envelope( + cucumber.Envelope( + test_run_finished=cucumber.TestRunFinished( + success=all(run.status is not Status.failed for run in runs), + timestamp=_timestamp(finished), + test_run_started_id=_RUN_ID, + ) + ) + ) + + +_RUN_ID = "provider-tck-run" + + +def _test_case_envelopes( + test_case_id: str, entry: _Pickle, run: ScenarioRun +) -> typing.Iterator[dict[str, typing.Any]]: + """One scenario: its test case, and what each of its steps did.""" + started_id = f"{test_case_id}-started" + setup_step_id = f"{test_case_id}-setup" + teardown_step_id = f"{test_case_id}-teardown" + + test_steps = [ + cucumber.TestStep(id=setup_step_id, hook_id=_SETUP_HOOK_ID), + *( + cucumber.TestStep(id=f"{test_case_id}-{position}", pickle_step_id=step_id) + for position, step_id in enumerate(entry.step_ids) + ), + cucumber.TestStep(id=teardown_step_id, hook_id=_TEARDOWN_HOOK_ID), + ] + yield _envelope( + cucumber.Envelope( + test_case=cucumber.TestCase( + id=test_case_id, + pickle_id=entry.id, + test_run_started_id=_RUN_ID, + test_steps=test_steps, + ) + ) + ) + yield _envelope( + cucumber.Envelope( + test_case_started=cucumber.TestCaseStarted( + attempt=0, + id=started_id, + test_case_id=test_case_id, + timestamp=_timestamp(run.started_ns), + ) + ) + ) + + for step_id, result in zip( + (step.id for step in test_steps), _step_runs(entry, run), strict=True + ): + yield from _step_envelopes(started_id, step_id, result) + + yield _envelope( + cucumber.Envelope( + test_case_finished=cucumber.TestCaseFinished( + test_case_started_id=started_id, + timestamp=_timestamp(run.finished_ns), + will_be_retried=False, + ) + ) + ) + + +def _step_runs(entry: _Pickle, run: ScenarioRun) -> list[StepRun]: + """Every step of the pickle and both hooks, including what never ran. + + A scenario that stopped early -- the ordinary shape of both a failure and a + skip -- has fewer recorded results than the pickle has steps, and the + remainder are reported ``SKIPPED``, which is what Cucumber means by it and + what makes the stream account for the whole scenario rather than the part of + it that executed. + + The last thing this does is make sure the scenario's own verdict survives the + trip. A consumer reads a test case's outcome as the worst of its steps, so a + verdict no step accounts for would be lost: a strict ``xfail`` that passed is + a failed scenario every one of whose steps passed, and there are other ways + for a runner to fail a test case between its steps. Whatever is left over is + attached to the after-hook, which is where a test case failing outside its + own steps belongs. + """ + unreached = StepRun( + status=Status.skipped, + started_ns=run.finished_ns, + finished_ns=run.finished_ns, + ) + steps = list(run.steps[: len(entry.step_ids)]) + steps += [unreached] * (len(entry.step_ids) - len(steps)) + setup = run.setup or StepRun( + status=Status.passed, + started_ns=run.started_ns, + finished_ns=run.started_ns, + ) + teardown = run.teardown or StepRun( + status=Status.passed, + started_ns=run.finished_ns, + finished_ns=run.finished_ns, + ) + + reported = Status.unknown + for result in (setup, *steps, teardown): + reported = worse(reported, result.status) + if worse(reported, run.status) is not reported: + teardown = StepRun( + status=run.status, + message=run.message, + started_ns=teardown.started_ns, + finished_ns=teardown.finished_ns, + ) + return [setup, *steps, teardown] + + +def _step_envelopes( + started_id: str, step_id: str, result: StepRun +) -> typing.Iterator[dict[str, typing.Any]]: + yield _envelope( + cucumber.Envelope( + test_step_started=cucumber.TestStepStarted( + test_case_started_id=started_id, + test_step_id=step_id, + timestamp=_timestamp(result.started_ns), + ) + ) + ) + exception = ( + cucumber.Exception(type=result.exception_type, message=result.message or None) + if result.exception_type + else None + ) + yield _envelope( + cucumber.Envelope( + test_step_finished=cucumber.TestStepFinished( + test_case_started_id=started_id, + test_step_id=step_id, + test_step_result=cucumber.TestStepResult( + duration=_duration(result.finished_ns - result.started_ns), + status=result.status, + message=result.message or None, + exception=exception, + ), + timestamp=_timestamp(result.finished_ns), + ) + ) + ) + + +def _hook_envelopes() -> typing.Iterator[dict[str, typing.Any]]: + for hook_id, hook_type, name in ( + ( + _SETUP_HOOK_ID, + cucumber.HookType.before_test_case, + "provider-tck setup: capability gate, provider registration", + ), + ( + _TEARDOWN_HOOK_ID, + cucumber.HookType.after_test_case, + "provider-tck teardown: provider shutdown", + ), + ): + yield _envelope( + cucumber.Envelope( + hook=cucumber.Hook( + id=hook_id, + name=name, + type=hook_type, + source_reference=cucumber.SourceReference( + uri="openfeature/contrib/tools/provider_tck/plugin.py" + ), + ) + ) + ) + + +def _meta(implementation: str, implementation_version: str) -> cucumber.Meta: + """Who produced the stream, and against which protocol version.""" + return cucumber.Meta( + cpu=cucumber.Product(name=platform.machine() or _UNKNOWN_VERSION), + implementation=cucumber.Product( + name=implementation, version=implementation_version + ), + os=cucumber.Product(name=platform.system() or _UNKNOWN_VERSION), + protocol_version=_protocol_version(), + runtime=cucumber.Product( + name=platform.python_implementation(), version=platform.python_version() + ), + ) + + +def messages_protocol_version() -> str: + """The Messages release this stream was produced against. + + Exposed because the report envelope has to record it too. Messages is + versioned and the implementations pin different releases -- this one is on + 34.2.0 while the Go TCK builds against v21 -- so a consumer holding two + reports cannot assume one schema validates both. Sharing this one function + with the stream's own Meta message means the envelope and the stream cannot + disagree about which release produced it. + """ + return _protocol_version() + + +def _protocol_version() -> str: + """The Messages version this stream is written against. + + Read from the installed library rather than written down, because the library + is what decides: a version pinned here would go on claiming 34.2.0 after a + dependency bump moved the types underneath it. + """ + try: + return importlib.metadata.version(_MESSAGES_DISTRIBUTION) + except importlib.metadata.PackageNotFoundError: + return _UNKNOWN_VERSION + + +def _envelope(envelope: cucumber.Envelope) -> dict[str, typing.Any]: + converted: dict[str, typing.Any] = cucumber.message_converter.to_dict(envelope) + return converted + + +def _timestamp(nanoseconds: int) -> cucumber.Timestamp: + return cucumber.Timestamp( + seconds=nanoseconds // 1_000_000_000, nanos=nanoseconds % 1_000_000_000 + ) + + +def _duration(nanoseconds: int) -> cucumber.Duration: + nanoseconds = max(nanoseconds, 0) + return cucumber.Duration( + seconds=nanoseconds // 1_000_000_000, nanos=nanoseconds % 1_000_000_000 + ) + + +def feature_uri(relative_filename: str) -> str: + """Normalise pytest-bdd's relative feature path into a Cucumber uri. + + pytest-bdd builds it with ``os.path.join``, so on Windows it arrives + backslash-separated. A uri is slash-separated everywhere, and the same string + has to appear in the ``Source``, the ``GherkinDocument`` and every ``Pickle`` + or nothing ties them together -- so a report emitted on Windows would + otherwise not be comparable with one emitted on Linux. + """ + return PurePath(relative_filename).as_posix() diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py index b8b1a73b..c55cb6b9 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py @@ -4,7 +4,12 @@ all it takes for the step definitions to be available. pytest-bdd resolves steps through the fixture system and fixtures from an installed plugin are visible to every test, which is what keeps an adoption down to one fixture and one call to -:func:`tck_scenarios`. +``scenarios(*feature_paths())``. + +The same mechanism is what makes the suite extensible: a step an adopter defines +in their own ``conftest.py`` is resolved by the same fixture lookup as one this +plugin ships, so their scenarios need no glue and no second harness. See +:mod:`~.extensions`. """ from __future__ import annotations @@ -17,6 +22,7 @@ from .capability import Capability, capability_for_marker from .config import TckConfig +from .emitter import ReportEmitter, bind_scenario, observe_provider_name from .state import TckState # The step modules are registered as plugins in their own right, not merely @@ -32,22 +38,33 @@ def pytest_configure(config: pytest.Config) -> None: - """Register the capability tags as markers. + """Register the capability tags as markers, and the report emitter. pytest-bdd turns every Gherkin tag into a marker with ``getattr(pytest.mark, tag)`` without registering it, which raises ``PytestUnknownMarkWarning`` for each one -- noise at best, and a hard failure in a project configured with ``-W error``. + + The emitter is registered unconditionally even though it writes nothing + unless :data:`~.report.REPORT_DIR_ENV` is set. Accumulating the outcomes + costs a dictionary entry per scenario, and deciding at the end of the session + rather than at the start is one fewer way for a run to discover too late that + it was not recording. """ for capability in Capability: config.addinivalue_line( "markers", f"{capability.value}: OpenFeature provider TCK capability {capability.tag}", ) + config.pluginmanager.register( + ReportEmitter(config), "openfeature-provider-tck-report" + ) @pytest.fixture -def tck_state(tck_config: TckConfig) -> typing.Iterator[TckState]: +def tck_state( + request: pytest.FixtureRequest, tck_config: TckConfig +) -> typing.Iterator[TckState]: """Per-scenario state, carried between step definitions.""" # Resetting here rather than in an autouse fixture ties the reset to the # scenarios that actually use the TCK, and guarantees it happens after the @@ -56,11 +73,22 @@ def tck_state(tck_config: TckConfig) -> typing.Iterator[TckState]: tck_config.control.prepare_scenario() state = TckState(config=tck_config) yield state + # The provider is identified in the report by what it called itself, and the + # only thing that ever holds an instance is the scenario that made one. + observe_provider_name(request.config, tck_config, state.provider_name) state.teardown() @pytest.fixture(autouse=True) -def _tck_capability_gate(request: pytest.FixtureRequest) -> None: +def _tck_report_binding(request: pytest.FixtureRequest) -> None: + """Attribute this scenario to its suite before anything can skip it.""" + bind_scenario(request) + + +@pytest.fixture(autouse=True) +def _tck_capability_gate( + request: pytest.FixtureRequest, _tck_report_binding: None +) -> None: """Skip a scenario whose capability the provider did not declare. ``pytest.skip`` here reports the scenario as skipped **with the reason**, @@ -75,6 +103,11 @@ def _tck_capability_gate(request: pytest.FixtureRequest) -> None: Checking markers first also means the gate costs nothing, and instantiates nothing, for tests that are not TCK scenarios. + + ``_tck_report_binding`` is requested rather than left to autouse ordering so + that the scenario has reached its suite before this fixture can skip it. A + scenario skipped here is exactly the one the conformance report must account + for, and one that never reached a suite could not be reported at all. """ gated = [ capability diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py new file mode 100644 index 00000000..b08e2aca --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py @@ -0,0 +1,533 @@ +"""The conformance report envelope: what was tested, what it claims, where the results are. + +A run of the suite produces a pass or a fail on a terminal, which is enough for +the person who started it and useless to anyone else. The report is the same run +written down in a form something other than a human can read -- a comparison +page, an aggregator, a release gate -- against a schema owned by the +specification rather than by this package, so that four languages emit the same +document. + +This document no longer describes the results. It is an envelope that identifies +the subject and points at a :mod:`Cucumber Messages <.messages>` stream beside +it. The per-scenario outcome list, the outcome enum and the field naming which +Scenario Outline row an entry came from have all been deleted, because Messages +already carries every one of them -- along with the executed feature source, +which no bespoke format had. + +Two things stay here, because Messages has no slot for either. + +The **declaration** is an input to reading the results rather than a summary of +them. A skipped scenario in the stream says the question was not put to this +provider; only the declaration says whether that is because the provider +declines the capability. Given the declaration and a scenario's tags -- both +present -- the reason for a skip follows, so it no longer has to be transported +once per scenario. + +The **tested subject**: no standard results format has a slot for "the provider +under test". Messages records the runtime and the OS, which is what produced the +answers, not what was being asked about. + +See https://github.com/open-feature/spec/issues/424 for the format and +``specification/assets/provider-tck/report/`` for the schema. +""" + +from __future__ import annotations + +import importlib.metadata +import importlib.resources +import json +import re +import typing +from dataclasses import dataclass, field +from pathlib import Path + +from .config import TckConfig +from .messages import ( + MESSAGES_FORMAT, + ScenarioIdentity, + ScenarioRun, + StepRun, + messages_protocol_version, +) + +__all__ = [ + "REPORT_DIR_ENV", + "SCHEMA_VERSION", + "PhaseOutcome", + "ReportCollector", + "Results", + "SuiteReport", + "envelope_file_name", + "stream_file_name", +] + +REPORT_DIR_ENV = "PROVIDER_TCK_REPORT_DIR" +"""Names the directory a conformance report is written to. + +An environment variable rather than a :class:`~.config.TckConfig` field, so that +emitting a report is a property of the *run* and not of the code: CI sets it, a +developer running the suite locally does not, and no adopter changes a line to +publish one. Each suite writes two files -- ``/.json``, the envelope, +and ``/.ndjson``, the results the envelope points at -- so several +suites in one pytest session, flagd's RPC and in-process resolvers say, each +produce their own pair without colliding. + +Unset means no report, which is the default and is not an error. +""" + +SCHEMA_VERSION = "1" +"""The major version of the report schema this emitter produces.""" + +TCK_IMPLEMENTATION = "python-sdk-contrib/tools/openfeature-provider-tck" +"""Which TCK implementation produced the report, as the schema spells it.""" + +PROVIDER_LANGUAGE = "python" + +SDK_DISTRIBUTION = "openfeature-sdk" +TCK_DISTRIBUTION = "openfeature-provider-tck" + +UNKNOWN = "unknown" +"""Stands in for an identity that could not be read. + +Seven characters, which is the schema's minimum for ``tck.specRevision``, so a +build that could not reach git still emits a document that validates and says +plainly that it does not know rather than inventing a commit. +""" + +_PACKAGE = "openfeature.contrib.tools.provider_tck" + +_REVISION_FILE = "spec_revision.json" +"""Written at build time from the spec submodule; see ``hatch_build_sync.py``. + +Read from a data file rather than the submodule because the submodule is not in +the published wheel: an adopter installing this package has no ``spec/`` +directory to interrogate, and the revision the assets came from is exactly what +the report has to name. +""" + +_TAG_PATTERN = re.compile(r"^[a-z0-9-]+$") +"""What the schema accepts as a tag, minus the leading at-sign. + +Tags that do not match are dropped rather than emitted, because an invalid +document helps nobody; the canonical feature files carry none, so this only bites +a feature file that has been forked, which is itself worth noticing. +""" + +_UNSAFE_IN_FILENAME = re.compile(r"[^A-Za-z0-9._-]") + + +@dataclass(frozen=True) +class Results: + """Where the executed results are, and what covers them.""" + + location: str + digest: str + format: str = MESSAGES_FORMAT + + def as_json(self) -> dict[str, typing.Any]: + # The format's version is recorded alongside its name because Cucumber + # Messages is versioned and the four implementations pin different + # releases. Without it a consumer validating this stream has to guess + # which schema to use, and guessing wrong is worse than not checking: a + # later schema accepts messages this producer could not have emitted, + # and an earlier one rejects messages that are perfectly valid. + document = { + "format": self.format, + "formatVersion": messages_protocol_version(), + "location": self.location, + } + if self.digest: + document["digest"] = self.digest + return document + + +@dataclass(frozen=True) +class PhaseOutcome: + """One pytest phase report, reduced to what the conformance report needs. + + Reduced rather than kept, because a :class:`pytest.TestReport` holds a + formatted traceback and holding a session's worth of them to classify at the + end would be a memory leak with a nice name. + """ + + when: str + """``setup``, ``call`` or ``teardown``.""" + + outcome: str + """``passed``, ``failed`` or ``skipped``, as pytest decided.""" + + xfail_reason: str | None = None + """Set when pytest marked this an expected failure.""" + + message: str = "" + """The skip reason, or the failure's headline, already trimmed.""" + + start: float = 0.0 + stop: float = 0.0 + + +Resolver = typing.Callable[ + [ScenarioIdentity, "list[PhaseOutcome]", "list[StepRun]"], ScenarioRun +] +"""Turns what pytest reported about one scenario into what the stream records. + +A callable rather than a method, because the mapping is entirely about pytest -- +which phase means what, and that an expected failure is still a failure -- and +this module deliberately knows nothing about pytest. +""" + + +@dataclass +class SuiteReport: + """What one suite -- one :class:`~.config.TckConfig` -- accumulates as it runs. + + Runs are keyed by pytest node id rather than appended to a list, which is + what makes "every scenario appears exactly once" a property of the structure + instead of a promise made by the code that fills it. + """ + + config: TckConfig + provider_name: str | None = None + runs: dict[str, ScenarioRun] = field(default_factory=dict) + + def observe_provider_name(self, name: str) -> None: + """Remember what the provider called itself through its own metadata. + + Last one wins, and they should all agree: a suite tests one provider. + """ + if name: + self.provider_name = name + + def record(self, node_id: str, run: ScenarioRun) -> None: + self.runs[node_id] = run + + @property + def sorted_runs(self) -> list[ScenarioRun]: + """The runs in a stable order: feature, then scenario, then row. + + Sorted by the whole identity, the Examples row included, so that two rows + of one outline reach the stream in the feature file's terms rather than + in whichever order the dictionary happened to be filled. + """ + return sorted( + self.runs.values(), + key=lambda run: ( + run.identity.uri, + run.identity.name, + run.identity.example, + ), + ) + + def counts(self) -> dict[str, int]: + """Status tallies, for a log line and for the tests that check them.""" + tally: dict[str, int] = {} + for run in self.runs.values(): + key = run.status.value.lower() + tally[key] = tally.get(key, 0) + 1 + return tally + + def build(self, results: Results) -> dict[str, typing.Any]: + """Assemble the envelope around a results payload already written.""" + document: dict[str, typing.Any] = { + "schemaVersion": SCHEMA_VERSION, + "provider": { + # What the provider calls itself, not the suite name: the suite + # name is chosen to read well in a failure message -- "flagd-rpc" + # -- which makes it the configuration, and it is reported as one. + # A provider with two materially different modes therefore + # produces two reports that are not interchangeable. + "name": self.provider_name or self.config.name, + "language": PROVIDER_LANGUAGE, + "configuration": self.config.name, + }, + "sdk": { + "name": SDK_DISTRIBUTION, + "version": distribution_version(SDK_DISTRIBUTION), + }, + "tck": { + "implementation": TCK_IMPLEMENTATION, + "version": distribution_version(TCK_DISTRIBUTION), + "specRevision": spec_revision(), + }, + "declaration": self._declaration(), + "results": results.as_json(), + } + + backend = self._backend() + if backend: + document["backend"] = backend + deviations = [deviation.as_json() for deviation in self.config.known_deviations] + if deviations: + # Omitted rather than emitted empty: stating no deviations is a + # claim, and an emitter that always emitted the field would make that + # claim on every provider's behalf whether or not it had checked. + document["knownDeviations"] = deviations + return document + + def _declaration(self) -> dict[str, typing.Any]: + """What the provider claims, which is what makes a skip legible. + + The declared set and the not-applicable set are disjoint and mean + different things -- a choice against a capability, and an impossibility. + :class:`~.config.TckConfig` refuses a configuration that puts a + capability in both, so a consumer never has to decide which one wins. + + Neither set is filtered here. A reserved capability -- one no scenario + carries, which the schema forbids in this block -- cannot be in a + ``TckConfig`` at all: it is refused at construction, and the default + capability set excludes it. Dropping one silently at emission time would + make a rejected configuration look like an accepted one, and leave the + adopter who wrote it believing the declaration they read back was the + declaration they asked for. + """ + declaration: dict[str, typing.Any] = { + "declared": self.config.sorted_capabilities + } + not_applicable = { + capability.tag: reason + for capability, reason in sorted( + self.config.not_applicable.items(), key=lambda item: item[0].tag + ) + } + if not_applicable: + declaration["notApplicable"] = not_applicable + return declaration + + def _backend(self) -> dict[str, typing.Any]: + backend: dict[str, typing.Any] = {} + description = getattr(self.config.control, "description", "") + if isinstance(description, str) and description: + backend["description"] = description + control_api = control_api_of(self.config.control) + if control_api: + backend["controlApi"] = control_api + return backend + + +class ReportCollector: + """Session-wide accumulator: which scenario belongs to which suite, and how it went. + + One pytest session can run several suites -- the TCK's own tests run two, and + a provider with more than one resolver runs one per resolver -- so outcomes + are attributed to a suite rather than to the session, and each suite writes + its own pair of files. + + Scenarios are enumerated at collection and resolved into runs only at the end + of the session. The order matters. A scenario skipped by a marker never runs + a fixture, so a design that learned of a scenario when its fixtures ran would + leave it out of the stream entirely -- and a report that silently omits what + it skipped satisfies "a skip is never reported as passed" while still + misleading the person reading it. + """ + + def __init__(self) -> None: + # Suites are keyed by the identity of their TckConfig, so two suites that + # happen to share a name stay distinct here; that collision is caught + # where it actually bites, when their file names turn out to be equal. + self._suites: dict[int, SuiteReport] = {} + self._suite_by_group: dict[str, SuiteReport] = {} + self._collected: dict[str, tuple[str, ScenarioIdentity]] = {} + self._phases: dict[str, list[PhaseOutcome]] = {} + self._steps: dict[str, list[StepRun]] = {} + + def collect(self, node_id: str, group: str, identity: ScenarioIdentity) -> None: + """Note that this scenario exists, and which group of tests it came from. + + The group is the module the scenario was generated into. pytest-bdd's + ``scenarios()`` injects its tests into the module that called it, and a + module resolves one ``tck_config``, so the module is what says which + suite a scenario belongs to -- and it says so without running anything. + """ + self._collected[node_id] = (group, identity) + + @property + def identities(self) -> list[ScenarioIdentity]: + """Every collected scenario, so the feature files can be parsed once.""" + return [identity for _, identity in self._collected.values()] + + def observe(self, node_id: str, phase: PhaseOutcome) -> None: + """Record one phase's result for a scenario, if it is one of ours.""" + if node_id in self._collected: + self._phases.setdefault(node_id, []).append(phase) + + def observe_step(self, node_id: str, step: StepRun) -> None: + """Record what one Gherkin step did, in execution order. + + Recorded as it happens rather than reconstructed afterwards, because + pytest reports a scenario and not its steps: only the runner knows which + step of eight failed, and a stream that marked all eight failed would be + saying something untrue about seven of them. + """ + self._steps.setdefault(node_id, []).append(step) + + def bind(self, node_id: str, config: TckConfig) -> None: + """Learn which suite a group of scenarios is testing. + + Called from a fixture, because the ``TckConfig`` is a fixture value and + there is no way to know it without asking for it. Only one scenario of a + group has to get this far for the whole group to be attributed. + """ + entry = self._collected.get(node_id) + if entry is not None: + self._suite_by_group[entry[0]] = self.suite_for(config) + + def suite_for(self, config: TckConfig) -> SuiteReport: + return self._suites.setdefault(id(config), SuiteReport(config=config)) + + @property + def suites(self) -> list[SuiteReport]: + return list(self._suites.values()) + + def resolve(self, resolver: Resolver) -> list[str]: + """Turn the collected phases into runs, and report what could not be. + + Returns the problems, one string each, and they are meant to be shouted + about rather than logged: a scenario that ran but is missing from the + stream is the one failure mode this format exists to rule out. + """ + problems: list[str] = [] + for node_id, (group, identity) in sorted(self._collected.items()): + suite = self._suite_by_group.get(group) + if suite is None: + problems.append( + f"{node_id}: no TckConfig was resolved for {group}, so its " + f"outcome belongs to no suite and is missing from every report" + ) + continue + phases = self._phases.get(node_id) + if not phases: + problems.append( + f"{node_id}: was collected but never ran, so the report for " + f"{suite.config.name!r} does not account for it" + ) + continue + suite.record( + node_id, resolver(identity, phases, self._steps.get(node_id, [])) + ) + return problems + + +def control_api_of(control: object) -> str: + """Report how the backend was driven, if the control says. + + Read off an optional attribute rather than added to the + :class:`~.control.BackendControl` protocol, because a protocol member would + make every existing control incomplete for the sake of one string. A control + that does not offer it simply omits the field, which is the honest answer: + the TCK cannot infer from the outside whether a control spoke the normative + HTTP API or reached into the process. + """ + value = getattr(control, "control_api", None) + if isinstance(value, str) and value in {"http", "in-process"}: + return value + return "" + + +def control_api_gap(control: object) -> str: + """Describe a control that does not say how it drives the backend, or "". + + The field is optional in the report and the attribute is optional here, both + so that introducing it made no existing control incomplete. Together they + make omission invisible: the suite passes, the report validates, and the + field is simply absent. That went unnoticed until reports from four + languages were compared side by side and two of them were silent about the + same kind of in-process backend. + + Every control either drives a real backend over the normative HTTP API or + manipulates one in this process, so there is no third case an absent value + legitimately describes -- which makes the silence worth breaking, in the run + output where an adopter will see it rather than in the report where they + will not. + """ + if control_api_of(control): + return "" + description = getattr(control, "description", "") or type(control).__name__ + return ( + f"{description} does not offer a control_api attribute, so the conformance " + f"report cannot say whether the backend was driven over HTTP or in process. " + f'Add one, returning "http" or "in-process".' + ) + + +def normalise_tags(tags: typing.Iterable[str]) -> tuple[str, ...]: + """Turn Gherkin tags as pytest-bdd holds them into the form the schema wants. + + pytest-bdd strips the leading at-sign; the schema requires it back. + """ + return tuple(sorted(f"@{tag}" for tag in tags if _TAG_PATTERN.match(tag))) + + +def report_stem(suite_name: str) -> str: + """Turn a suite name into a file name stem. + + Suite names are chosen to read well in a failure message rather than to be + path-safe, so anything not obviously safe becomes a hyphen. Without this a + suite named ``flagd/rpc`` would quietly write outside the directory it was + given. + """ + cleaned = _UNSAFE_IN_FILENAME.sub("-", suite_name).strip("-.") + return cleaned or "report" + + +def envelope_file_name(suite_name: str) -> str: + return f"{report_stem(suite_name)}.json" + + +def stream_file_name(suite_name: str) -> str: + """The results payload, beside the envelope that points at it. + + A sibling rather than a subdirectory so that ``results.location`` is a bare + file name, which is a relative reference that survives the whole pair being + moved, uploaded or served from somewhere other than where it was written. + """ + return f"{report_stem(suite_name)}.ndjson" + + +def write_envelope( + directory: Path, suite_name: str, document: dict[str, typing.Any] +) -> Path: + """Write one envelope, returning where it went.""" + directory.mkdir(parents=True, exist_ok=True) + path = directory / envelope_file_name(suite_name) + path.write_text( + json.dumps(document, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" + ) + return path + + +def distribution_version(distribution: str) -> str: + """Read an installed distribution's version. + + Read rather than declared, because a declared version is a second place to + be wrong: the report would go on claiming 0.8.2 after a dependency bump moved + the actual code underneath it. + """ + try: + return importlib.metadata.version(distribution) + except importlib.metadata.PackageNotFoundError: + return UNKNOWN + + +def spec_revision() -> str: + """Return the spec commit these feature files came from. + + Captured at build time rather than read here, because the submodule that + holds the answer is not in the wheel. A build that could not reach git says + so with :data:`UNKNOWN` instead of inventing a commit, and an installation + old enough to predate the generated file degrades the same way rather than + failing to emit a report at all. + + The asset tree hash that used to accompany it is gone. It was carried so that + a consumer could tell whether two runs executed the same questions; the + results payload now carries the executed feature source itself, which answers + that directly rather than by proxy. + """ + reference = importlib.resources.files(_PACKAGE) / _REVISION_FILE + try: + data = json.loads(reference.read_text(encoding="utf-8")) + except (OSError, ValueError): + return UNKNOWN + if not isinstance(data, dict): + return UNKNOWN + revision = data.get("specRevision") + return revision if isinstance(revision, str) and len(revision) >= 7 else UNKNOWN diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/state.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/state.py index 71ea4150..1b1faa86 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/state.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/state.py @@ -93,6 +93,13 @@ class TckState: config: TckConfig client: OpenFeatureClient | None = None + provider_name: str | None = None + """What the provider called itself through its own metadata. + + Observed rather than configured, because it is what the conformance report + identifies the provider by: ``TckConfig.name`` is chosen to read well in a + failure message, which makes it the *configuration* rather than the provider. + """ flag_key: str | None = None flag_type: FlagType | None = None default_value: typing.Any = None diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/provider_steps.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/provider_steps.py index bca37fae..59520879 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/provider_steps.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/provider_steps.py @@ -30,6 +30,7 @@ def a_stable_provider(tck_state: TckState) -> None: if provider is None: msg = "TckConfig.new_provider returned None" raise AssertionError(msg) + _observe_metadata_name(tck_state, provider) try: _set_provider_within(provider, config.domain, config.ready_timeout) @@ -79,6 +80,7 @@ def an_unavailable_provider(tck_state: TckState) -> None: if provider is None: msg = "TckConfig.new_unavailable_provider returned None" raise AssertionError(msg) + _observe_metadata_name(tck_state, provider) # A raising initialize is already converted to PROVIDER_ERROR by the SDK's # registry, so this is belt and braces: a provider that raises anyway must @@ -90,6 +92,21 @@ def an_unavailable_provider(tck_state: TckState) -> None: tck_state.client = api.get_client(config.domain) +def _observe_metadata_name(tck_state: TckState, provider: FeatureProvider) -> None: + """Note what the provider calls itself, for the conformance report. + + Before registration rather than after, so that a provider which fails to + initialise -- the ``@unavailable`` case, and any genuine failure -- is still + identified in the report by its own name. Metadata is a pure accessor by + contract, but a provider that raises from it must not take the scenario down + with it: the name is for a report, and no scenario asserts on it. + """ + with contextlib.suppress(Exception): + name = provider.get_metadata().name + if name: + tck_state.provider_name = name + + def _set_provider_within( provider: FeatureProvider, domain: str, timeout: float ) -> None: diff --git a/tools/openfeature-provider-tck/tests/conftest.py b/tools/openfeature-provider-tck/tests/conftest.py index a5e6726f..47e88028 100644 --- a/tools/openfeature-provider-tck/tests/conftest.py +++ b/tools/openfeature-provider-tck/tests/conftest.py @@ -3,35 +3,66 @@ A conformance suite that quietly goes green on scenarios it did not run is worse than no suite at all -- and the same is true of one that quietly goes green on a scenario it *did* run and fail. So the one scenario the Python SDK cannot -currently satisfy is marked ``xfail(strict=True)`` here, which: +currently satisfy is recorded twice, in two forms that answer different +questions. -* keeps it visible in the report, as XFAIL with the reason attached; -* fails the suite if it ever *passes*, so the marker is removed the moment the - SDK is fixed rather than lingering as a lie. +``xfail(strict=True)`` keeps the *run* honest: the scenario is expected to fail, +and the suite fails if it ever passes, so the marker is removed the moment the +SDK is fixed rather than lingering as a lie. -This lives in the TCK's own self-test rather than in the shared package. It is a -fact about the SDK under test, not part of the conformance definition, and -Appendix F deliberately leaves a general "known deviations" concept as an open -question (spec#417, Q4). If that concept lands, this moves into it. +:class:`KnownDeviation` keeps the *report* honest. The results payload reports +the scenario as failed regardless of the marker -- an expected failure is still a +failure, and softening it there would hide exactly what the marker exists to keep +visible -- and the envelope carries the acknowledgement beside it, with the issue +it is tracked under. That is what lets a consumer tell a known and tracked gap +from a surprise without the result itself being weakened. + +The two are declared together here so they cannot drift: the reason on the marker +and the summary in the report are the same sentence. """ from __future__ import annotations import pytest +from openfeature.contrib.tools.provider_tck import KnownDeviation + # The Scenario Outline row that asks for boolean-flag as an Integer. _BOOL_AS_INT = ( "test_requesting_the_wrong_type_returns_the_code_default[boolean-flag-Integer-1]" ) +_ISSUE = "https://github.com/open-feature/python-sdk/issues/619" + _REASON = ( "python-sdk: a boolean satisfies an Integer request. The client type-checks with " "isinstance(value, int) and bool is a subclass of int in Python, so boolean-flag " "requested as an Integer returns True with reason STATIC and no error code, where " "the specification requires the code default and TYPE_MISMATCH. " - "See https://github.com/open-feature/python-sdk/issues/619" + f"See {_ISSUE}" ) +KNOWN_DEVIATIONS = (KnownDeviation(issue=_ISSUE, summary=_REASON),) +"""What the report acknowledges. + +No ``capability``: the scenario carries no capability tag, because returning the +code default on a type mismatch is mandatory. ``@numeric-coercion`` is a +neighbouring question -- whether 0.5 satisfies an integer request -- and this +provider satisfies it, so attributing the deviation there would be wrong twice +over. +""" + + +@pytest.fixture(scope="session") +def tck_known_deviations() -> tuple[KnownDeviation, ...]: + """The deviations a suite in this package declares. + + A fixture rather than an import so that the marker below and the report's + acknowledgement are written down once, in one place, and a suite picks it up + the same way it picks up everything else it is given. + """ + return KNOWN_DEVIATIONS + def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: for item in items: diff --git a/tools/openfeature-provider-tck/tests/test_canonical_set.py b/tools/openfeature-provider-tck/tests/test_canonical_set.py new file mode 100644 index 00000000..21a5f4d5 --- /dev/null +++ b/tools/openfeature-provider-tck/tests/test_canonical_set.py @@ -0,0 +1,342 @@ +"""That a run which executed less than the canonical set says so, and publishes nothing. + +The capability gate rules out the loud way a conformance suite can go green on +scenarios it did not run: an undeclared capability is reported as skipped, with +its reason. Nothing ruled out the quiet way, where the scenarios were never +collected at all. ``-k``, ``-m``, ``--deselect``, a test module that stopped +calling ``scenarios()`` on the canonical path -- each runs less of the suite, and +none of them is an error to pytest. + +Go measured the consequence: ``-run`` on a single scenario passed green and +emitted a well-formed report covering one of twenty-nine canonical scenarios. +Nothing in that document said so, and nothing reading it could have known. + +So the properties here are about what a report is allowed to be written from. +Everything that has to be checked end to end is, because the question is about a +whole pytest session rather than about what a function returns. +""" + +from __future__ import annotations + +import dataclasses +import json +import os +import subprocess +import sys +import typing +from pathlib import Path + +import pytest + +from openfeature.contrib.tools.provider_tck import ( + EXTENSIONS_DIRECTORY, + PARTIAL_ENV, + REPORT_DIR_ENV, +) +from openfeature.contrib.tools.provider_tck.canonical import ( + canonical_scenarios, + describe, + missing_canonical, + partial_run_allowed, +) +from openfeature.contrib.tools.provider_tck.extensions import is_canonical_uri +from openfeature.contrib.tools.provider_tck.messages import ( + ScenarioIdentity, + ScenarioRun, +) + +SUITE_NAME = "guarded" + +_SUITE_MODULE = '''\ +"""A one-fixture adoption, generated so a partial run can be checked end to end.""" + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.provider_tck import ( + Capability, + InProcessControl, + TckConfig, + feature_paths, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="guarded", + control=control, + new_provider=control.new_provider, + capabilities={Capability.EVENTS, Capability.OBJECT}, + ) + + +scenarios(*feature_paths()) +''' + +_CONFTEST_MODULE = """\ +import pytest + +DEVIATION = "[boolean-flag-Integer-1]" + + +def pytest_collection_modifyitems(items): + for item in items: + if item.name.endswith(DEVIATION): + item.add_marker(pytest.mark.xfail(reason="python-sdk#619")) +""" + +# Three scenarios of an adopter's own, so that a run which drops one canonical +# scenario still executes more scenarios than the canonical set contains. The +# count is what makes "an extension cannot close a gap" checkable rather than +# asserted. +_VENDOR_FEATURE = """\ +Feature: Vendor rules + + Background: + Given a stable provider + + Scenario: A vendor rule resolves + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" + + Scenario: A vendor rule resolves again + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" + + Scenario: And once more + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" +""" + +# The canonical scenario the filtered runs below leave out. Named rather than +# counted, so a change to the canonical assets cannot leave these passing while +# they select nothing. +EXCLUDED_SELECTOR = "unknown_flag_key" +EXCLUDED_SCENARIO = "An unknown flag key returns the code default" + + +@dataclasses.dataclass(frozen=True) +class Run: + """One subprocess run of the generated adoption.""" + + reports: Path + result: subprocess.CompletedProcess[str] + + @property + def stdout(self) -> str: + return self.result.stdout + + @property + def envelopes(self) -> list[Path]: + return sorted(self.reports.glob("*.json")) + + +def _run( + tmp_path: Path, + *arguments: str, + partial: bool = False, + extension: bool = False, +) -> Run: + directory = tmp_path / "adoption" + directory.mkdir(parents=True, exist_ok=True) + (directory / "test_guarded.py").write_text(_SUITE_MODULE, encoding="utf-8") + (directory / "conftest.py").write_text(_CONFTEST_MODULE, encoding="utf-8") + if extension: + features = directory / EXTENSIONS_DIRECTORY + features.mkdir(parents=True, exist_ok=True) + (features / "vendor.feature").write_text(_VENDOR_FEATURE, encoding="utf-8") + + reports = tmp_path / "reports" + environment = dict(os.environ) + environment[REPORT_DIR_ENV] = str(reports) + if partial: + environment[PARTIAL_ENV] = "1" + else: + environment.pop(PARTIAL_ENV, None) + + result = subprocess.run( # noqa: S603 + [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "no:cacheprovider", + str(directory), + *arguments, + ], + capture_output=True, + text=True, + env=environment, + check=False, + ) + return Run(reports=reports, result=result) + + +def _identity(uri: str, name: str, path: Path) -> ScenarioIdentity: + return ScenarioIdentity(uri=uri, path=path, name=name, tags=()) + + +# -- a run that executed the whole set --------------------------------------- + + +@pytest.fixture(scope="module") +def complete(tmp_path_factory: pytest.TempPathFactory) -> Run: + """The unfiltered run, which is what everything else is measured against.""" + return _run(tmp_path_factory.mktemp("complete"), extension=True) + + +def test_the_whole_canonical_set_still_writes_a_report(complete: Run) -> None: + """Including the scenarios the capability gate skipped. + + A gated skip *ran*: it was asked, and the report accounts for it with a + reason. Treating it as missing would make the guard contradict the one rule + the suite is built around. + """ + assert complete.result.returncode == 0, complete.stdout + assert [path.name for path in complete.envelopes] == [f"{SUITE_NAME}.json"] + assert "canonical scenarios did not run" not in complete.stdout + # This adoption declares neither @stale nor @lifecycle, so some canonical + # scenarios were skipped -- which is the case being asserted about. + assert " skipped" in complete.stdout + + +# -- and one that did not ---------------------------------------------------- + + +def test_a_filtered_run_fails_and_writes_no_report(tmp_path: Path) -> None: + """The Go hazard, at the point it would have produced the document. + + A well-formed report covering one scenario of twenty-nine is worse than no + report, because nothing in it says which twenty-eight were never asked. + """ + run = _run(tmp_path, "-k", EXCLUDED_SELECTOR) + + assert run.result.returncode != 0, run.stdout + assert "of 29 canonical scenarios did not run" in run.stdout + assert not run.envelopes, "a partial run must publish nothing" + assert not list(run.reports.glob("*.ndjson")) + # The message names scenarios rather than only counting them, and says how + # to filter deliberately. + assert EXCLUDED_SCENARIO not in run.stdout, "that one is the scenario that ran" + assert PARTIAL_ENV in run.stdout + + +def test_acknowledging_a_partial_run_does_not_make_it_publishable( + tmp_path: Path, +) -> None: + """``PROVIDER_TCK_PARTIAL`` buys a green run, never a document. + + Someone working on one scenario should not have to fight the guard; nobody + should be able to turn a partial run into a conformance claim. Those are + different requests, and only the first is granted. Java's TCK spells the + same escape hatch the same way. + """ + run = _run(tmp_path, "-k", EXCLUDED_SELECTOR, partial=True) + + assert run.result.returncode == 0, run.stdout + assert "canonical scenarios did not run" in run.stdout + assert f"{PARTIAL_ENV} is set" in run.stdout + assert not run.envelopes, "acknowledged or not, it is not a conformance run" + + +def test_an_extension_cannot_close_a_gap(tmp_path: Path) -> None: + """Three extension scenarios do not make up for one canonical one. + + The adoption below runs thirty-one scenarios where the canonical set has + twenty-nine, and is still one short: an adopter's scenarios are theirs, and + counting them towards the specification's set would let any gap be filled by + adding a feature file. + """ + run = _run(tmp_path, "-k", f"not {EXCLUDED_SELECTOR}", extension=True) + + assert run.result.returncode != 0, run.stdout + assert "1 of 29 canonical scenarios did not run" in run.stdout + assert f"features/errors.feature: {EXCLUDED_SCENARIO}" in run.stdout + assert not run.envelopes + + +# -- what the canonical set is ----------------------------------------------- + + +def test_the_canonical_set_is_read_from_the_packaged_assets() -> None: + """One entry per Scenario Outline row, which is what a runner generates.""" + scenarios = canonical_scenarios() + assert len(scenarios) == 29 + assert all(is_canonical_uri(uri) for uri, _, _ in scenarios) + # An outline contributes rows, not a single templated entry. + assert any(row for _, _, row in scenarios) + + +def test_a_run_of_nothing_is_missing_everything() -> None: + assert set(missing_canonical([])) == canonical_scenarios() + + +def test_an_extension_run_is_neither_counted_nor_blamed(tmp_path: Path) -> None: + """Matched by path rather than by uri. + + The uri is derived, and a check that rested on the same derivation it exists + to corroborate would be checking its own arithmetic. + """ + vendor = ScenarioRun( + identity=_identity( + "extensions/vendor.feature", + "A vendor rule resolves", + tmp_path / EXTENSIONS_DIRECTORY / "vendor.feature", + ) + ) + # Even one that claims a canonical uri outright -- which the report refuses + # separately -- cannot reduce the missing set. + impostor = ScenarioRun( + identity=_identity( + "features/errors.feature", + EXCLUDED_SCENARIO, + tmp_path / "features" / "errors.feature", + ) + ) + assert set(missing_canonical([vendor, impostor])) == canonical_scenarios() + + +def test_a_missing_scenario_is_described_by_its_row() -> None: + assert describe(("features/x.feature", "A scenario", ())) == ( + "features/x.feature: A scenario" + ) + assert describe(("features/x.feature", "An outline", (("key", "a"),))) == ( + "features/x.feature: An outline [key=a]" + ) + + +@pytest.mark.parametrize( + ("value", "allowed"), + [("1", True), ("true", True), ("TRUE", True), ("yes", True), ("on", True)], +) +def test_the_acknowledgement_is_read_generously(value: str, allowed: bool) -> None: + assert partial_run_allowed({PARTIAL_ENV: value}) is allowed + + +@pytest.mark.parametrize("value", ["", "0", "false", "no", " ", "maybe"]) +def test_anything_else_is_not_an_acknowledgement(value: str) -> None: + """Including a typo: the default has to be the safe one.""" + assert partial_run_allowed({PARTIAL_ENV: value}) is False + assert partial_run_allowed({}) is False + + +def test_the_envelope_of_a_complete_run_is_unchanged(complete: Run) -> None: + """The guard withholds a report; it does not add anything to one.""" + envelope: dict[str, typing.Any] = json.loads( + complete.envelopes[0].read_text(encoding="utf-8") + ) + assert set(envelope) == { + "schemaVersion", + "provider", + "sdk", + "tck", + "backend", + "declaration", + "results", + } diff --git a/tools/openfeature-provider-tck/tests/test_controllable_conformance.py b/tools/openfeature-provider-tck/tests/test_controllable_conformance.py index 77b3c3d0..a9f0db5f 100644 --- a/tools/openfeature-provider-tck/tests/test_controllable_conformance.py +++ b/tools/openfeature-provider-tck/tests/test_controllable_conformance.py @@ -20,13 +20,14 @@ from openfeature.contrib.tools.provider_tck import ( Capability, InProcessControl, + KnownDeviation, TckConfig, features_path, ) @pytest.fixture(scope="session") -def tck_config() -> TckConfig: +def tck_config(tck_known_deviations: tuple[KnownDeviation, ...]) -> TckConfig: """Declare the provider under test and what it can do. ``STALE`` and ``UNAVAILABLE_INIT`` stay undeclared: there is still no @@ -50,6 +51,9 @@ def tck_config() -> TckConfig: Capability.OBJECT, Capability.NUMERIC_COERCION, }, + # The same SDK bug, against the same issue: it is a defect in the client + # rather than in either provider, so both suites acknowledge it. + known_deviations=tck_known_deviations, ) diff --git a/tools/openfeature-provider-tck/tests/test_extensions.py b/tools/openfeature-provider-tck/tests/test_extensions.py new file mode 100644 index 00000000..3789c68d --- /dev/null +++ b/tools/openfeature-provider-tck/tests/test_extensions.py @@ -0,0 +1,699 @@ +"""What an adopter's own scenarios may and may not do. + +An adopter with provider-specific behaviour -- flagd's ``fractional`` targeting, +a proprietary rollout rule -- has to be able to pin it in the same run as the +contract it sits on top of, or they end up maintaining a second harness beside +the one the TCK gives them. So the properties checked here are the ones that make +that safe rather than merely possible: + +* an extension scenario runs **inside** the canonical suite -- same session, same + provider, same report -- with a step definition the adopter wrote in their own + ``conftest.py`` and nothing else registered; +* an adoption without extensions runs exactly what it ran before, scenario for + scenario and field for field; +* an extension can neither replace a canonical scenario nor be reported as one. + +The last is not hypothetical. Java's suite discovered a same-named feature file +in a second classpath root silently *replacing* the canonical one, and the run +went green having asked the adopter's questions instead of the specification's. +The Python route to the same place is narrower and just as quiet: pytest-bdd +names a feature file by its parent directory joined to its own name, so a file at +``tck-extensions/features/errors.feature`` arrives under the uri the canonical +``errors.feature`` already occupies. + +Most of this is checked against real pytest sessions in subprocesses, because +every one of the properties is about how a whole session runs rather than about +what a function returns. +""" + +from __future__ import annotations + +import collections +import dataclasses +import json +import os +import subprocess +import sys +import typing +from pathlib import Path + +import pytest + +from openfeature.contrib.tools.provider_tck import ( + EXTENSIONS_DIRECTORY, + REPORT_DIR_ENV, + feature_paths, + features_path, +) +from openfeature.contrib.tools.provider_tck.extensions import ( + is_canonical_uri, + reserved_prefix_problem, + uri_collisions, + uri_for, +) + +CANONICAL_FEATURE = "errors.feature" +"""The canonical file the shadowing fixture copies, chosen because it is the one +whose scenarios an extension could most plausibly want to restate.""" + +VENDOR_URI = "extensions/vendor.feature" +VENDOR_SCENARIO = "A vendor rule resolves through the suite's own provider" + + +# -- the generated adoption -------------------------------------------------- + +_SUITE_MODULE = '''\ +"""A one-fixture adoption, generated so extensions can be checked end to end.""" + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.provider_tck import ( + Capability, + InProcessControl, + TckConfig, + feature_paths, + features_path, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="{name}", + control=control, + new_provider=control.new_provider, + capabilities={{Capability.EVENTS, Capability.OBJECT}}, + ) + + +scenarios({call}) +''' + +_EXTENSION_CALL = "*feature_paths()" +_CANONICAL_CALL = "features_path()" + +# The step the adopter writes, in the adopter's own conftest.py and nowhere else. +# It asks the TCK's own per-scenario state what provider this scenario is running +# against, which is what makes "the same backend lifecycle" checkable rather than +# asserted: a second harness would have a second provider, or none. +_CONFTEST_MODULE = """\ +import pytest +from pytest_bdd import then + +from openfeature.contrib.tools.provider_tck import TckState + +DEVIATION = "[boolean-flag-Integer-1]" + + +@then("the vendor rule ran against the provider the suite registered") +def vendor_rule_ran(tck_state: TckState) -> None: + assert tck_state.client is not None, "no provider was registered" + assert tck_state.provider_name == "In-Memory Provider", tck_state.provider_name + + +def pytest_collection_modifyitems(items): + for item in items: + if item.name.endswith(DEVIATION): + item.add_marker(pytest.mark.xfail(reason="python-sdk#619")) +""" + +# Deliberately reuses the canonical step vocabulary and adds exactly one step of +# its own, which is the shape an adopter's feature file actually takes. +_VENDOR_FEATURE = """\ +Feature: Vendor rules + + Background: + Given a stable provider + + Scenario: A vendor rule resolves through the suite's own provider + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" + And the vendor rule ran against the provider the suite registered +""" + +# The same Feature name and the same Scenario name as the file above, with a +# different step list -- so that a run reporting one against the other's source +# would be wrong in a way nothing downstream could notice. +_NESTED_FEATURE = """\ +Feature: Vendor rules + + Background: + Given a stable provider + + Scenario: A vendor rule resolves through the suite's own provider + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" +""" + +# An adopter's own directory named `features`, which is the one way a +# non-canonical file can still reach the reserved prefix. Its scenario is +# deliberately trivial: the point is the file name, not what it asks. +_RESERVED_FEATURE = """\ +Feature: A feature file in a directory named features + + Scenario: A flag resolves + Given a stable provider + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" +""" + +_RESERVED_SUITE = '''\ +"""An adoption that hands scenarios() a directory of its own named features.""" + +import pathlib + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.provider_tck import ( + Capability, + InProcessControl, + TckConfig, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="reserved", + control=control, + new_provider=control.new_provider, + capabilities={Capability.EVENTS}, + ) + + +scenarios(str(pathlib.Path(__file__).parent / "features")) +''' + + +# -- reading a run back ------------------------------------------------------ + + +@dataclasses.dataclass(frozen=True) +class Case: + """One scenario as the stream reports it.""" + + uri: str + name: str + row: tuple[tuple[str, str], ...] + status: str + + @property + def identity(self) -> tuple[str, str, tuple[tuple[str, str], ...]]: + return (self.uri, self.name, self.row) + + +_SEVERITY = [ + "UNKNOWN", + "PASSED", + "SKIPPED", + "PENDING", + "UNDEFINED", + "AMBIGUOUS", + "FAILED", +] +"""Cucumber's own ordering: a test case is as bad as its worst step.""" + + +@dataclasses.dataclass(frozen=True) +class Report: + """One suite's pair of documents, read the way a consumer reads them.""" + + envelope: dict[str, typing.Any] + sources: dict[str, str] + cases: list[Case] + + @property + def canonical(self) -> list[Case]: + return [case for case in self.cases if is_canonical_uri(case.uri)] + + @property + def extensions(self) -> list[Case]: + return [case for case in self.cases if not is_canonical_uri(case.uri)] + + def named(self, name: str) -> list[Case]: + return [case for case in self.cases if case.name == name] + + @property + def identities(self) -> set[tuple[str, str, tuple[tuple[str, str], ...]]]: + return {case.identity for case in self.cases} + + +@dataclasses.dataclass(frozen=True) +class Run: + """One subprocess run of a generated adoption.""" + + directory: Path + reports: Path + result: subprocess.CompletedProcess[str] + + def report(self, name: str) -> Report: + path = self.reports / f"{name}.json" + assert path.exists(), ( + f"no report at {path}; pytest exited {self.result.returncode}\n" + f"{self.result.stdout}\n{self.result.stderr}" + ) + envelope = json.loads(path.read_text(encoding="utf-8")) + return _read(envelope, self.reports / envelope["results"]["location"]) + + +def _read(envelope: dict[str, typing.Any], stream_path: Path) -> Report: + """Assemble a stream into scenarios by following the protocol's own links.""" + sources: dict[str, str] = {} + names: dict[str, str] = {} + rows: dict[str, tuple[tuple[str, str], ...]] = {} + pickles: dict[str, dict[str, typing.Any]] = {} + test_cases: dict[str, dict[str, typing.Any]] = {} + started: dict[str, str] = {} + results: dict[str, list[str]] = collections.defaultdict(list) + + for line in stream_path.read_text(encoding="utf-8").splitlines(): + message = json.loads(line) + kind = next(iter(message)) + body = message[kind] + if kind == "source": + sources[body["uri"]] = body["data"] + elif kind == "gherkinDocument": + _index_document(body, names, rows) + elif kind == "pickle": + pickles[body["id"]] = body + elif kind == "testCase": + test_cases[body["id"]] = body + elif kind == "testCaseStarted": + started[body["id"]] = body["testCaseId"] + elif kind == "testStepFinished": + results[body["testCaseStartedId"]].append(body["testStepResult"]["status"]) + + cases = [] + for started_id, case_id in started.items(): + pickle = pickles[test_cases[case_id]["pickleId"]] + ast = pickle["astNodeIds"] + cases.append( + Case( + uri=pickle["uri"], + name=names[ast[0]], + row=rows.get(ast[1], ()) if len(ast) > 1 else (), + status=max(results[started_id], key=_SEVERITY.index), + ) + ) + return Report(envelope=envelope, sources=sources, cases=cases) + + +def _index_document( + document: dict[str, typing.Any], + names: dict[str, str], + rows: dict[str, tuple[tuple[str, str], ...]], +) -> None: + def visit(children: typing.Iterable[dict[str, typing.Any]]) -> None: + for child in children: + if "rule" in child: + visit(child["rule"].get("children", ())) + continue + scenario = child.get("scenario") + if scenario is None: + continue + names[scenario["id"]] = scenario["name"] + for examples in scenario.get("examples", ()): + header = examples.get("tableHeader") + if header is None: + continue + headers = [cell["value"] for cell in header["cells"]] + for row in examples.get("tableBody", ()): + cells = [cell["value"] for cell in row["cells"]] + rows[row["id"]] = tuple(zip(headers, cells, strict=True)) + + feature = document.get("feature") + if feature is not None: + visit(feature.get("children", ())) + + +def _pytest(directory: Path, reports: Path) -> subprocess.CompletedProcess[str]: + environment = dict(os.environ) + environment[REPORT_DIR_ENV] = str(reports) + return subprocess.run( # noqa: S603 + [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "no:cacheprovider", + str(directory), + ], + capture_output=True, + text=True, + env=environment, + check=False, + ) + + +def _suite(name: str, call: str = _EXTENSION_CALL) -> str: + return _SUITE_MODULE.format(name=name, call=call) + + +def _run( + tmp_path_factory: pytest.TempPathFactory, + modules: dict[str, str], + features: dict[str, str] | None = None, +) -> Run: + """Write an adoption, run it, and hand back what it wrote. + + Both mappings are keyed by a path relative to the adoption directory, so a + fixture can put a module or a feature file wherever the property under test + needs it -- including inside a directory named ``features``, which is the + case that has to be refused. + """ + directory = tmp_path_factory.mktemp("adoption") + for relative, body in {**modules, **(features or {})}.items(): + path = directory / Path(relative) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(body, encoding="utf-8") + + reports = tmp_path_factory.mktemp("reports") + return Run(directory=directory, reports=reports, result=_pytest(directory, reports)) + + +@pytest.fixture(scope="module") +def adoption(tmp_path_factory: pytest.TempPathFactory) -> Run: + """One session running three adoptions of the same provider. + + ``before`` is the call an adopter writes today, ``scenarios(features_path())``, + which sees no extension however many are lying beside it. ``after`` is + ``scenarios(*feature_paths())`` with an ordinary extension beside it. + ``shadowed`` is the same again, in a directory of its own, with an extension + that is a verbatim copy of a canonical feature file placed under a directory + named ``features`` -- so that both routes to a canonical identity, the file's + own name and its parent's, are taken at once. + + One session rather than three, because a subprocess pytest run is by far the + most expensive thing in this file and the three suites are independent: each + resolves its own ``TckConfig`` and writes its own pair of documents. + """ + canonical = (Path(features_path()) / CANONICAL_FEATURE).read_text(encoding="utf-8") + return _run( + tmp_path_factory, + { + "test_before.py": _suite("before", _CANONICAL_CALL), + "test_after.py": _suite("after", _EXTENSION_CALL), + "conftest.py": _CONFTEST_MODULE, + "shadow/test_shadow.py": _suite("shadowed"), + }, + { + f"{EXTENSIONS_DIRECTORY}/vendor.feature": _VENDOR_FEATURE, + f"shadow/{EXTENSIONS_DIRECTORY}/features/{CANONICAL_FEATURE}": canonical, + }, + ) + + +# -- an extension runs inside the canonical suite ---------------------------- + + +def test_an_extension_scenario_runs_in_the_canonical_suite(adoption: Run) -> None: + """One suite, one report, both sets of scenarios in it. + + The report is written per ``TckConfig``, so an extension scenario appearing + in the same report as the canonical ones is not a presentational detail: it + is the same suite, which is the same provider registration and the same + backend control. + """ + assert adoption.result.returncode == 0, adoption.result.stdout + report = adoption.report("after") + + vendor = report.named(VENDOR_SCENARIO) + assert len(vendor) == 1, report.extensions + assert vendor[0].status == "PASSED" + assert vendor[0].uri == VENDOR_URI + + assert report.canonical, "the canonical scenarios must have run too" + assert {case.status for case in report.canonical} <= {"PASSED", "SKIPPED", "FAILED"} + + +def test_the_extension_step_came_from_the_adopters_conftest(adoption: Run) -> None: + """Nothing was registered, imported or configured to make that step resolve. + + pytest collects ``conftest.py`` on its own and pytest-bdd resolves steps + through the fixture system, so a step defined beside the test module is in + scope for scenarios generated into it. If it were not, the step would be + ``UNDEFINED`` in the stream rather than absent, which is why this asserts on + the payload rather than on the exit status. + """ + conftest = (adoption.directory / "conftest.py").read_text(encoding="utf-8") + assert "the vendor rule ran against the provider the suite registered" in conftest + + vendor = adoption.report("after").named(VENDOR_SCENARIO)[0] + assert vendor.status == "PASSED", ( + f"an unresolved step is reported UNDEFINED, not missing: {vendor}" + ) + + +def test_the_extension_feature_is_reported_under_its_own_prefix( + adoption: Run, +) -> None: + """Which is what keeps an adopter's claim apart from the specification's. + + ``extensions/`` is the prefix the Go and JavaScript suites mount extensions + under too, so a consumer holding reports from several languages applies one + rule. + """ + report = adoption.report("after") + assert [case.uri for case in report.extensions] == [VENDOR_URI] + assert report.sources[VENDOR_URI] == _VENDOR_FEATURE + assert all(case.uri.startswith("features/") for case in report.canonical) + + +def test_the_envelope_is_unaffected_by_an_extension(adoption: Run) -> None: + """The report schema has no slot for extensions and needs none. + + Everything an extension adds is a scenario in the results payload, where a + uri already distinguishes it. An envelope field would be a second place for + the same fact to live. + """ + envelope = adoption.report("after").envelope + assert envelope["provider"]["configuration"] == "after" + assert set(envelope) == { + "schemaVersion", + "provider", + "sdk", + "tck", + "backend", + "declaration", + "results", + } + + +# -- and changes nothing for an adopter who has none ------------------------- + + +def test_the_canonical_scenarios_are_the_ones_that_always_ran( + adoption: Run, +) -> None: + """An extension adds; it does not alter. + + ``before`` is the call an adopter writes today and sees no extension. Every + scenario it ran, ``after`` ran too -- same rows, same outcomes, same sources + -- and the only difference between the two is what the extension added. An + adopter who has no extensions is the same comparison with the right-hand side + empty, which is what ``feature_paths()`` returning the canonical path alone + makes true by construction rather than by luck. + """ + assert adoption.result.returncode == 0, adoption.result.stdout + before = adoption.report("before") + after = adoption.report("after") + + assert not before.extensions, "features_path() must see no extension" + assert {case.identity: case.status for case in after.canonical} == { + case.identity: case.status for case in before.cases + } + assert after.identities - before.identities == {(VENDOR_URI, VENDOR_SCENARIO, ())} + assert { + uri: source for uri, source in after.sources.items() if is_canonical_uri(uri) + } == before.sources + + +def test_an_extension_does_not_change_the_envelope_a_suite_writes( + adoption: Run, +) -> None: + """Everything but the two fields that necessarily differ. + + ``results`` names a file and digests its bytes, and the bytes carry + timestamps; ``configuration`` is the suite name, which is what tells the two + generated suites apart in the first place. + """ + before = dict(adoption.report("before").envelope) + after = dict(adoption.report("after").envelope) + for envelope in (before, after): + del envelope["results"] + envelope["provider"] = { + key: value + for key, value in envelope["provider"].items() + if key != "configuration" + } + assert after == before + + +# -- and cannot stand in for a canonical scenario ---------------------------- + + +def test_an_extension_cannot_replace_a_canonical_feature_file( + adoption: Run, +) -> None: + """The hazard Java measured, checked at the point it would have bitten. + + A verbatim copy of ``errors.feature`` under ``tck-extensions/features/`` + reaches pytest-bdd as ``features/errors.feature`` -- the canonical uri. The + uri the payload reports is derived from where the file is instead, so the + canonical source is still the packaged one, the copy is reported as an + extension, and both ran. + """ + report = adoption.report("shadowed") + canonical_uri = f"features/{CANONICAL_FEATURE}" + extension_uri = f"extensions/features/{CANONICAL_FEATURE}" + + packaged = (Path(features_path()) / CANONICAL_FEATURE).read_text(encoding="utf-8") + assert report.sources[canonical_uri] == packaged + assert report.sources[extension_uri] == packaged + + # Both files ran: the copy neither replaced the canonical one nor was + # silently dropped for colliding with it. + canonical = { + case.identity for case in report.canonical if case.uri == canonical_uri + } + copied = {case.identity for case in report.cases if case.uri == extension_uri} + assert canonical, "the canonical feature file did not run" + assert len(copied) == len(canonical) + assert not any(is_canonical_uri(case.uri) for case in report.extensions) + + +def test_a_shadowing_extension_does_not_disturb_the_canonical_run( + adoption: Run, +) -> None: + """The canonical scenarios are the same ones, with the same outcomes. + + Compared against a run that has no extension at all rather than against a + number written down here, so a change to the canonical assets cannot leave + this passing while the copy quietly displaces something. + """ + shadow = adoption.report("shadowed") + baseline = adoption.report("before") + assert {case.identity: case.status for case in shadow.canonical} == { + case.identity: case.status for case in baseline.cases + } + + +def test_a_directory_named_features_is_refused( + tmp_path_factory: pytest.TempPathFactory, +) -> None: + """The one collision the naming convention cannot rule out on its own. + + An adopter can still hand ``scenarios()`` a directory of their own named + ``features``, and its files are then named exactly as canonical ones would + be. No report is written for that suite: a document presenting an adopter's + feature file as the specification's is worse than no document, because it is + the one thing a consumer cannot check. + """ + run = _run( + tmp_path_factory, + {"test_reserved.py": _RESERVED_SUITE}, + {"features/local.feature": _RESERVED_FEATURE}, + ) + assert run.result.returncode != 0, run.result.stdout + assert "features/local.feature is not a canonical feature file" in run.result.stdout + assert EXTENSIONS_DIRECTORY in run.result.stdout, "the message must say the fix" + assert not list(run.reports.glob("*.json")), "no report may be written" + + +def test_two_extension_files_cannot_share_one_uri( + tmp_path_factory: pytest.TempPathFactory, +) -> None: + """Deriving the uri from the location narrows the collision; it does not end it. + + A ``tck-extensions`` directory nested inside another one reaches the same + uri as its namesake at the root, and so would two test modules sharing one + ``tck_config``. A Messages stream carries one source per uri, so the second + file's scenarios would be reported against the first file's pickles wherever + the names matched -- which here they do, deliberately. Refused rather than + resolved. + """ + nested = f"{EXTENSIONS_DIRECTORY}/nested/{EXTENSIONS_DIRECTORY}/vendor.feature" + run = _run( + tmp_path_factory, + {"test_collide.py": _suite("collide"), "conftest.py": _CONFTEST_MODULE}, + { + f"{EXTENSIONS_DIRECTORY}/vendor.feature": _VENDOR_FEATURE, + nested: _NESTED_FEATURE, + }, + ) + assert run.result.returncode != 0, run.result.stdout + assert f"{VENDOR_URI} is the uri of 2 different feature files" in run.result.stdout + assert not list(run.reports.glob("*.json")), "no report may be written" + + +def test_distinct_extension_paths_do_not_collide(tmp_path: Path) -> None: + """The same check, as a function, on the layouts that are fine.""" + root = tmp_path / EXTENSIONS_DIRECTORY + assert not uri_collisions( + [ + ("extensions/vendor.feature", root / "vendor.feature"), + ("extensions/a/vendor.feature", root / "a" / "vendor.feature"), + # One file reached by two routes is one file, not a collision. + ("extensions/vendor.feature", root / "a" / ".." / "vendor.feature"), + ] + ) + + +# -- deriving the uri -------------------------------------------------------- + + +def test_the_canonical_assets_keep_the_reserved_prefix() -> None: + canonical = Path(features_path()) / CANONICAL_FEATURE + assert uri_for(canonical) == f"features/{CANONICAL_FEATURE}" + assert is_canonical_uri(f"features/{CANONICAL_FEATURE}") + assert reserved_prefix_problem(f"features/{CANONICAL_FEATURE}", canonical) is None + + +def test_an_extension_keeps_its_layout_below_the_extensions_prefix( + tmp_path: Path, +) -> None: + """Whatever the adopter's own directory layout under the root looks like. + + Including one that reproduces the canonical name, which is the collision the + derivation exists for: pytest-bdd would have called the second of these + ``features/errors.feature``. + """ + root = tmp_path / EXTENSIONS_DIRECTORY + assert uri_for(root / "vendor.feature") == "extensions/vendor.feature" + assert ( + uri_for(root / "features" / CANONICAL_FEATURE) + == f"extensions/features/{CANONICAL_FEATURE}" + ) + assert ( + uri_for(root / "a" / "b" / "vendor.feature") == "extensions/a/b/vendor.feature" + ) + assert not is_canonical_uri("extensions/features/errors.feature") + + +def test_a_file_that_is_neither_is_left_to_pytest_bdd(tmp_path: Path) -> None: + """``None`` rather than a guess: the caller falls back to what the runner said.""" + assert uri_for(tmp_path / "loose.feature") is None + + +def test_a_local_file_under_the_reserved_prefix_is_a_problem(tmp_path: Path) -> None: + local = tmp_path / "features" / "local.feature" + problem = reserved_prefix_problem("features/local.feature", local) + assert problem is not None + assert EXTENSIONS_DIRECTORY in problem + assert str(local) in problem + + +def test_feature_paths_is_the_canonical_set_when_there_is_no_extension_directory() -> ( + None +): + """This test module has no ``tck-extensions`` beside it, and gets one path.""" + assert not (Path(__file__).parent / EXTENSIONS_DIRECTORY).exists() + assert feature_paths() == (features_path(),) diff --git a/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py b/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py index 9b9e4a19..5d6868c2 100644 --- a/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py +++ b/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py @@ -22,6 +22,7 @@ from openfeature.contrib.tools.provider_tck import ( Capability, + KnownDeviation, TckConfig, canonical_flag_set, features_path, @@ -47,6 +48,15 @@ class PlainMemoryControl: this error would mean the capability had been declared anyway. """ + @property + def control_api(self) -> str: + """Report that this control manipulates a provider in this process. + + There is no backend to drive: the in-memory provider is rebuilt in + process for every scenario, which is exactly what "in-process" names. + """ + return "in-process" + @property def description(self) -> str: return "the Python SDK's InMemoryProvider, rebuilt per scenario" @@ -70,7 +80,7 @@ def _new_provider() -> FeatureProvider: @pytest.fixture(scope="session") -def tck_config() -> TckConfig: +def tck_config(tck_known_deviations: tuple[KnownDeviation, ...]) -> TckConfig: """Declare the provider under test and what it can do. Each omission is a fact about the provider rather than a convenience: @@ -93,6 +103,11 @@ def tck_config() -> TckConfig: vacuously while the feature was gated on ``EVENTS``, which is precisely the failure mode the split of ``@lifecycle`` from ``@events`` exists to end. A skip with a reason is the honest outcome. + + ``known_deviations`` is the one thing here that is not a claim about what + this provider supports: it is the acknowledgement of a scenario the SDK + fails, which the results payload still reports as a failure. See + ``conftest.py``. """ return TckConfig( name="in-memory", @@ -103,6 +118,7 @@ def tck_config() -> TckConfig: Capability.OBJECT, Capability.NUMERIC_COERCION, }, + known_deviations=tck_known_deviations, ) diff --git a/tools/openfeature-provider-tck/tests/test_report.py b/tools/openfeature-provider-tck/tests/test_report.py new file mode 100644 index 00000000..df3361f0 --- /dev/null +++ b/tools/openfeature-provider-tck/tests/test_report.py @@ -0,0 +1,1191 @@ +"""What the conformance report must never do. + +The report exists because a runner's summary cannot be checked by anything +downstream. So the tests that matter here are not about JSON shape; they are +about the two properties a consumer is entitled to assume, neither of which is +guaranteed by the code that happens to assemble the documents: + +* a scenario skipped for an undeclared capability is never reported as passed, + and the reason it was skipped is recoverable; +* every scenario the run collected appears exactly once, which is what makes the + first property checkable rather than merely asserted -- a document that quietly + dropped what it skipped would satisfy the letter of it and still mislead. + +Both are now checked against the *results payload* rather than against the +envelope, because that is where the results moved: a run writes an envelope and a +`Cucumber Messages`_ stream beside it, and the envelope says only what was tested, +what the provider claims and where the results are. The assertions are written +the way a consumer reads the stream -- a test case is as bad as its worst step -- +so that what these tests check is what a consumer would see rather than an +internal representation. + +Both are checked against a real pytest session in a subprocess, because both are +properties of how the suite runs rather than of how the documents are assembled. +That session is also the only place a skip, a pass and a failure occur together, +and the only place the payload can be seen to disagree with the runner's summary +-- which it does, deliberately, for a known deviation. + +.. _Cucumber Messages: https://github.com/cucumber/messages +""" + +from __future__ import annotations + +import collections +import dataclasses +import hashlib +import json +import os +import subprocess +import sys +import typing +from pathlib import Path + +import pytest + +from openfeature.contrib.tools.provider_tck import ( + DECLARABLE_CAPABILITIES, + EXTENSIONS_DIRECTORY, + RESERVED_CAPABILITIES, + Capability, + KnownDeviation, + TckConfig, + features_path, +) +from openfeature.contrib.tools.provider_tck.emitter import ( + classify_phase, + scenario_run, +) +from openfeature.contrib.tools.provider_tck.messages import ( + MESSAGES_FORMAT, + ScenarioIdentity, + Status, + StepRun, + _Pickle, + _step_runs, + feature_uri, +) +from openfeature.contrib.tools.provider_tck.report import ( + REPORT_DIR_ENV, + PhaseOutcome, + Results, + SuiteReport, + control_api_of, + envelope_file_name, + normalise_tags, + spec_revision, + stream_file_name, +) + +# The generated suite's name is deliberately not path-safe. +SUITE_NAME = "report/fixture" +SUITE_FILE = "report-fixture.json" + +UNKNOWN_KEY_SCENARIO = "An unknown flag key returns the code default" + +# The type-mismatch matrix: eleven Examples rows under one scenario name, one of +# which the Python SDK fails. It is the case row identity exists for. +MISMATCH_SCENARIO = "Requesting the wrong type returns the code default" + +# The row that fails, spelled as the feature file spells it -- strings, because +# Gherkin has no types and "1" is not 1. +DEVIATING_ROW = {"key": "boolean-flag", "requested": "Integer", "default": "1"} + +DEVIATION_ISSUE = "https://github.com/open-feature/python-sdk/issues/619" + +# How Cucumber orders its statuses. A test case is as bad as its worst step, and +# this is the rule a consumer applies to derive a scenario's outcome from the +# stream -- so it is the rule these tests apply too. +SEVERITY = [ + "UNKNOWN", + "PASSED", + "SKIPPED", + "PENDING", + "UNDEFINED", + "AMBIGUOUS", + "FAILED", +] + +_SUITE_MODULE = '''\ +"""A one-fixture adoption, generated so the report can be checked end to end.""" + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.provider_tck import ( + Capability, + InProcessControl, + KnownDeviation, + TckConfig, + features_path, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="{name}", + control=control, + new_provider=control.new_provider, + capabilities={capabilities}, + not_applicable={not_applicable}, + known_deviations={deviations}, + ) + + +scenarios(features_path()) +''' + +CAPABILITIES = "{Capability.EVENTS, Capability.OBJECT, Capability.NUMERIC_COERCION}" +"""What the main generated suite declares: enough to produce a skip and a pass.""" + +NOT_APPLICABLE = '{Capability.STALE: "this provider has no connection to lose"}' +"""One capability the provider cannot have rather than merely does not declare.""" + +DEVIATIONS = ( + "(KnownDeviation(" + f'issue="{DEVIATION_ISSUE}", ' + 'summary="a boolean satisfies an Integer request"),)' +) + +# One scenario skipped outright and one known deviation marked xfail, so the run +# produces a skip, a pass and a failure and finishes green while the payload +# does not. +_CONFTEST_MODULE = """\ +import pytest + +SKIPPED = "test_an_unknown_flag_key_returns_the_code_default" +DEVIATION = "test_requesting_the_wrong_type_returns_the_code_default[boolean-flag-Integer-1]" + + +def pytest_collection_modifyitems(items): + for item in items: + if item.name == SKIPPED: + item.add_marker(pytest.mark.skip(reason="deliberately not run here")) + elif item.name == DEVIATION: + item.add_marker(pytest.mark.xfail(reason="python-sdk#619", strict=True)) +""" + + +# A Scenario Outline whose second Examples block carries a tag of its own, which +# no canonical feature file does yet. Written here so that the one case where two +# rows of an outline are gated differently is covered. +_TAGGED_FEATURE = """\ +Feature: Per-Examples tags + + Background: + Given a stable provider + + Scenario Outline: Requesting the wrong type returns the code default + Given a -flag with key "" and a default value "" + When the flag was evaluated with details + Then the resolved details value should be "" + And the reason should be "ERROR" + And the error-code should be "TYPE_MISMATCH" + And no exception should have been thrown + + Examples: ungated + | key | requested | default | + | string-flag | Boolean | false | + | string-flag | Integer | 1 | + + @object + Examples: gated behind a capability this suite does not declare + | key | requested | default | + | string-flag | Float | 0.1 | +""" + +_TAGGED_SUITE = '''\ +"""A suite whose extension feature file tags one Examples block of an outline. + +The canonical set runs alongside it, because a suite that leaves the canonical +set out writes no report at all -- see ``_canonical_set_ran``. +""" + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.provider_tck import ( + Capability, + InProcessControl, + TckConfig, + feature_paths, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="per-examples", + control=control, + new_provider=control.new_provider, + capabilities={Capability.EVENTS}, + ) + + +scenarios(*feature_paths()) +''' + + +# -- reading the payload back the way a consumer would ----------------------- + + +@dataclasses.dataclass(frozen=True) +class Case: + """One scenario as the stream reports it, assembled from its messages.""" + + uri: str + name: str + """The scenario name from the *AST*, so an outline's rows share it.""" + + row: tuple[tuple[str, str], ...] + """The Examples row, resolved from the pickle's AST node ids.""" + + tags: frozenset[str] + status: str + """The worst of the test case's steps, which is Cucumber's rule.""" + + setup_message: str + """What the before-hook said, which is where a skip's reason lands.""" + + step_statuses: tuple[str, ...] + + @property + def identity(self) -> tuple[str, str, tuple[tuple[str, str], ...]]: + return (self.uri, self.name, self.row) + + +@dataclasses.dataclass(frozen=True) +class Stream: + """A parsed Cucumber Messages stream.""" + + kinds: collections.Counter[str] + sources: dict[str, str] + cases: list[Case] + + def named(self, name: str) -> list[Case]: + return [case for case in self.cases if case.name == name] + + @property + def statuses(self) -> collections.Counter[str]: + return collections.Counter(case.status for case in self.cases) + + +@dataclasses.dataclass +class _Index: + """The stream's messages, keyed the way the protocol says they relate.""" + + kinds: collections.Counter[str] = dataclasses.field( + default_factory=collections.Counter + ) + sources: dict[str, str] = dataclasses.field(default_factory=dict) + names: dict[str, str] = dataclasses.field(default_factory=dict) + rows: dict[str, tuple[tuple[str, str], ...]] = dataclasses.field( + default_factory=dict + ) + pickles: dict[str, typing.Any] = dataclasses.field(default_factory=dict) + test_cases: dict[str, typing.Any] = dataclasses.field(default_factory=dict) + hooks: dict[str, str] = dataclasses.field(default_factory=dict) + started: dict[str, str] = dataclasses.field(default_factory=dict) + results: dict[str, list[tuple[str, typing.Any]]] = dataclasses.field( + default_factory=dict + ) + + def add(self, kind: str, body: typing.Any) -> None: + self.kinds[kind] += 1 + if kind == "source": + self.sources[body["uri"]] = body["data"] + elif kind == "gherkinDocument": + _index_document(body, self.names, self.rows) + elif kind == "pickle": + self.pickles[body["id"]] = body + elif kind == "testCase": + self.test_cases[body["id"]] = body + for step in body["testSteps"]: + if "hookId" in step: + self.hooks[step["id"]] = step["hookId"] + elif kind == "testCaseStarted": + self.started[body["id"]] = body["testCaseId"] + elif kind == "testStepFinished": + self.results.setdefault(body["testCaseStartedId"], []).append( + (body["testStepId"], body["testStepResult"]) + ) + + +def _read_stream(path: Path) -> Stream: + """Assemble the stream into test cases the way a consumer has to. + + Deliberately written against the protocol rather than against this package: + a pickle's ``astNodeIds`` are followed back into the ``GherkinDocument`` to + recover the scenario name and the Examples row, and a test case's status is + computed as the worst of its steps. If the stream does not actually support + those two operations, these tests fail -- which is the point. + """ + index = _Index() + for line in path.read_text(encoding="utf-8").splitlines(): + message = json.loads(line) + kind = next(iter(message)) + index.add(kind, message[kind]) + + cases = [ + _case(index, started_id, case_id) + for started_id, case_id in index.started.items() + ] + return Stream(kinds=index.kinds, sources=index.sources, cases=cases) + + +def _case(index: _Index, started_id: str, case_id: str) -> Case: + pickle = index.pickles[index.test_cases[case_id]["pickleId"]] + ast = pickle["astNodeIds"] + steps = index.results[started_id] + setup: dict[str, typing.Any] = next( + ( + result + for step_id, result in steps + if index.hooks.get(step_id, "").endswith("setup") + ), + {}, + ) + return Case( + uri=pickle["uri"], + name=index.names[ast[0]], + row=index.rows.get(ast[1], ()) if len(ast) > 1 else (), + tags=frozenset(tag["name"] for tag in pickle.get("tags", ())), + status=max((result["status"] for _, result in steps), key=SEVERITY.index), + setup_message=setup.get("message", ""), + step_statuses=tuple( + result["status"] for step_id, result in steps if step_id not in index.hooks + ), + ) + + +def _index_document( + document: dict[str, typing.Any], + names: dict[str, str], + rows: dict[str, tuple[tuple[str, str], ...]], +) -> None: + def visit(children: typing.Iterable[dict[str, typing.Any]]) -> None: + for child in children: + if "rule" in child: + visit(child["rule"].get("children", ())) + continue + scenario = child.get("scenario") + if scenario is None: + continue + names[scenario["id"]] = scenario["name"] + for examples in scenario.get("examples", ()): + header = examples.get("tableHeader") + if header is None: + continue + headers = [cell["value"] for cell in header["cells"]] + for row in examples.get("tableBody", ()): + cells = [cell["value"] for cell in row["cells"]] + rows[row["id"]] = tuple(zip(headers, cells, strict=True)) + + feature = document.get("feature") + if feature is not None: + visit(feature.get("children", ())) + + +@dataclasses.dataclass(frozen=True) +class Run: + """One subprocess run of the generated suite: both documents it wrote.""" + + directory: Path + result: subprocess.CompletedProcess[str] + envelope: dict[str, typing.Any] + stream: Stream + stream_path: Path + + @property + def declared(self) -> set[str]: + return set(self.envelope["declaration"]["declared"]) + + +# -- helpers ----------------------------------------------------------------- + + +class _StubControl: + """A control that says nothing about how it drove the backend.""" + + @property + def description(self) -> str: + return "a stub" + + def prepare_scenario(self) -> None: + return None + + def change_flag(self) -> None: + return None + + +class _HttpControl(_StubControl): + @property + def control_api(self) -> str: + return "http" + + +def _config_leaving_capabilities_to_their_default() -> TckConfig: + """A config that does not narrow ``capabilities``, so the field default runs. + + Written out rather than routed through ``_config``, which supplies a narrow + set of its own: the default is the whole point of this one. It needs an + unavailable-provider factory because ``@unavailable`` is declarable and the + default therefore declares it. + """ + settings: dict[str, typing.Any] = { + "name": "stub", + "control": _StubControl(), + "new_provider": lambda: None, + "new_unavailable_provider": lambda: None, + } + return TckConfig(**settings) + + +def _config(**overrides: typing.Any) -> TckConfig: + settings: dict[str, typing.Any] = { + "name": "stub", + "control": _StubControl(), + "new_provider": lambda: None, + "capabilities": {Capability.EVENTS}, + } + settings.update(overrides) + return TckConfig(**settings) + + +def _identity(*tags: str, name: str = "a scenario") -> ScenarioIdentity: + return ScenarioIdentity( + uri="features/events.feature", + path=Path(features_path()) / "events.feature", + name=name, + tags=tags, + ) + + +def _results() -> Results: + return Results(location="stub.ndjson", digest="sha256:" + "0" * 64) + + +def _examples_from_the_feature_file(feature: str, outline: str) -> list[dict[str, str]]: + """Read an outline's Examples tables straight out of the Gherkin. + + Hand-read rather than taken from a parser, because a parser is what produced + the values under test: asking it what it should have said would check + nothing. It is a small reader for a small shape -- the tables in these files + are plain pipe-delimited rows -- and it exists so that "the stream says what + the table said" is checked against the table. + """ + source = Path(features_path()) / f"{feature}.feature" + lines = source.read_text(encoding="utf-8").splitlines() + rows: list[dict[str, str]] = [] + headers: list[str] = [] + inside = False + + for line in lines: + stripped = line.strip() + if stripped.startswith(("Scenario:", "Scenario Outline:")): + inside = stripped.split(":", 1)[1].strip() == outline + headers = [] + elif not inside: + continue + elif stripped.startswith("Examples"): + headers = [] + elif stripped.startswith("|"): + cells = [cell.strip() for cell in stripped.strip("|").split("|")] + if headers: + rows.append(dict(zip(headers, cells, strict=True))) + else: + headers = cells + + assert rows, f"no Examples rows found for {outline!r} in {feature}.feature" + return rows + + +def _phase(outcome: str, when: str = "call", **extra: typing.Any) -> PhaseOutcome: + return PhaseOutcome(when=when, outcome=outcome, **extra) + + +def _pytest( + *arguments: str, report_dir: Path | None = None +) -> subprocess.CompletedProcess[str]: + environment = dict(os.environ) + environment.pop(REPORT_DIR_ENV, None) + if report_dir is not None: + environment[REPORT_DIR_ENV] = str(report_dir) + return subprocess.run( # noqa: S603 + [sys.executable, "-m", "pytest", "-q", "-p", "no:cacheprovider", *arguments], + capture_output=True, + text=True, + env=environment, + check=False, + ) + + +def _write_suite( + directory: Path, + name: str = SUITE_NAME, + capabilities: str = CAPABILITIES, + not_applicable: str = NOT_APPLICABLE, + deviations: bool = True, +) -> Path: + directory.mkdir(parents=True, exist_ok=True) + (directory / "test_suite.py").write_text( + _SUITE_MODULE.format( + name=name, + capabilities=capabilities, + not_applicable=not_applicable, + deviations=DEVIATIONS if deviations else "()", + ), + encoding="utf-8", + ) + if deviations: + (directory / "conftest.py").write_text(_CONFTEST_MODULE, encoding="utf-8") + return directory + + +def _run_suite( + tmp_path_factory: pytest.TempPathFactory, + file_name: str = SUITE_FILE, + **suite: typing.Any, +) -> Run: + """Run one generated suite in a subprocess and read what it wrote.""" + directory = _write_suite(tmp_path_factory.mktemp("suite"), **suite) + reports = tmp_path_factory.mktemp("reports") + result = _pytest(str(directory), report_dir=reports) + + path = reports / file_name + assert path.exists(), ( + f"no report at {path}; pytest exited {result.returncode}\n" + f"{result.stdout}\n{result.stderr}" + ) + envelope = json.loads(path.read_text(encoding="utf-8")) + stream_path = path.parent / envelope["results"]["location"] + return Run( + directory=directory, + result=result, + envelope=envelope, + stream=_read_stream(stream_path), + stream_path=stream_path, + ) + + +@pytest.fixture(scope="module") +def run(tmp_path_factory: pytest.TempPathFactory) -> Run: + """One real run of the generated suite, with a report asked for.""" + return _run_suite(tmp_path_factory) + + +@pytest.fixture(scope="module") +def narrow_run(tmp_path_factory: pytest.TempPathFactory) -> Run: + """A run of a suite that leaves a whole Scenario Outline gated. + + ``@object`` is undeclared so that every row of one outline is skipped by the + capability gate, which is the case that has to keep saying which row it + skipped. + """ + return _run_suite( + tmp_path_factory, + file_name="narrow.json", + name="narrow", + capabilities="{Capability.NUMERIC_COERCION}", + not_applicable="{}", + deviations=False, + ) + + +# -- the two properties that matter ------------------------------------------ + + +def test_a_capability_skip_is_never_reported_as_passed(run: Run) -> None: + """The rule Appendix F states, checked against the payload, not the runner.""" + gated = [case for case in run.stream.cases if case.tags - run.declared] + assert gated, "the generated suite is meant to have scenarios behind those" + + for case in gated: + assert case.status == "SKIPPED", case + # Cucumber's SKIPPED is per step, so "never as passed" has to hold of + # every step and not merely of the rolled-up verdict. + assert set(case.step_statuses) == {"SKIPPED"}, case + assert case.setup_message, f"a skip must say why: {case}" + + +def test_the_reason_for_a_gated_skip_follows_from_the_two_documents(run: Run) -> None: + """Which is why the per-scenario reason no longer has to be transported. + + The envelope says what the provider declares; the payload says which tags + each scenario carries and that it was skipped. A consumer with both can name + the capability responsible without the emitter having written it down once + per scenario, and that derivation is what the declaration exists for. + """ + skipped = [case for case in run.stream.cases if case.status == "SKIPPED"] + gated = [case for case in skipped if case.tags - run.declared] + assert gated, "no skip was attributable to an undeclared capability" + + # The derivation is checked against the reason the runner actually gave: for + # every skip the two documents attribute to a capability, that capability is + # the one the gate named. If they disagreed, the declaration would be the + # wrong thing to read a skip against. + for case in gated: + responsible = case.tags - run.declared + assert any(tag in case.setup_message for tag in responsible), case + + # And it distinguishes: the one scenario skipped for another reason carries + # no undeclared tag, so it is not attributed to a capability at all. + others = [case for case in skipped if case not in gated] + assert others, "the generated suite is meant to skip one scenario outright" + for case in others: + assert not case.tags - run.declared, case + assert "capability" not in case.setup_message, case + + +def test_every_collected_scenario_appears_exactly_once(run: Run) -> None: + """The property that makes the rule above checkable rather than promised. + + A test case is identified by its uri, its scenario name **and its Examples + row**, all three recovered from the stream by following a pickle's AST node + ids. Name alone is shared by every row of a Scenario Outline, so keying on it + would let eleven rows of the type-mismatch matrix collapse into one and this + test would not notice. + + Counted against pytest's own collection rather than against a number written + down here, so that adding a scenario to the specification cannot leave this + passing while the payload loses one. + """ + identities = [case.identity for case in run.stream.cases] + assert len(identities) == len(set(identities)), "a scenario is reported twice" + + collected = _pytest("--collect-only", str(run.directory)) + assert len(identities) == sum( + 1 for line in collected.stdout.splitlines() if "::test_" in line + ) + + +def test_the_statuses_account_for_every_scenario(run: Run) -> None: + counts = run.stream.statuses + assert set(counts) <= set(SEVERITY), "a status outside the protocol's own" + assert sum(counts.values()) == len(run.stream.cases) + # A skip, a pass and a failure all occur, which is what makes the run worth + # asserting against at all. + assert set(counts) == {"PASSED", "SKIPPED", "FAILED"}, counts + + +def test_the_payload_does_not_repeat_the_runner_summary(run: Run) -> None: + """A known deviation is a failure in the payload even when pytest is green. + + The suite marks the one scenario the Python SDK cannot satisfy as an expected + failure, so pytest exits zero. The provider still did not satisfy it, and a + payload that agreed with the summary would hide exactly what the marker was + added to keep visible. The acknowledgement goes in the envelope instead. + """ + assert run.result.returncode == 0, run.result.stdout + failed = [case for case in run.stream.cases if case.status == "FAILED"] + assert len(failed) == 1 + assert failed[0].row == tuple(DEVIATING_ROW.items()) + + deviations = run.envelope["knownDeviations"] + assert [deviation["issue"] for deviation in deviations] == [DEVIATION_ISSUE] + + +def test_a_scenario_skipped_for_another_reason_is_still_reported(run: Run) -> None: + """A run that chose not to execute a scenario still accounts for it. + + Skipped, and present, even though a marker skip never runs a fixture -- and + with its own reason rather than the capability gate's, which is what tells + the two apart now that the payload has one status for both. + """ + matching = run.stream.named(UNKNOWN_KEY_SCENARIO) + assert len(matching) == 1 + assert matching[0].status == "SKIPPED" + assert "deliberately not run here" in matching[0].setup_message + # Nothing about the provider's declaration explains this one, which is how a + # consumer tells it from a capability skip. + assert not matching[0].tags - run.declared + + +# -- which row of an outline ------------------------------------------------- + + +def test_an_outline_row_is_identified_by_its_ast_node_id(run: Run) -> None: + """The eleven rows of the type-mismatch matrix are told apart, and only here. + + All eleven share one scenario name, which is the feature file's name and must + stay that way: it is what a report from Go or JavaScript carries for the same + row. What tells them apart is the pickle's second ``astNodeIds`` entry, the + id of the table row it was compiled from, which resolves in the + ``GherkinDocument`` to exactly the cells the feature file wrote. That is the + identity four implementations were each reinventing as a bespoke ``example`` + field before this format carried it. + """ + rows = run.stream.named(MISMATCH_SCENARIO) + expected = _examples_from_the_feature_file("errors", MISMATCH_SCENARIO) + assert len(rows) == len(expected) == 11 + + observed = [dict(case.row) for case in rows] + assert len(observed) == len({case.row for case in rows}), "two rows collapsed" + assert sorted(map(sorted, (row.items() for row in observed))) == sorted( + map(sorted, (row.items() for row in expected)) + ) + + # Verbatim strings, because Gherkin has no types: a "1" in a table is the + # one-character cell the feature file contains. + for row in observed: + assert all(isinstance(value, str) for value in row.values()), row + + +def test_a_scenario_that_is_not_an_outline_has_no_row(run: Run) -> None: + """One AST node id, so there is no row to resolve and nothing to say.""" + plain = run.stream.named(UNKNOWN_KEY_SCENARIO) + assert len(plain) == 1 + assert plain[0].row == () + + +def test_a_capability_skipped_outline_row_is_still_identified( + narrow_run: Run, +) -> None: + """A skipped row is exactly as ambiguous as a failed one. + + Row identity comes from the pickle rather than from the run, so it does not + depend on the scenario having executed -- which is what lets a row the + capability gate stopped before its first step be told apart from its siblings + just as well as one that failed. + """ + outline = "Requesting a structured flag as a scalar returns the code default" + expected = _examples_from_the_feature_file("errors", outline) + rows = narrow_run.stream.named(outline) + assert len(rows) == len(expected) + + for case in rows: + assert case.status == "SKIPPED", case + assert case.row, f"a skipped outline row must say which row: {case}" + assert sorted(map(sorted, (dict(case.row).items() for case in rows))) == sorted( + map(sorted, (row.items() for row in expected)) + ) + + +def test_a_row_gated_by_its_examples_block_is_the_only_one_skipped( + tmp_path: Path, +) -> None: + """Gherkin lets one Examples block of an outline carry its own tags. + + Two rows of one Scenario Outline can therefore differ in which capability + gates them. Those tags are on neither the scenario, the feature nor the rule, + and a payload built from those three would show the skipped row as carrying + no capability -- leaving the envelope's declaration unable to explain the + skip, which is the one derivation this format asks a consumer to make. + + No canonical feature file does this yet, so the feature file is written here + -- as an extension, because a suite that leaves the canonical set out writes + no report to read back. + """ + directory = tmp_path / "suite" + extensions = directory / EXTENSIONS_DIRECTORY + extensions.mkdir(parents=True) + (extensions / "tagged.feature").write_text(_TAGGED_FEATURE, encoding="utf-8") + (directory / "test_tagged.py").write_text(_TAGGED_SUITE, encoding="utf-8") + + reports = tmp_path / "reports" + result = _pytest(str(directory), report_dir=reports) + path = reports / "per-examples.json" + assert path.exists(), f"pytest exited {result.returncode}\n{result.stdout}" + + envelope = json.loads(path.read_text(encoding="utf-8")) + stream = _read_stream(path.parent / envelope["results"]["location"]) + by_row = { + dict(case.row)["requested"]: case + for case in stream.cases + if case.uri == "extensions/tagged.feature" + } + + # Every row is still reported: nothing about gating one row of an outline may + # drop its siblings from the payload. + assert set(by_row) == {"Boolean", "Integer", "Float"}, by_row + assert by_row["Boolean"].status == "PASSED" + assert by_row["Integer"].status == "PASSED" + + gated = by_row["Float"] + assert gated.status == "SKIPPED", gated + assert Capability.OBJECT.tag in gated.tags, gated + assert Capability.OBJECT.tag not in envelope["declaration"]["declared"] + + +# -- the payload, and the envelope that points at it ------------------------- + + +def test_the_envelope_points_at_the_payload_it_describes(run: Run) -> None: + """Referenced rather than inlined, and covered by a digest. + + A stream carries the feature sources and is far larger than the envelope, so + a consumer deciding whether it cares about a report should not have to fetch + a whole run to find out -- and needs to be able to tell that what it did + fetch is what the envelope described. + """ + results = run.envelope["results"] + assert results["format"] == MESSAGES_FORMAT + # A bare file name, so the reference survives the pair being moved together. + assert results["location"] == run.stream_path.name + assert Path(results["location"]).parent == Path() + + digest = hashlib.sha256(run.stream_path.read_bytes()).hexdigest() + assert results["digest"] == f"sha256:{digest}" + + +def test_the_payload_carries_the_source_of_every_feature_it_ran(run: Run) -> None: + """Which is what replaced recording an asset tree hash. + + A hash said only whether two runs executed the same assets. The source says + what the assets were, so a consumer can read the questions that were actually + asked rather than trusting a recorded revision to stand for them. + """ + uris = {case.uri for case in run.stream.cases} + assert uris, "no test case named a feature file" + assert set(run.stream.sources) == uris + + for uri, data in run.stream.sources.items(): + on_disk = Path(features_path()) / Path(uri).name + assert data == on_disk.read_text(encoding="utf-8"), uri + + assert "assetsTree" not in run.envelope["tck"] + + +def test_the_payload_is_a_well_formed_messages_stream(run: Run) -> None: + """The message types a consumer needs are all present, once each per scenario.""" + kinds = run.stream.kinds + cases = len(run.stream.cases) + assert kinds["meta"] == 1 + assert kinds["testRunStarted"] == 1 + assert kinds["testRunFinished"] == 1 + assert kinds["source"] == kinds["gherkinDocument"] == len(run.stream.sources) + assert kinds["pickle"] == cases + assert kinds["testCase"] == kinds["testCaseStarted"] == cases + assert kinds["testCaseFinished"] == cases + assert kinds["testStepStarted"] == kinds["testStepFinished"] + + +def test_the_declaration_is_an_input_not_a_summary(run: Run) -> None: + """Which is why it cannot be derived from the payload and is stated here. + + Declared and not-applicable are disjoint and mean different things -- a + choice against a capability, and an impossibility -- and the payload can + express neither, because a skip in it says only that the question was not + put to this provider. + """ + declaration = run.envelope["declaration"] + assert declaration["declared"] == [ + Capability.EVENTS.tag, + Capability.NUMERIC_COERCION.tag, + Capability.OBJECT.tag, + ] + assert declaration["notApplicable"] == { + Capability.STALE.tag: "this provider has no connection to lose" + } + assert Capability.STALE.tag not in declaration["declared"] + + +def test_the_provider_and_its_configuration_are_reported_separately(run: Run) -> None: + assert run.envelope["provider"]["name"] == "In-Memory Provider" + assert run.envelope["provider"]["configuration"] == SUITE_NAME + assert run.envelope["provider"]["language"] == "python" + + +def test_the_envelope_names_what_ran_it(run: Run) -> None: + assert run.envelope["schemaVersion"] == "1" + assert ( + run.envelope["tck"]["implementation"] + == "python-sdk-contrib/tools/openfeature-provider-tck" + ) + assert run.envelope["sdk"]["name"] == "openfeature-sdk" + assert run.envelope["sdk"]["version"] + assert len(run.envelope["tck"]["specRevision"]) >= 7 + assert run.envelope["backend"]["controlApi"] == "in-process" + + +def test_the_envelope_carries_no_results_of_its_own(run: Run) -> None: + """The fields Cucumber Messages made redundant, checked to be gone. + + Not a shape test for its own sake: while both existed there were two places + for the same fact to disagree, which is the whole reason the per-scenario + list was deleted rather than kept alongside the payload. + """ + assert "scenarios" not in run.envelope + assert "capabilities" not in run.envelope + + +def test_the_spec_revision_comes_from_the_build() -> None: + """Generated beside the assets, because the submodule is not in the wheel.""" + assert len(spec_revision()) >= 7 + + +# -- opting in --------------------------------------------------------------- + + +def test_no_report_is_written_without_the_environment_variable( + tmp_path: Path, +) -> None: + """The default, and not an error: emitting is a property of the run.""" + directory = _write_suite(tmp_path / "suite") + result = _pytest(str(directory), report_dir=None) + assert result.returncode == 0, result.stdout + assert "report written" not in result.stdout + assert not list(tmp_path.rglob("*.json")) + assert not list(tmp_path.rglob("*.ndjson")) + + +def test_a_report_that_cannot_be_written_fails_the_run(tmp_path: Path) -> None: + """Loudly, because a pipeline that silently got no report serves a stale one. + + The destination is placed under a regular file, which no platform will let + ``mkdir`` turn into a directory. The run itself passes, so a non-zero exit + can only have come from the failure to write. + """ + blocker = tmp_path / "not-a-directory" + blocker.write_text("", encoding="utf-8") + directory = _write_suite(tmp_path / "suite") + result = _pytest(str(directory), report_dir=blocker / "reports") + assert "could not write the conformance report" in result.stdout + assert result.returncode != 0 + + +# -- assembling the documents ------------------------------------------------ + + +def test_a_failure_is_not_revised_away_by_a_later_phase() -> None: + """A scenario whose steps passed and whose teardown blew up is a failure.""" + run = scenario_run( + _identity(), + [ + _phase("passed", when="setup"), + _phase("passed", when="call"), + _phase("failed", when="teardown", message="teardown exploded"), + ], + [StepRun(status=Status.passed)], + ) + assert run.status is Status.failed + assert run.message == "teardown exploded" + + +def test_a_verdict_no_step_accounts_for_reaches_the_stream_anyway() -> None: + """A strict xfail that passes fails a scenario every step of which passed. + + A consumer reads a test case's outcome as the worst of its steps, so a + verdict left only in this package's own head would be lost on the way out. + It is attached to the after-hook, which is where a test case failing outside + its own steps belongs. + """ + run = scenario_run( + _identity(), + [ + _phase("passed", when="setup"), + _phase("failed", when="call", message="[XPASS(strict)] python-sdk#619"), + _phase("passed", when="teardown"), + ], + [StepRun(status=Status.passed)], + ) + assert run.status is Status.failed + assert run.teardown is not None + assert run.teardown.status is Status.passed, "the phase itself did pass" + + # The stream is where the discrepancy has to be resolved, and it is: + # `_step_runs` upgrades the after-hook when nothing else carries the verdict. + steps = _step_runs(_Pickle(id="p", step_ids=("s",), payload={}), run) + worst = max((step.status.value for step in steps), key=SEVERITY.index) + assert worst == Status.failed.value + assert "XPASS(strict)" in steps[-1].message + + +def test_a_gated_skip_is_skipped_for_every_step() -> None: + """The gate stops a scenario in setup, so no step of it ran.""" + run = scenario_run( + _identity("@stale"), + [_phase("skipped", when="setup", message="provider does not declare @stale")], + [], + ) + assert run.status is Status.skipped + assert run.setup is not None + assert run.setup.message == "provider does not declare @stale" + + +def test_the_declaration_reports_what_the_configuration_declares() -> None: + suite = SuiteReport(config=_config(capabilities={Capability.EVENTS})) + declaration = suite.build(_results())["declaration"] + assert declaration["declared"] == [Capability.EVENTS.tag] + assert "notApplicable" not in declaration + + +def test_a_not_applicable_capability_is_reported_with_its_reason() -> None: + """Impossible is not the same claim as undeclared, and the report keeps both.""" + suite = SuiteReport( + config=_config( + capabilities={Capability.EVENTS}, + not_applicable={Capability.NUMERIC_COERCION: "no integer type"}, + ) + ) + declaration = suite.build(_results())["declaration"] + assert declaration["notApplicable"] == { + Capability.NUMERIC_COERCION.tag: "no integer type" + } + + +def test_a_capability_cannot_be_both_declared_and_impossible() -> None: + with pytest.raises(ValueError, match="both claim @events"): + _config( + capabilities={Capability.EVENTS}, + not_applicable={Capability.EVENTS: "a reason"}, + ) + + +def test_a_not_applicable_capability_must_say_why() -> None: + with pytest.raises(ValueError, match="no reason for @stale"): + _config( + capabilities={Capability.EVENTS}, not_applicable={Capability.STALE: " "} + ) + + +def test_a_reserved_capability_cannot_be_declared() -> None: + """A tag no scenario carries is a claim nothing can check, so it is refused. + + Refused at construction rather than dropped at emission time, for the same + reason a capability claimed as both declared and impossible is: the adopter + wrote it down and meant something by it, and a config silently different + from the one they wrote is worse than one that will not build. This is also + where their own code is still on the stack. + """ + for reserved in RESERVED_CAPABILITIES: + with pytest.raises(ValueError, match=f"reserved capabilities {reserved.tag}"): + _config(capabilities={Capability.EVENTS, reserved}) + with pytest.raises(ValueError, match=f"reserved capabilities {reserved.tag}"): + _config( + capabilities={Capability.EVENTS}, + not_applicable={reserved: "no scenario asks"}, + ) + + +def test_a_reserved_capability_cannot_reach_the_declaration() -> None: + """Including by the route that actually caused it: declaring everything. + + The schema forbids a capability no executed scenario carries from appearing + in ``declaration.declared``, because such a tag cannot produce a skip and so + plays no part in reading the results -- it only invites a reader to believe + something was verified when nothing examined it. A real report from another + implementation asserts ``@targeting`` and ``@caching`` for exactly this + reason: that adoption declares "every capability except X" and collected the + reserved tags on the way past. So the default is the declarable set rather + than the whole enum. + """ + declared = SuiteReport( + config=_config_leaving_capabilities_to_their_default() + ).build(_results())["declaration"]["declared"] + + assert declared == sorted(capability.tag for capability in DECLARABLE_CAPABILITIES) + assert RESERVED_CAPABILITIES, "the rule is vacuous if nothing is reserved" + for reserved in RESERVED_CAPABILITIES: + assert reserved.tag not in declared + + +def test_known_deviations_are_omitted_rather_than_emitted_empty() -> None: + """Stating none is a claim; omitting the field is silence.""" + assert "knownDeviations" not in SuiteReport(config=_config()).build(_results()) + + acknowledged = SuiteReport( + config=_config( + known_deviations=( + KnownDeviation( + issue=DEVIATION_ISSUE, + summary="a boolean satisfies an Integer request", + capability=Capability.NUMERIC_COERCION, + ), + ) + ) + ).build(_results())["knownDeviations"] + assert acknowledged == [ + { + "issue": DEVIATION_ISSUE, + "summary": "a boolean satisfies an Integer request", + "capability": Capability.NUMERIC_COERCION.tag, + } + ] + + +def test_the_provider_name_falls_back_to_the_suite_name() -> None: + """A suite whose every scenario was skipped never saw a provider. + + Reporting the suite name is more useful than the empty string the schema + would reject. + """ + envelope = SuiteReport(config=_config()).build(_results()) + assert envelope["provider"]["name"] == "stub" + + +def test_the_control_api_is_omitted_when_the_control_does_not_say() -> None: + plain = SuiteReport(config=_config()).build(_results()) + assert "controlApi" not in plain["backend"] + http = SuiteReport(config=_config(control=_HttpControl())).build(_results()) + assert http["backend"]["controlApi"] == "http" + + +def test_control_api_ignores_a_value_the_schema_would_reject() -> None: + class Odd(_StubControl): + control_api = "carrier pigeon" + + assert control_api_of(Odd()) == "" + + +@pytest.mark.parametrize( + ("suite_name", "expected"), + [ + ("in-memory", "in-memory"), + ("flagd/rpc", "flagd-rpc"), + ("../escape", "escape"), + ("...", "report"), + ], +) +def test_a_suite_name_cannot_write_outside_its_directory( + suite_name: str, expected: str +) -> None: + """Suite names are chosen to read well in a failure message, not to be paths.""" + assert envelope_file_name(suite_name) == f"{expected}.json" + assert stream_file_name(suite_name) == f"{expected}.ndjson" + + +def test_only_tags_the_schema_accepts_are_carried() -> None: + assert normalise_tags({"events", "Not A Tag", "stale"}) == ("@events", "@stale") + + +def test_a_feature_uri_is_slash_separated_on_every_platform() -> None: + """The same string has to appear in the source, the document and the pickles. + + pytest-bdd builds it with ``os.path.join``, so on Windows it arrives + backslash-separated -- and a report emitted there would otherwise not be + comparable with one emitted on Linux. + """ + assert feature_uri("features/errors.feature") == "features/errors.feature" + assert feature_uri(os.path.join("features", "errors.feature")) == ( + "features/errors.feature" + ) + + +# -- classifying one phase --------------------------------------------------- + + +def test_an_expected_failure_is_still_a_failure() -> None: + """An xfail marker records a known deviation; it does not excuse one.""" + status, message = classify_phase( + _phase("skipped", xfail_reason="the SDK coerces a bool to an int") + ) + assert status is Status.failed + assert "the SDK coerces a bool to an int" in message + + +def test_a_phase_that_merely_worked_says_nothing_in_particular() -> None: + assert classify_phase(_phase("passed", when="setup")) == (Status.passed, "") + assert classify_phase(_phase("passed", when="call")) == (Status.passed, "") + + +def test_a_skip_keeps_its_reason() -> None: + assert classify_phase( + _phase("skipped", when="setup", message="provider does not declare @stale") + ) == (Status.skipped, "provider does not declare @stale") + assert classify_phase(_phase("skipped", when="setup")) == ( + Status.skipped, + "skipped", + ) diff --git a/uv.lock b/uv.lock index b0ab379c..b52619cf 100644 --- a/uv.lock +++ b/uv.lock @@ -825,6 +825,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/50/a9caea39ad19c431c1a3f8a31114df65b260cdfe67786b6c7e7c040c4c44/cryptography-49.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be9fcb48a55f023493482827d4f459bd263cc20efde64f204b97c123201850c6", size = 3783731, upload-time = "2026-06-12T20:02:43.319Z" }, ] +[[package]] +name = "cucumber-messages" +version = "34.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/b8/16f4f31045776b7b7fc962e72ebc2a8bf4a12df2f18d8c9be4e258cbe3bb/cucumber_messages-34.2.0.tar.gz", hash = "sha256:712102e0a0f7fb7a3d068a2754b31ce9b605fb04ab65f9f37282f9c27f7254d4", size = 11703, upload-time = "2026-07-19T13:07:38.64Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/81/00577acc873ebce750da0429c6726e111c75e62caa71843166e522e19b80/cucumber_messages-34.2.0-py3-none-any.whl", hash = "sha256:2b20b7a7151b2ccc296b8f101b413b2b6b8a2f90c3af748d5a357b59243ee2fc", size = 13006, upload-time = "2026-07-19T13:07:37.692Z" }, +] + [[package]] name = "docker" version = "7.1.0" @@ -844,7 +853,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1995,6 +2004,8 @@ name = "openfeature-provider-tck" version = "0.1.0" source = { editable = "tools/openfeature-provider-tck" } dependencies = [ + { name = "cucumber-messages" }, + { name = "gherkin-official" }, { name = "openfeature-sdk" }, { name = "pytest" }, { name = "pytest-bdd" }, @@ -2009,6 +2020,8 @@ dev = [ [package.metadata] requires-dist = [ + { name = "cucumber-messages", specifier = ">=34.0.0,<35.0.0" }, + { name = "gherkin-official", specifier = ">=29.0.0" }, { name = "openfeature-sdk", specifier = ">=0.8.2" }, { name = "pytest", specifier = ">=8.4.0" }, { name = "pytest-bdd", specifier = ">=8.1.0,<9.0.0" },