diff --git a/.github/component_owners.yml b/.github/component_owners.yml index 4e4a03415e..2d20b2b5ea 100644 --- a/.github/component_owners.yml +++ b/.github/component_owners.yml @@ -46,6 +46,8 @@ components: - toddbaert tools/flagd-http-connector: - liran2000 + tools/tck: + - aepfli ignored-authors: - renovate-bot diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2fbdc4604a..03974f0ced 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,8 +7,57 @@ on: - reopened branches: - main + # Temporary, for the duration of the provider conformance suite's review. + # Without it a stacked pull request gets no CI at all: this filter matches + # the pull request's BASE branch, so only the suite PR itself -- the one + # targeting main -- was ever checked, and the report and adoption PRs + # stacked on it were merged-in-theory and tested never. Remove once the + # chain has landed. See open-feature/spec#417. + - 'feat/provider-tck*' jobs: + # Fast canary for the Provider TCK: runs the full applicable conformance suite three times -- + # against the SDK's InMemoryProvider, against MultiProvider wrapping one of them, and against + # the TCK's own controllable provider, which has a real initialisation and so is the only one + # of the three that covers the @lifecycle scenarios. No Docker, no Compose stack and no + # network. It finishes in seconds, so a broken step definition, a mis-wired capability gate or + # a regression in the shared harness is reported long before the containerised provider suites + # in `main` get there — and it points at the TCK rather than at whichever provider noticed + # first. + # + # Deliberately not a gate on `main`: the two run in parallel so a green run is not delayed. + # The same suite also runs inside `main` as part of the reactor build; this job exists to + # report it fast and in isolation. + tck: + name: Provider TCK (no Docker) + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + with: + # The feature files, canonical flag set and control-API document are copied in from + # the open-feature/spec submodule at generate-resources; without it there is no suite. + submodules: recursive + + - name: Set up JDK 21 + uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5 + with: + java-version: 21 + distribution: 'temurin' + cache: maven + + - name: Cache local Maven repository + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}21-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}21-maven- + + - name: Verify the TCK against the in-memory provider + # No `e2e` profile and no Docker: the in-memory suite is not gated behind either. + run: mvn --batch-mode --activate-profiles codequality -pl tools/tck -am clean verify + main: strategy: matrix: diff --git a/.gitmodules b/.gitmodules index fcfa3cf548..d9b390dac7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -18,3 +18,6 @@ path = tools/flagd-api-testkit/test-harness url = https://github.com/open-feature/test-harness.git branch = v3.10.1 +[submodule "tools/tck/spec"] + path = tools/tck/spec + url = https://github.com/open-feature/spec.git diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 7e3e7f267e..b5c49cb438 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -17,6 +17,7 @@ "tools/flagd-http-connector": "0.0.5", "tools/flagd-api": "1.0.0", "tools/flagd-api-testkit": "0.2.1", + "tools/tck": "0.1.0", "tools/flagd-core": "2.0.1", ".": "1.0.0", "providers/optimizely": "1.0.0" diff --git a/pom.xml b/pom.xml index 29963bb064..bf7b0f7e58 100644 --- a/pom.xml +++ b/pom.xml @@ -28,6 +28,7 @@ + tools/tck tools/flagd-api-testkit tools/flagd-api tools/flagd-core diff --git a/release-please-config.json b/release-please-config.json index d3281c2d59..54129737e7 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -205,6 +205,17 @@ "README.md" ] }, + "tools/tck": { + "package-name": "dev.openfeature.contrib.tools.tck", + "release-type": "simple", + "bump-minor-pre-major": true, + "bump-patch-for-minor-pre-major": true, + "versioning": "default", + "extra-files": [ + "pom.xml", + "README.md" + ] + }, "tools/flagd-api-testkit": { "package-name": "dev.openfeature.contrib.tools.flagdapitestkit", "release-type": "simple", diff --git a/tools/tck/.gitignore b/tools/tck/.gitignore new file mode 100644 index 0000000000..b472ee8dd3 --- /dev/null +++ b/tools/tck/.gitignore @@ -0,0 +1,7 @@ +# Copied from the `spec` submodule at build time (mvn generate-resources). +# Do not edit these files directly — they are the language-agnostic definition of the +# provider contract and live in open-feature/spec, under +# specification/assets/provider-tck/ (Appendix F). +src/main/resources/gherkin/ +src/main/resources/flags/ +src/main/resources/openapi/ diff --git a/tools/tck/README.md b/tools/tck/README.md new file mode 100644 index 0000000000..242d8009b1 --- /dev/null +++ b/tools/tck/README.md @@ -0,0 +1,446 @@ +# OpenFeature Provider TCK (Java) + +A conformance suite any OpenFeature Java provider can adopt to verify that it implements the +provider contract of the [specification](https://openfeature.dev/specification/). + +It is the Java implementation of [Appendix F][appendix-f], which defines the scenarios, the canonical +flag set, the control API and the capability vocabulary that every language's TCK shares. **Read it +for anything true of the suite rather than of this artifact** — what is tested and what is not, the +rules for declaring a capability, what a known deviation means, and how to run an adoption in CI. +Tracking issue: [open-feature/spec#417][tracking]. + +> **Status: proof of concept.** The scenario set covers each architectural mechanism once rather than +> exhaustively. Expect breaking changes. + +## Installation + + +```xml + + dev.openfeature.contrib.tools + tck + 0.1.0 + test + +``` + + +Java 11+ and JUnit 5. Docker is needed only by providers with an external backend. + +**Testcontainers is not transitive.** `ContainerizedProviderTckTest` owns a `ComposeContainer`, so this +artifact compiles against Testcontainers but declares it `provided` and `optional`. A containerised +adopter adds `org.testcontainers:testcontainers` itself — one line for the adopters that need it, and +it keeps Testcontainers off the classpath of every backend-less adopter, which would otherwise resolve +it for a class it never loads. + +**The SDK is a `provided` version range**, `[1.21.0,1.99999)`, inherited from this repository's parent +POM — never a pin. A conformance suite that forces an SDK upgrade before you can run it is one nobody +runs; the TCK uses only long-stable API. + +## Quick start + +Two base classes, and one question chooses between them: **does your provider talk to something +outside the JVM?** + +| | Extend | Backend control | +| --- | --- | --- | +| External backend | `ContainerizedProviderTckTest` | `HttpBackendControl`, over the HTTP control API | +| No backend — in-memory, environment variables, a local file | `ProviderTckTest` | an in-process `BackendControl` | + +### A provider with a backend + +Write a Compose file at `src/test/resources/tck/docker-compose.yaml` exposing your backend and its +[control API][control-api], then one test class: + +```java +public class MyProviderTest extends ContainerizedProviderTckTest { + + @Override + public File composeFile() { + return new File("src/test/resources/tck/docker-compose.yaml"); + } + + @Override + public List backendPorts() { + return Collections.singletonList(5000); + } + + @Override + public FeatureProvider createProvider(BackendEndpoint endpoint) { + return new MyProvider(endpoint.host(), endpoint.port(5000)); + } + + @Override + public FeatureProvider createUnavailableProvider() { + return new MyProvider("localhost", 9999); + } +} +``` + +That is the whole adoption — one file, no registration. The class is simultaneously the JUnit suite +and the harness; the Compose lifecycle, port discovery, control API calls, provider registration, +event awaiting and teardown belong to the TCK, and the three artifacts are packaged in the JAR, so an +adoption needs no submodule. **If you find yourself adding test infrastructure to this class, that is +a defect here — please open an issue.** + +`createUnavailableProvider()` should point at a closed port on localhost, not at your stack: the stack +stays up and outages are simulated through the control API. Give it a short connection deadline, +because the failure scenarios assert that failure is reported *promptly*. + +**Seed your backend with the [canonical flag set][flags]** — several of its properties are +load-bearing and easy to break while seeding, so read the [assets README][assets] rather than retyping +the file. + +### A provider with no backend + +`InProcessBackendControl` implements the in-process path for the SDK's `InMemoryProvider`, seeded from +the packaged flag set. The adoption is three methods: + +```java +public class MyProviderTest extends ProviderTckTest { + + private final InProcessBackendControl control = new InProcessBackendControl(); + + @Override + public BackendControl backendControl() { + return control; + } + + @Override + public FeatureProvider createProvider() { + return control.createProvider(); + } + + @Override + public Set capabilities() { + return EnumSet.of(Capability.EVENTS, Capability.CONFIGURATION_CHANGE, Capability.OBJECT); + } +} +``` + +One object backs both methods because in-process the flag store and the provider are the same thing: +`changeFlag()` has to reach the live provider instance to emit an event from it. + +This path is a narrow allowance for providers with **nothing to contract with**, not an invitation to +skip the control API — see [Appendix F, "Providers with no backend"][appendix-f]. It leaves +`disconnect()` and `reconnect()` throwing, and withholding `STALE` and `UNAVAILABLE_INIT` is what +keeps that honest: declare one anyway and the scenario fails with an `UnsupportedOperationException` +naming the fix, because reaching an unsupported operation from a scenario that actually ran is a +test-configuration bug, never a skip. + +### Several provider modes + +A provider with more than one transport writes **one class per mode and nothing else** — no +registration, no system property, no build configuration. Each class is its own suite with its own +Compose stack, and abstract classes are not run, so an intermediate base is safe. Per-mode differences +may include timing as well as wiring: flagd's in-process resolver syncs the whole ruleset before +reporting ready, so it needs a longer initialisation deadline than its RPC mode. + +## The options + +Eight Compose concepts, with the same names and defaults in every language's TCK, spelled here as +overridable methods on `ContainerizedProviderTckTest`. + +| Concept | Required | Default | Java | +| --- | --- | --- | --- | +| Compose file | yes | — | `File composeFile()` — resolved relative to the Maven module directory | +| Backend service | no | `backend` | `String backendService()` — the service hosting both the control API and the backend | +| Backend ports | yes | — | `List backendPorts()` — container-internal ports the *provider* connects to. Do not list the control port; it is exposed automatically | +| Control port | no | `8080` | `int controlPort()` | +| Additional ports | no | none | `Map> additionalPorts()` — extra service → ports, resolved through the endpoint by service name | +| Backend configuration | no | `default` | `String backendConfiguration()` — the name passed to `POST /start` | +| Startup timeout | no | 60s | `Duration startupTimeout()` — the stack and its control API becoming reachable | +| Endpoint | — | — | `BackendEndpoint` — `host()` and `port(internalPort)`, optionally qualified by service | + +**Never pin host ports.** They are mapped dynamically and discovered after startup, which is why the +provider comes from a factory rather than a constant. The stack may hold any number of extra +containers; the TCK only cares about the conventions above. + +`backendConfiguration()` names a configuration the **backend** understands; `configuration()` names the +mode of the **provider**. The bare word `configuration` meant opposite things in three of the four +languages' first drafts, which is why these two are spelled apart. + +### Timeouts + +| Method | Default | What it bounds | +| --- | --- | --- | +| `eventTimeout()` | 12s | waiting for a provider event | +| `readyTimeout()` | 30s | waiting for a provider to reach a lifecycle state | +| `startupTimeout()` | 60s | the Compose stack and its control API becoming reachable | + +Set `eventTimeout()` to comfortably exceed your worst-case detection latency, or the suite reports +timeouts that are really impatience; scenarios asserting promptness as part of their point use the +explicit `within {int}ms` step, which always wins. Every entry is a **bound on an await** and none is +a pause — nothing sleeps after a control call, for the reason Appendix F's control-API invariants give. + +### Identifying the run + +`configuration()` names which of the provider's *modes* this suite exercised — flagd's RPC and +in-process resolvers run two suites whose results are not interchangeable. It defaults to the suite +class name, hyphenated with the JUnit suffix dropped. **Check it if your suite lives in a package that +already names the provider**, which is what the layout below recommends: `InProcessTest` in +`...providers/flagd/tck/` derives `in-process`, which says nothing about whose in-process mode it was +to a reader away from this repository. flagd's two suites state `flagd-rpc` and `flagd-in-process` +outright. + +`BackendControl.controlApi()` says which contract a run was conducted under, `ControlApi.HTTP` or +`ControlApi.IN_PROCESS`. It is abstract and the enum closed, because Appendix F requires the control +to *state* the path rather than have the harness infer it from a concrete type. + +## Declaring capabilities + +Each scenario exercising an optional part of the contract carries a tag, and a provider declares what +it supports. Undeclared ones are reported as **skipped with the reason** — never as passed. Each name +below is a member of `Capability`. + +| Capability | Tag | | Capability | Tag | +| --- | --- | --- | --- | --- | +| `LIFECYCLE` | `@lifecycle` | | `UNAVAILABLE_INIT` | `@unavailable` | +| `REINITIALIZATION` | `@reinitialization` | | `NUMERIC_COERCION` | `@numeric-coercion` | +| `EVENTS` | `@events` | | `TARGETING` | `@targeting` | +| `STALE` | `@stale` | | `STANDARD_REASONS` | `@standard-reasons` | +| `CONFIGURATION_CHANGE` | `@configuration-change` | | `STRING_TYPING` | `@string-typing` | +| `OBJECT` | `@object` | | `LARGE_INTEGERS` | `@large-integers` ¹ | +| `VARIANTS` | `@variants` | | `CACHING` | `@caching` ² | +| `DISABLED_FLAGS` | `@disabled-flags` | | `FULLY_TYPED_VALUES` | `@fully-typed-values` | + +¹ not declarable in Java    ² reserved, not declarable + +What each tag means, and — more importantly — **when to declare one and when to withhold it** are +[Appendix F's][appendix-f], under "Capabilities" and "Rules for declaring". The decision is per +*scenario* rather than per tag, and three of the four implementations got that wrong in three +different directions, so it is worth the read. + +The default is every *declarable* capability. **Narrow it, do not widen it**: start from the default, +run the suite, and remove only what your provider genuinely cannot do. + +```java +@Override +public Set capabilities() { + return Capability.declarableExcept(Capability.STALE); +} +``` + +**Use `Capability.declarable()` and `declarableExcept(...)`, not `EnumSet.complementOf(...)`.** +`complementOf(EnumSet.of(X))` reads as "everything except X" and in fact means "every other enum +constant", reserved and inexpressible ones included. The flagd suite said exactly that and published +`"declared": [..., "@targeting", "@caching"]` for two capabilities nobody had claimed. The two +factories mean what the first one looks like, and naming a refused capability directly is an error +rather than a quiet correction, so you cannot get this wrong silently either. + +**`LARGE_INTEGERS` is inexpressible in Java, and you do not have to know anything about it.** +`Client.getIntegerDetails` takes and returns a 32-bit `Integer`, which has no room for 2^53 − 1, so no +Java provider can be asked the question until the SDK grows a wider accessor. Appendix F's rule is +that such a capability is refused by the implementation rather than left to every adopter to remember +— this module had the same paragraph restated in four places before it was. Its scenario is skipped +with a reason naming the **SDK**, so a report's reader can tell *"this provider declined"* from *"no +Java provider can be asked"*; the 32-bit precision scenario is untagged and always runs. A reserved +capability is a different thing and its reason says so. Neither refusal is a defect, and neither needs +a `KnownDeviation`. + +**`STRING_TYPING` and `FULLY_TYPED_VALUES` are a pair, and the narrower one is a claim about the +backend.** `@string-typing` asks whether a boolean or an integer flag requested as a string is a +`TYPE_MISMATCH` rather than its string representation; `@fully-typed-values` asks the same of a float +and of a structure. Every scenario the second gates carries the first as well, so declare the second +only alongside the first — and expect to withhold it alone, because a store that records booleans and +integers natively while keeping floats and structures as text can answer two of the four cases and +not the other two. It was a single tag until specification revision `bda599f1`; the split exists +because one tag let a real defect in one language be published as a permitted absence, and Appendix F +carries the measurement that showed it. Neither tag rests on a `MUST`, so withholding either needs no +`KnownDeviation`. + +### Known deviations + +**A `knownDeviations` entry says: this provider fails to do something it is required to do.** The +requirement must be a numbered `MUST`, or a rule the implementation bound itself to elsewhere. Where +the specification *permits* the choice, withholding the capability **is** the honest report. + +```java +@Override +public List knownDeviations() { + return List.of(KnownDeviation.tracked( + Capability.NUMERIC_COERCION, + "https://github.com/open-feature/java-sdk-contrib/issues/1234", + "float-flag through the integer API returns 0 with no error code")); +} +``` + +The two legitimate shapes, and why the declared-and-failing one is preferred, are +[Appendix F's][appendix-f]. `summary` is required and `issue` is not — `KnownDeviation.untracked(...)` +is the untracked form. The capability may be `null` for a mandatory, ungated scenario; it may not be +`CACHING` or `LARGE_INTEGERS`, since no scenario was ever put to your provider for either, and both +are refused where you write them. Empty is the default, and it is silence rather than a claim of +having none. + +## Running it + +```bash +# once, if this module is not in your local repository yet +mvn -pl tools/tck -am -DskipTests install + +mvn -Ptck -pl providers/ test +``` + +**Resist adding `-am` to the second line.** It pulls this module — and anything else the adoption +depends on — into the reactor and runs their suites before the first scenario, so a failure in any of +them comes out as a `-Ptck` failure. The one-off `install` is what `-am` was there for. + +Scenarios run **serially**, enforced over any `cucumber.execution.parallel.enabled=true` in your +module: control API state is global to the stack, so concurrent scenarios corrupt each other and the +symptom looks like a flaky provider. The stack starts once per suite and is never restarted. + +**Put the adoption in a `tck` package of its own**, beside your module's other test packages rather +than inside one — in `providers/flagd` that is +`src/test/java/dev/openfeature/contrib/providers/flagd/tck/`, a sibling of `e2e` rather than a corner +of it. Why a conformance suite is not a kind of end-to-end test, and why selection by directory beats +selection by filename, are [Appendix F's][appendix-f] under "Running the suite in CI". Once the +directory selects, names that repeat it say the same thing twice — `FlagdRpcTckTest` in package +`...flagd.tck` is `RpcTest` — but **keep the `*Test` suffix**, which Surefire's default includes need, +and check `configuration()` when you rename. + +### The Maven mechanism + +Appendix F has the reasoning for excluding an adoption from the default build and giving it a step of +its own; this is only how that is spelled here. The exclusion is the `testExclusions` property, which +the parent POM feeds to Surefire and gives no default, so a module that wants the gate declares it, +listing every Docker-dependent package — `providers/flagd` has its legacy `e2e` suites too: + +```xml + + **/e2e/*.java,**/tck/*.java + +``` + +It is a **Surefire** exclusion and not a compiler one, so the adoption still compiles against the +harness in every build — the "keep it typechecked by something that runs ordinarily" property the +appendix asks for, which Maven gives for free. + +**Then resolve the property under every profile your CI activates, rather than reading the POM.** Both +halves of the appendix's warning happened in this repository — one adoption never declared the +property, the other had a profile putting it back — and both were found by running this: + +```bash +mvn -Pe2e -pl providers/ help:evaluate -Dexpression=testExclusions -DforceStdout +``` + +The dedicated step is a profile, one per adopting module, named `tck` because JavaScript's `nx tck` +target and Python's `poe test-tck` task already spell it that way. It drops the `tck` directory from +the exclusion **and** narrows Surefire's includes to it: + +```xml + + tck + + + **/e2e/*.java + + + org.apache.maven.plugins + maven-surefire-plugin + **/tck/*.java + + +``` + +**Both halves are needed.** Dropping alone runs the module's unit tests alongside the suites; +narrowing alone leaves the exclusion in force and runs nothing. + +## Extending it + +A provider with features of its own — flagd's `fractional` targeting, a vendor's proprietary mode — +extends the suite rather than maintaining a second one. Two files, no annotations: + +``` +src/test/resources/extensions/fractional.feature +src/test/java/openfeature/tck/extensions/FractionalSteps.java // package openfeature.tck.extensions +``` + +Both are already selected by `ProviderTckTest`, so your scenarios run **inside** the suite: same +backend lifecycle, same `@BeforeAll`, same `BackendControl`, and canonical steps are on the glue path +too, so an extension scenario can open with `Given a stable provider`. A step class may take +`TckState` as a constructor argument exactly as the canonical steps do, and reach the backend control +and endpoint through `TckRuntime.get()` — **build a client of your own instead and you resolve against +a provider this suite never registered.** If a scenario is portable across providers, send it to the +TCK rather than keeping it as an extension. + +`gherkin/` and `extensions/` are Appendix F's names, and being two distinct directories is what makes +shadowing unreachable here: two classpath roots holding the same directory are scanned additively, but +two holding the same directory *and* the same file name are not — one wins silently, so a +`gherkin/errors.feature` in your test resources would *replace* the canonical file and the suite would +report success having run yours. The `extensions/` directory ships in this JAR holding only a README, +because a classpath resource selector naming a resource on no classpath root is a hard discovery error +rather than an empty selection. + +`ProviderTck` names every value the suite's annotations carry — `FEATURES`, `EXTENSIONS`, `GLUE`, +`EXTENSION_GLUE`, `ALL_GLUE` and the rest of the Cucumber configuration — so an adopter who does write +a `@ConfigurationParameter` composes rather than copies, an annotation value having to be a +compile-time constant. Keep `ProviderTck.GLUE` in it; dropping it makes every canonical step undefined. + +## Java notes + +**Suite discovery** relies on the JUnit Platform auto-registering `TckSuiteListener`, declared in this +JAR's `META-INF/services/org.junit.platform.launcher.TestExecutionListener`, which Surefire, Gradle +and IDEs all do by default. If your launcher disables listener auto-registration, register the harness +explicitly at `src/test/resources/META-INF/services/dev.openfeature.contrib.tools.tck.ProviderTckHarness` +and select between several with `-Dopenfeature.tck.harness=RpcTest`. + +**This module's own suites are the reference adoption to copy**, and all three run without Docker in +under a second: `InMemoryProviderTckTest` against the SDK's `InMemoryProvider`, +`ControllableProviderTckTest` against a provider with a real initialisation — the only Docker-free +cover the `@lifecycle` feature has — and `MultiProviderTckTest` against `MultiProvider` wrapping one +`InMemoryProvider`, where any difference from the first is attributable to delegation and nothing +else. That last one has already paid for itself: it cannot declare `CONFIGURATION_CHANGE`, because +`MultiProvider` never subscribes to its children and swallows their events — +[java-sdk#1882](https://github.com/open-feature/java-sdk/issues/1882), reproduced from the outside. +Appendix F's carve-out for a self-test withholding a capability over a defect applies to these and +**not** to an adoption. + +**The packaged artifacts are generated.** The Gherkin, flag set and control-API document live in +[open-feature/spec][spec] under `specification/assets/provider-tck/`; the `spec` submodule is updated +at `initialize` and the three directories are copied into `src/main/resources/` at +`generate-resources`. **Do not edit `src/main/resources/gherkin/`, `flags/` or `openapi/`** — they are +git-ignored, and changes belong upstream. Appendix F asks that a stale checkout be unrunnable rather +than merely discouraged, since a rebase moves the gitlink while only `git submodule update` moves the +working tree; three things enforce that here. The checkout runs on every build, skippable only through +the dedicated `-Dtck.spec.checkout.skip=true`; the generated directories are emptied before the copy, +so a file present in the old pin and not the new one cannot survive; and `CanonicalAssetDigestTest` +fails the build by digest over all three directories, which is the only one of the three that catches +a pin whose sole change is *content*. + +**A fourth check faces the other way: a pin that arrives with a capability tag this module has not +learned.** An unknown tag gates nothing, so its scenarios stay mandatory for *every* adopter — the +suite does not report a new capability, it quietly keeps demanding the old behaviour, and the only +symptom is the one provider that legitimately cannot support it failing while the rest stay green. +`CanonicalTagCoverageTest` fails this build for it, and `CapabilityGate.requireKnownVocabulary` fails +an adopter's run for it, which is where Appendix F asks the check to be in force. It applies to +`gherkin/` only: a feature file of your own under `extensions/` is expected to carry tags this +vocabulary does not know. + +**The step vocabulary** is inherited from the [flagd test harness](https://github.com/open-feature/test-harness) +wherever it was already provider-neutral, so flagd's feature files ported with a near-zero diff; only +`Given a stable flagd provider` and `Given a unavailable flagd provider` were renamed, to drop the +vendor. The canonical set adds seven steps of its own, and each carries its rationale as javadoc on +the method that binds it, in `steps/ProviderSteps` and `steps/FlagSteps` — read those before writing +an extension step, since several of them exist to keep a scenario vendor-neutral in a way that is not +obvious from the wording. + +## Known gaps + +[Appendix F][appendix-f] carries the suite's gaps — context passthrough beyond the targeting key, +per-flag control operations, caching, `@stale` without containers, hooks, flag metadata, and the +requirements not yet covered. One is Java's alone: **`TckRuntime` is static**, so TCK suites run one +at a time within a JVM fork. Several suites in one fork is fine — they run sequentially, each with its +own Compose stack — but they cannot run concurrently. + +## Contributing + +See the repository [CONTRIBUTING.md](../../CONTRIBUTING.md). New scenarios should be portable across +providers: a scenario that can only pass against one vendor's backend semantics belongs in that +provider's own suite, not here. + +[appendix-f]: https://github.com/open-feature/spec/blob/main/specification/appendix-f-provider-conformance.md +[assets]: https://github.com/open-feature/spec/blob/main/specification/assets/provider-tck/README.md +[control-api]: https://github.com/open-feature/spec/blob/main/specification/assets/provider-tck/openapi/control-api.yaml +[flags]: https://github.com/open-feature/spec/blob/main/specification/assets/provider-tck/flags/canonical-flags.json +[spec]: https://github.com/open-feature/spec +[tracking]: https://github.com/open-feature/spec/issues/417 diff --git a/tools/tck/lombok.config b/tools/tck/lombok.config new file mode 100644 index 0000000000..df71bb6a0f --- /dev/null +++ b/tools/tck/lombok.config @@ -0,0 +1,2 @@ +config.stopBubbling = true +lombok.addLombokGeneratedAnnotation = true diff --git a/tools/tck/pom.xml b/tools/tck/pom.xml new file mode 100644 index 0000000000..99e5987db4 --- /dev/null +++ b/tools/tck/pom.xml @@ -0,0 +1,378 @@ + + + 4.0.0 + + dev.openfeature.contrib + parent + [1.0,2.0) + ../../pom.xml + + dev.openfeature.contrib.tools + tck + 0.1.0 + + + ${groupId}.tck + + + false + + 3.27.7 + 4.3.0 + 2.22.1 + 2.0.17 + 2.0.4 + 1.3.0 + + + tck + + Language-agnostic conformance test suite (TCK) for OpenFeature providers. + Bundles the canonical Gherkin feature files, Cucumber step definitions and + abstract JUnit Platform Suite base classes that own the full test lifecycle. + Providers with an external backend extend ContainerizedProviderTckTest, which + starts the vendor-supplied Docker Compose stack, discovers dynamically mapped + ports and drives the standardised HTTP control API. Providers with no backend + extend ProviderTckTest and supply in-process backend control. Provider authors + implement a small factory interface either way. + + https://openfeature.dev + + + + aepfli + Simon Schrottner + OpenFeature + https://openfeature.dev/ + + + + + + + + + + + io.cucumber + cucumber-java + + + + + + io.cucumber + cucumber-junit-platform-engine + + + + + io.cucumber + cucumber-picocontainer + compile + + + + + org.junit.platform + junit-platform-suite + compile + + + + + org.junit.platform + junit-platform-launcher + compile + + + + + org.opentest4j + opentest4j + ${opentest4j.version} + compile + + + + + org.assertj + assertj-core + ${assertj.version} + compile + + + + + org.awaitility + awaitility + ${awaitility.version} + compile + + + + + org.testcontainers + testcontainers + ${testcontainers.version} + provided + true + + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson-databind.version} + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + + org.junit.jupiter + junit-jupiter + + test + + + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.6.3 + + + update-spec-submodule + initialize + + exec + + + ${tck.spec.checkout.skip} + git + + submodule + update + --init + spec + + + + + + + + + maven-clean-plugin + 3.5.0 + + + clear-generated-spec-assets + generate-resources + + clean + + + true + + + ${basedir}/src/main/resources/gherkin + + + ${basedir}/src/main/resources/flags + + + ${basedir}/src/main/resources/openapi + + + + + + + + + + maven-resources-plugin + 3.5.0 + + + copy-provider-tck-gherkin + generate-resources + + copy-resources + + + ${basedir}/src/main/resources/gherkin/ + + + ${basedir}/spec/specification/assets/provider-tck/gherkin/ + + **/*.feature + + + + + + + copy-provider-tck-flags + generate-resources + + copy-resources + + + ${basedir}/src/main/resources/flags/ + + + ${basedir}/spec/specification/assets/provider-tck/flags/ + + **/*.json + + + + + + + copy-provider-tck-openapi + generate-resources + + copy-resources + + + ${basedir}/src/main/resources/openapi/ + + + ${basedir}/spec/specification/assets/provider-tck/openapi/ + + **/*.yaml + + + + + + + + + + + diff --git a/tools/tck/spec b/tools/tck/spec new file mode 160000 index 0000000000..ff68adb4c7 --- /dev/null +++ b/tools/tck/spec @@ -0,0 +1 @@ +Subproject commit ff68adb4c7617ad2d980988241e92603bc247926 diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/BackendControl.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/BackendControl.java new file mode 100644 index 0000000000..8f142453d7 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/BackendControl.java @@ -0,0 +1,109 @@ +package dev.openfeature.contrib.tools.tck; + +/** + * The single seam between the TCK's step definitions and whatever manipulates the backend. + * + *

Step definitions never talk to a backend directly. They talk to this interface, which is why + * the same Gherkin runs unchanged against a containerised backend driven over HTTP + * ({@link HttpBackendControl}) and against a provider manipulated in-process + * ({@link InProcessBackendControl}). Nothing below this line knows about ports, containers or + * transports. + * + *

Which implementation is right for your provider

+ * + *

A provider that talks to a backend uses {@link HttpBackendControl} by extending + * {@link ContainerizedProviderTckTest}; in-process control is for a provider with no + * backend to contract with — see {@link InProcessBackendControl}. That allowance is narrow, and + * Appendix + * F says why a custom in-JVM control reaching an external backend through a side channel passes + * while proving nothing. + * + *

Operations a backend may not support

+ * + *

{@link #prepareScenario()}, {@link #changeFlag()} and {@link #controlApi()} are mandatory. The + * two connection operations are not: a provider with nothing to disconnect from leaves them at + * their defaults, which throw {@link UnsupportedOperationException}. That exception is a + * test-configuration bug, never a skip — the scenarios needing connection control + * are gated behind {@link Capability#STALE} and {@link Capability#UNAVAILABLE_INIT}, so reaching a + * default means a capability was declared the backend cannot back up, and a silent no-op there + * would report the scenario as passed. + * + * @see Capability + * @see ProviderTckTest + */ +public interface BackendControl { + + /** + * Brings the backend to the state every scenario starts from: reachable, with flag state at the + * baseline of the canonical flag set. + * + *

Called once before each scenario. This is the TCK's only isolation mechanism — scenarios + * share one backend for the whole suite, and containers are never restarted between them. + */ + void prepareScenario(); + + /** + * Mutates flag configuration so that a conforming provider observes a configuration change and + * resolves a different value for {@code changing-flag} afterwards. + * + *

Which value it changes to is deliberately unspecified; the suite asserts only that the + * resolved value differs from what it was before. + */ + void changeFlag(); + + /** + * Makes the backend unreachable for the rest of the scenario, without stopping any container. + * + * @throws UnsupportedOperationException if this backend has no connection to lose + */ + default void disconnect() { + throw unsupported("disconnect"); + } + + /** + * Makes the backend reachable again after {@link #disconnect()}, preserving flag state so the + * provider observes an availability change rather than a configuration change. + * + * @throws UnsupportedOperationException if this backend has no connection to restore + */ + default void reconnect() { + throw unsupported("reconnect"); + } + + /** + * Returns a short description of what is being controlled, for startup logging and for the + * failure messages of unsupported operations. + * + * @return a human-readable description of this backend control + */ + default String description() { + return getClass().getSimpleName(); + } + + /** + * Returns how the backend is driven, as one of the two kinds the provider contract recognises. + * + *

Appendix F requires the control to state this rather than the harness to infer it, and is + * why there is deliberately no default: an omitted value would be an unfalsifiable claim rather + * than no claim. Both implementations the TCK ships answer it, so the only author who has to is + * the one writing a custom control — precisely the case where it cannot be inferred. + * + * @return which of the two control contracts this run is conducted under + */ + ControlApi controlApi(); + + /** + * Builds the exception the connection-control defaults throw. + * + * @param operation the operation that is not supported + * @return the exception to throw + */ + default UnsupportedOperationException unsupported(String operation) { + return new UnsupportedOperationException(description() + " does not support '" + operation + + "'. This is a test-configuration bug rather than a provider defect: a scenario " + + "needing connection control ran, so the harness declared Capability.STALE or " + + "Capability.UNAVAILABLE_INIT for a backend that cannot simulate an outage. " + + "Remove those capabilities from the harness, or supply a BackendControl that " + + "implements them."); + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/BackendEndpoint.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/BackendEndpoint.java new file mode 100644 index 0000000000..88b37db0e1 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/BackendEndpoint.java @@ -0,0 +1,73 @@ +package dev.openfeature.contrib.tools.tck; + +import org.testcontainers.containers.ComposeContainer; + +/** + * Addresses of the running backend stack, handed to + * {@link ContainerizedProviderTckTest#createProvider(BackendEndpoint)}. + * + *

This type exists because host ports are only known after the Compose stack has + * started, which is why the harness exposes a factory method rather than a pre-built provider. The + * mapping is stable for the lifetime of the suite, since the stack is started once and never + * restarted — see the no-container-restart invariant in {@code openapi/control-api.yaml}. + */ +public final class BackendEndpoint { + + private final ComposeContainer compose; + private final String defaultService; + + BackendEndpoint(ComposeContainer compose, String defaultService) { + this.compose = compose; + this.defaultService = defaultService; + } + + /** + * Returns the host the stack is reachable on. + * + *

This is not necessarily {@code localhost}: with a remote Docker daemon, Docker Desktop on + * some platforms, or a rootless setup, it can be an arbitrary address. Always use this value + * rather than hard-coding a host. + * + * @return the Docker host serving the backend stack + */ + public String host() { + return compose.getServiceHost(defaultService, null); + } + + /** + * Returns the host the named service is reachable on. + * + * @param service the Compose service name + * @return the Docker host serving that service + */ + public String host(String service) { + return compose.getServiceHost(service, null); + } + + /** + * Resolves the dynamically mapped host port for a container-internal port on the default + * backend service. + * + * @param internalPort the container-internal port, as declared by + * {@link ContainerizedProviderTckTest#backendPorts()} + * @return the host port the service is reachable on + */ + public int port(int internalPort) { + return port(defaultService, internalPort); + } + + /** + * Resolves the dynamically mapped host port for a container-internal port on a named service. + * + *

Use this for multi-service stacks — a proxy, an edge service, a sidecar. The service and + * port must have been declared via {@link ContainerizedProviderTckTest#additionalPorts()}, + * otherwise Testcontainers has not exposed it and this call fails. + * + * @param service the Compose service name + * @param internalPort the container-internal port + * @return the host port the service is reachable on + */ + public int port(String service, int internalPort) { + return compose.getServicePort(service, internalPort); + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/CanonicalFlags.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/CanonicalFlags.java new file mode 100644 index 0000000000..0c498f9ecd --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/CanonicalFlags.java @@ -0,0 +1,224 @@ +package dev.openfeature.contrib.tools.tck; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.openfeature.sdk.Value; +import dev.openfeature.sdk.providers.memory.Flag; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * The canonical flag set, decoded from the definition this artifact packages. + * + *

{@code flags/canonical-flags.json} is one of the three language-agnostic conformance artifacts. + * It is not owned here: it lives in open-feature/spec under + * {@code specification/assets/provider-tck/}, is copied in from the {@code spec} submodule at build + * time and is packaged into the release JAR, which is why this class reads it off the classpath. + * + *

Why decoded rather than transcribed. A hand-written copy inside the TCK would + * have the in-process self-tests verify the suite against a second baseline of our own, so a rename + * in the spec makes them pass against the wrong flags while reporting green. + * + *

The file is flagd's flag-definition format — + * {"flags": {"<key>": {"state", "variants", "defaultVariant"}}}. {@code $comment} + * members are documentation and are ignored wherever they appear. + * + *

What the decoding has to preserve

+ * + *

A loader that "cleans up" values destroys exactly what the scenarios test, so three properties + * of the file survive it deliberately: + * + *

    + *
  • {@code missing-flag} is absent, which is what the {@code FLAG_NOT_FOUND} scenario tests. + * Nothing here adds flags the file does not define. + *
  • no flag carries a {@link dev.openfeature.sdk.providers.memory.ContextEvaluator}, so every + * evaluation resolves the flag's default variant whatever the context — which is what lets a + * provider declaring {@link Capability#STANDARD_REASONS} report {@code STATIC} rather than + * {@code TARGETING_MATCH} for them. The TCK tests a provider's mapping of a response, not a + * backend's evaluation logic. + *
  • a number keeps the width and the kind it was written with. {@code 10} becomes an + * {@link Integer} and {@code 10.0} a {@link Double}, because + * {@link dev.openfeature.sdk.providers.memory.InMemoryProvider} matches a variant by type: an + * {@code integral-float-flag} decoded as the integer {@code 10} would let the + * lossless-coercion scenario pass without anything being coerced. {@code 2147483647} fits an + * {@link Integer} and stays one, so {@code getIntegerDetails} can ask for it; 2^53 − 1 does + * not and becomes a {@link Long}. See {@link #number}. + *
+ */ +final class CanonicalFlags { + + /** + * Classpath location of the canonical flag definition, packaged by the {@code + * copy-provider-tck-flags} execution in this module's POM. + */ + static final String RESOURCE = "flags/canonical-flags.json"; + + /** Documentation member, ignored wherever it appears. */ + private static final String COMMENT = "$comment"; + + private static final String ENABLED = "ENABLED"; + private static final String DISABLED = "DISABLED"; + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private CanonicalFlags() {} + + /** + * Reads the packaged canonical flag definition. + * + * @return the raw JSON bytes + * @throws IllegalStateException if the definition is not on the classpath + */ + static byte[] definition() { + ClassLoader loader = CanonicalFlags.class.getClassLoader(); + try (InputStream in = loader.getResourceAsStream(RESOURCE)) { + if (in == null) { + throw new IllegalStateException("The canonical flag definition " + RESOURCE + " is not on the " + + "classpath. It is copied in from the spec submodule by the copy-provider-tck-flags " + + "execution and packaged into this artifact, so a run without it is a build problem " + + "rather than a provider defect: run 'mvn generate-resources' on tools/tck, " + + "having checked the spec submodule out."); + } + return in.readAllBytes(); + } catch (IOException e) { + throw new UncheckedIOException("Could not read the canonical flag definition " + RESOURCE, e); + } + } + + /** + * Decodes the packaged canonical flag definition into {@code InMemoryProvider} flags. + * + * @return the canonical flag set, in the order the file defines it, unmodifiable + * @throws IllegalStateException if the definition is missing or is not the shape this decoder + * expects, which for a pinned spec revision means the pin moved under it + */ + static Map> flagSet() { + return decode(definition()); + } + + /** + * Decodes a canonical flag definition. + * + * @param json the definition, in flagd's flag-definition format + * @return the flag set, in the order the document defines it, unmodifiable + * @throws IllegalStateException if the document is not the shape this decoder expects + */ + static Map> decode(byte[] json) { + JsonNode root; + try { + root = MAPPER.readTree(json); + } catch (IOException e) { + throw new IllegalStateException("The canonical flag definition " + RESOURCE + " is not valid JSON", e); + } + + JsonNode flags = root.path("flags"); + if (!flags.isObject()) { + throw new IllegalStateException("The canonical flag definition " + RESOURCE + + " has no 'flags' object. The format is {\"flags\": {\"\": {...}}}."); + } + + Map> decoded = new LinkedHashMap<>(); + for (Iterator> it = flags.fields(); it.hasNext(); ) { + Map.Entry entry = it.next(); + if (COMMENT.equals(entry.getKey())) { + continue; + } + decoded.put(entry.getKey(), flag(entry.getKey(), entry.getValue())); + } + if (decoded.isEmpty()) { + throw new IllegalStateException( + "The canonical flag definition " + RESOURCE + " defines no flags, so there is nothing to seed."); + } + return Collections.unmodifiableMap(decoded); + } + + /** Decodes one flag definition. */ + private static Flag flag(String key, JsonNode definition) { + String state = definition.path("state").asText(null); + if (!ENABLED.equals(state) && !DISABLED.equals(state)) { + throw new IllegalStateException("Canonical flag '" + key + "' has state '" + state + "', which is neither " + + ENABLED + " nor " + DISABLED + "."); + } + + String defaultVariant = definition.path("defaultVariant").asText(null); + if (defaultVariant == null) { + throw new IllegalStateException("Canonical flag '" + key + "' names no defaultVariant."); + } + + JsonNode variants = definition.path("variants"); + if (!variants.isObject()) { + throw new IllegalStateException("Canonical flag '" + key + "' has no 'variants' object."); + } + + Map values = new LinkedHashMap<>(); + for (Iterator> it = variants.fields(); it.hasNext(); ) { + Map.Entry variant = it.next(); + if (COMMENT.equals(variant.getKey())) { + continue; + } + values.put(variant.getKey(), value(key, variant.getKey(), variant.getValue())); + } + if (!values.containsKey(defaultVariant)) { + throw new IllegalStateException("Canonical flag '" + key + "' resolves to variant '" + defaultVariant + + "', which it does not define. Its variants are " + values.keySet() + "."); + } + + return Flag.builder() + .variants(values) + .defaultVariant(defaultVariant) + .disabled(DISABLED.equals(state)) + .build(); + } + + /** Converts one variant value to what {@code InMemoryProvider}'s type matching expects. */ + private static Object value(String key, String variant, JsonNode node) { + switch (node.getNodeType()) { + case BOOLEAN: + return node.booleanValue(); + case STRING: + return node.textValue(); + case NUMBER: + return number(key, variant, node); + case OBJECT: + case ARRAY: + // The same conversion the feature files go through for an Object value, so a seeded + // structure and an expected one are comparable: see TckValues. + return Value.objectToValue(MAPPER.convertValue(node, Object.class)); + case NULL: + return null; + default: + throw new IllegalStateException("Canonical flag '" + key + "', variant '" + variant + "' is a " + + node.getNodeType() + ", which is not a flag value."); + } + } + + /** + * Splits a JSON number on how it was written, and on whether it fits. + * + *

This is the load-bearing half of the decoding. A literal with a fraction or an exponent is a + * {@link Double} and an integral one is an {@link Integer} or, where 32 bits have no room for it, + * a {@link Long}. {@code InMemoryProvider} matches a variant against the requested type, so it is + * the decoded type that decides whether {@code integer-flag} is an integer flag and + * {@code integral-float-flag} a float one — and the file's own comment warns that a loader which + * turns {@code 10.0} back into {@code 10} lets the lossless-coercion scenario pass without + * coercing anything. + */ + private static Object number(String key, String variant, JsonNode node) { + if (!node.isIntegralNumber()) { + return node.doubleValue(); + } + if (node.canConvertToInt()) { + return node.intValue(); + } + if (node.canConvertToLong()) { + return node.longValue(); + } + throw new IllegalStateException("Canonical flag '" + key + "', variant '" + variant + "' is " + node.asText() + + ", which does not fit a Long. No canonical value exceeds 2^53 - 1."); + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/Capability.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/Capability.java new file mode 100644 index 0000000000..1662ef9968 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/Capability.java @@ -0,0 +1,442 @@ +package dev.openfeature.contrib.tools.tck; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.EnumSet; +import java.util.List; +import java.util.Optional; + +/** + * An optional part of the OpenFeature provider contract that a provider may or may not support. + * + *

Each capability is one Gherkin tag, declared through {@link ProviderTckHarness#capabilities()}. + * A scenario carrying an undeclared tag is reported as skipped, never as passed; an + * untagged scenario is mandatory. The vocabulary, what each tag means and the rules for declaring + * are + * Appendix + * F's; the javadoc below adds only what is specific to this SDK or to this implementation. + * + *

Two entries are not an adopter's choice at all, and {@link #requireDeclarable} refuses both + * rather than leaving them to be remembered. A {@linkplain #reserved() reserved} one — {@link + * #CACHING} — has no scenarios in any language; an {@linkplain #inexpressible() inexpressible} one — + * {@link #LARGE_INTEGERS} — has scenarios that run elsewhere and no way to ask them through this + * SDK. Build a declaration with {@link #declarable()} or {@link #declarableExcept}, never with + * {@code EnumSet.allOf} or {@code EnumSet.complementOf}, which sweep both up on the way past. + * + *

{@link #STALE} and {@link #UNAVAILABLE_INIT} are the two that need a backend the provider can + * be cut off from, so they are what a harness with an in-process {@link BackendControl} leaves + * undeclared. Every step that would reach {@link BackendControl#disconnect()}, + * {@link BackendControl#reconnect()} or {@link ProviderTckHarness#createUnavailableProvider()} sits + * in a scenario carrying one of the two; declaring them anyway surfaces as an + * {@link UnsupportedOperationException} rather than a skip, which is deliberate. + */ +public enum Capability { + + /** + * Provider performs an initialisation that reaches its backend, with an observable outcome. + * + *

The test is whether initialisation acquires something it did not already hold and + * can be refused, not whether the thing acquired is across a socket. The TCK's own + * {@code ControllableProviderTckTest} declares this against a store in the same JVM, because + * that store is read at {@code initialize()} time and can decline. The SDK's + * {@code InMemoryProvider} cannot, which is why {@code InMemoryProviderTckTest} withholds it. + * + *

Deliberately not {@link #EVENTS}, and in Java the reason is concrete: + * {@code dev.openfeature.sdk.FeatureProviderStateManager} emits {@code PROVIDER_READY} and + * {@code PROVIDER_ERROR} around {@code initialize} for any provider, whether or not it + * is an {@code EventProvider}, so a provider with no initialisation of its own reaches + * {@code READY} exactly as {@code NoOpProvider} would. + */ + LIFECYCLE("@lifecycle"), + + /** + * Provider can be initialised again after {@code shutdown} and serves flags afterwards. + * + *

Gates exactly one scenario, "A provider that was shut down can be initialized again", and + * it is gated because + * Requirement + * 2.5.2 permits reuse rather than requiring it — so withholding it is a + * choice and needs no {@link KnownDeviation}. Appendix F records why the scenario is gated + * separately from {@link #LIFECYCLE} rather than left mandatory. + * + *

Declaring {@code LIFECYCLE} and withholding this one is the expected combination for a + * provider whose initialisation reaches a backend it does not reopen. The scenario carries both + * tags, so a provider declaring neither sees it skipped for {@code @lifecycle}. + */ + REINITIALIZATION("@reinitialization"), + + /** Provider emits lifecycle events at all ({@code PROVIDER_READY}, {@code PROVIDER_ERROR}). */ + EVENTS("@events"), + + /** Provider enters {@code STALE} and emits {@code PROVIDER_STALE} when the backend is lost. */ + STALE("@stale"), + + /** Provider detects flag configuration changes and emits {@code PROVIDER_CONFIGURATION_CHANGED}. */ + CONFIGURATION_CHANGE("@configuration-change"), + + /** Provider supports structured (object) flag values. */ + OBJECT("@object"), + + /** + * Provider names the variant it resolved. + * + *

Gated because a variant is optional rather than required: + * Requirement + * 2.2.4 is a {@code SHOULD} and + * {@code types.md} + * types the field as optional. Withholding it needs no {@link KnownDeviation}. The value + * assertions are untagged and unaffected; the reason is the same shape of question one + * requirement further on and is gated the same way — see {@link #STANDARD_REASONS}. + */ + VARIANTS("@variants"), + + /** + * Provider resolves a flag disabled in the management system to the caller's default value. + * + *

Gated on whether the provider is told the flag was deliberately disabled, + * which is the part of this tag no other document states. A provider that evaluates locally — + * flagd's resolvers, an in-memory provider — reads the state itself. A provider whose backend + * decides can only substitute the caller's default if the response distinguishes a disabled flag + * from an absent one; where it does not, the provider has nothing to act on, and withholding + * this needs no {@link KnownDeviation}. + * + *

Do not assume a remote-evaluation protocol is in that position. The + * obvious reading — the caller's default never leaves the process, so the server has nothing to + * echo back — is wrong for at least one protocol: OFREP's {@code codeDefaultFlag} is a success + * carrying a {@code reason} and no {@code value}, which tells the provider to use the code + * default. {@code OfrepTest} in {@code providers/ofrep} has the protocol citation and the probed + * response. Check what the response actually carries before concluding a provider cannot hold + * this tag. + * + *

The value is asserted here and not the reason, because the value rests on a {@code MUST} + * and the reason on a {@code SHOULD} that permits any string. Reason {@code DISABLED} is pinned + * in {@code gherkin/reason.feature} instead, on a scenario carrying this tag and + * {@code @standard-reasons} together. No variant is asserted either, so this capability and + * {@link #VARIANTS} deliberately do not compose. + */ + DISABLED_FLAGS("@disabled-flags"), + + /** Provider reports an error state rather than hanging when initialised against a dead backend. */ + UNAVAILABLE_INIT("@unavailable"), + + /** + * Provider coerces between the integer and float types only when the coercion is lossless. + * + *

Lossless coercion is permitted; lossy coercion must fail with {@code TYPE_MISMATCH}. The + * rule is borrowed rather than normative — it is flagd's + * numeric + * coercion ADR, this capability is named after it, and no OpenFeature requirement says what + * a provider owes a value that does not fit the accessor it was asked through + * (open-feature/spec#430). A + * provider that behaves differently is not violating the specification, and a report must not be + * read as saying it is. Appendix F carries the rest, including which of declaring and + * withholding is honest for which provider. + * + *

Java-specific consequence: a provider that keeps the two numeric types strictly apart in + * both directions — as the SDK's own {@code InMemoryProvider} does, so the self-tests in this + * module withhold the tag — cannot attempt the behaviour and needs no {@link KnownDeviation}. + */ + NUMERIC_COERCION("@numeric-coercion"), + + /** + * Provider reports {@code TYPE_MISMATCH} for a boolean or integer flag requested as a string, + * rather than the value's string representation. + * + *

The same gap as {@link #NUMERIC_COERCION}, one type further out, and gated for a stronger + * reason: every value has a string representation, so a backend that stores flag values as + * strings satisfies the string accessor for every flag and has no mismatch to report. + * Its flags are strings, and + * Requirement + * 2.2.3 asks it to populate {@code value} with the resolved flag value, which it did. + * + *

Nothing in the specification contradicts that, because the specification never says what + * the type of a flag value is. {@code TYPE_MISMATCH} appears once, as a row in + * the error code + * table, and no requirement obliges anyone to raise it; the only normative statement about + * value type is + * Requirement + * 1.3.4, a {@code SHOULD} and on the client rather than the provider. So + * a provider that withholds this tag is not violating the specification and + * owes no {@link KnownDeviation} — the same instrument, and the same reasoning, as the numeric + * rule it sits beside. The open question is + * open-feature/spec#433. + * + *

Boolean and integer only. Gates the two rows of the + * {@code gherkin/errors.feature} Scenario Outline — {@code boolean-flag} and + * {@code integer-flag} requested as strings. The float and structured cases carried this tag + * too until specification revision {@code bda599f1} moved them behind + * {@link #FULLY_TYPED_VALUES}; they still carry this one as well, so withholding it skips all + * four and declaring it alone runs only these two. {@link #FULLY_TYPED_VALUES} says why the + * one tag became two. + * + *

These two are the rows a partially typed backend can still answer: a boolean and + * an integer are types such a store records natively, so failing them is the provider's own + * doing rather than the backend's shape. That is what makes this tag worth asking separately — + * a Flagsmith-backed provider records no native float or structure and yet does record these + * two, and Java answers both. + * + *

Java-specific consequence: {@code Client.getStringDetails} is the one accessor every + * backend can satisfy, so this tag is a claim about the backend's typing rather than + * about anything the SDK does. A provider over a typed backend — flagd's resolvers, OFREP — + * declares it; one over a backend that stores values as strings withholds it with the reason + * recorded. + */ + STRING_TYPING("@string-typing"), + + /** + * Backend records a native type for float and structured values too, so the string-typing + * question can be asked of them. + * + *

Strictly narrower than {@link #STRING_TYPING} and always declared alongside it: the two + * scenarios this gates — {@code float-flag} and {@code object-flag} requested as strings — + * carry both tags, so withholding either skips them. Declaring this one without + * {@code STRING_TYPING} claims something no scenario will check. + * + *

Why the split exists, since a single tag looks simpler. It was a single + * tag, over all four cases, until specification revision {@code bda599f1}. Measurement across + * three languages against one Flagsmith backend showed the problem: {@code float-flag} and + * {@code object-flag} were stringified by every provider, because that store records no native + * float or structure type and no provider over it can report a mismatch — a permitted absence. + * {@code boolean-flag} and {@code integer-flag} were not: the store does record those two, Go + * and Java answered them, and JavaScript returned {@code "true"} and {@code "10"} because of + * its own code. Under one tag that provider withholds, and a real defect is published as a + * permitted absence — the suite goes quiet on a bug. Appendix F states the general rule: a + * capability coarser than the variation providers actually show hides defects inside permitted + * absences. + * + *

So the unit of declaration is the question the backend can be asked, not the + * accessor the SDK offers. Withholding this while declaring {@code STRING_TYPING} is the + * expected combination for a partially typed store, and it needs no {@link KnownDeviation} for + * the reason {@code STRING_TYPING} gives: the behaviour is not required. + * + *

Nothing about this is Java-specific. {@code Client.getStringDetails} asks all four cases + * equally well; what differs is whether the backend has a type to mismatch against. + */ + FULLY_TYPED_VALUES("@fully-typed-values"), + + /** + * Provider resolves integers up to 2^53 − 1 exactly. + * + *

{@linkplain #inexpressible() Inexpressible} in Java, so no Java provider may + * declare it and {@link #requireDeclarable} refuses one that tries. + * {@code Client.getIntegerDetails} takes and returns a 32-bit {@link Integer}, so a Java + * provider has nowhere to put {@code 9007199254740991} however faithfully its backend serves it. + * The scenario exists and passes in languages whose accessor is wide enough, which is what makes + * this an inexpressible capability rather than a {@linkplain #reserved() reserved} one, and its + * skip reason names the SDK rather than the provider — see {@link CapabilityGate}. + * + *

Refused centrally, per Appendix F, so nothing is left for an adoption to withhold and no + * {@link KnownDeviation} is owed. No backend fixture would change the answer; only a wider SDK + * accessor would. The 32-bit precision scenario — {@code large-integer-flag}, 2^31 − 1 — is + * untagged and always runs. + */ + LARGE_INTEGERS( + "@large-integers", + "Client.getIntegerDetails takes and returns a 32-bit Integer, so 9007199254740991 cannot be " + + "asked for by any Java provider, however faithfully its backend serves it"), + + /** + * Provider resolves a flag differently for a matching evaluation context. + * + *

Gates the three {@code targeting-key-flag} scenarios, and it is the only tag under which + * dropping the evaluation context is caught by a resolved value rather than needing an echo + * endpoint. The flag's rule, and the fact that it is specified by behaviour rather than by + * syntax, are in the + * canonical + * flag set's README; that context beyond the targeting key is still unverified is an open + * question in Appendix F. + */ + TARGETING("@targeting"), + + /** + * Provider reports the standard resolution reasons, with the meanings Appendix F gives them. + * + *

Gates {@code gherkin/reason.feature} in its entirety, and it is a claim rather than + * an exemption: declaring it says "I use the standard vocabulary with the standard + * meanings", and that file is what checks the claim. + * Requirement + * 2.2.5 is a {@code SHOULD} that permits any string, so a provider that does not declare + * this loses nothing and owes no {@link KnownDeviation} — its values, variants and error codes + * are asserted everywhere else on {@code MUST} requirements. + * + *

Appendix F's {@code @standard-reasons} section holds the situation-to-reason table that is + * the content of the claim, why {@code STATIC} rather than {@code DEFAULT} for a rule-less flag, + * why {@code ERROR} is asserted even though the SDK may have written it, and which reasons are + * not asserted at all. Two of the scenarios compose with {@link #TARGETING} and + * {@link #DISABLED_FLAGS}, so a provider declaring this one alone runs the rest and skips those + * two with their own reason. + */ + STANDARD_REASONS("@standard-reasons"), + + /** + * Provider caches evaluation results and invalidates them on configuration change. + * + *

{@linkplain #reserved() Reserved}; no scenario carries this tag yet. + */ + CACHING("@caching", true); + + private final String tag; + private final boolean reserved; + private final String inexpressibleBecause; + + Capability(String tag) { + this.tag = tag; + this.reserved = false; + this.inexpressibleBecause = null; + } + + Capability(String tag, boolean reserved) { + this.tag = tag; + this.reserved = reserved; + this.inexpressibleBecause = null; + } + + Capability(String tag, String inexpressibleBecause) { + this.tag = tag; + this.reserved = false; + this.inexpressibleBecause = inexpressibleBecause; + } + + /** + * Returns the Gherkin tag, including the leading {@code @}, that gates this capability. + * + * @return the Gherkin tag for this capability + */ + public String tag() { + return tag; + } + + /** + * Returns whether this capability is reserved, and so must not be declared. + * + *

Reserved means the tag is part of the shared vocabulary but no scenario carries it, so it + * can gate nothing and listing it in a report would invite a reader to believe it was verified. + * + * @return {@code true} if no scenario carries this capability's tag + */ + public boolean reserved() { + return reserved; + } + + /** + * Returns whether this capability is one the Java SDK cannot express, and so must not be + * declared by any provider written against it. + * + *

The opposite case to {@link #reserved()}, and kept apart from it deliberately: a reserved + * capability has no scenarios anywhere and expires when the specification writes them, while an + * inexpressible one has scenarios that pass elsewhere and lasts until this SDK changes. Both are + * refused by {@link #requireDeclarable}, with different messages, and their scenarios are + * skipped with different reasons. + * + * @return {@code true} if no provider written against this SDK can be asked this capability's + * scenarios + */ + public boolean inexpressible() { + return inexpressibleBecause != null; + } + + /** + * Returns why this SDK cannot express this capability, for the messages that have to say so. + * + * @return the reason, or {@code null} if this capability is expressible + */ + String inexpressibleBecause() { + return inexpressibleBecause; + } + + /** + * Looks up the capability gated by a Gherkin tag. + * + * @param tag a Gherkin tag including the leading {@code @} + * @return the matching capability, or empty if the tag does not gate a capability + */ + public static Optional fromTag(String tag) { + return Arrays.stream(values()).filter(c -> c.tag.equals(tag)).findFirst(); + } + + /** + * Returns every capability a provider written against this SDK may declare. + * + *

This, not {@code EnumSet.allOf(Capability.class)}, is what "everything" means for a + * declaration: {@linkplain #reserved() reserved} and {@linkplain #inexpressible() inexpressible} + * capabilities are left out. It is the default, and a set a Java provider may declare unchanged. + * Narrow it only for things this provider cannot do. + * + * @return the declarable capabilities, as a fresh mutable set + */ + public static EnumSet declarable() { + EnumSet declarable = EnumSet.allOf(Capability.class); + declarable.removeIf(capability -> capability.reserved() || capability.inexpressible()); + return declarable; + } + + /** + * Returns every declarable capability except the given ones. + * + *

The counterpart to {@code EnumSet.complementOf}, and the reason it exists: a provider + * saying "everything except the one thing I cannot do" wants everything declarable + * except that thing, whereas {@code complementOf} hands back the reserved and inexpressible tags + * as well. What belongs in {@code excluded} is a fact about this provider. + * + * @param excluded capabilities to withhold; reserved and inexpressible capabilities are absent + * regardless + * @return the declarable capabilities minus {@code excluded}, as a fresh mutable set + */ + public static EnumSet declarableExcept(Capability... excluded) { + EnumSet declared = declarable(); + for (Capability capability : excluded) { + declared.remove(capability); + } + return declared; + } + + /** + * Rejects a declaration that claims something no result could check. + * + *

Fails the run rather than warning and dropping it: the declaration is the one part of a + * conformance report that no result can check, so a claim that cannot possibly be true is worth + * stopping for, and the fix is to call {@link #declarable()} or {@link #declarableExcept}. + * + *

The {@linkplain #reserved() reserved} and {@linkplain #inexpressible() inexpressible} cases + * are reported separately rather than in one message, because they are different facts and + * expire on different events. Both lists are gathered before either is thrown, so a declaration + * that gets both wrong hears about both. + * + *

Nothing else is refused. A capability whose scenario the provider cannot satisfy is one the + * results contradict, which is what a conformance run is for. + * + * @param declared the capabilities a harness declares + * @throws IllegalArgumentException if any of them is reserved or inexpressible + */ + public static void requireDeclarable(Collection declared) { + List reservedTags = new ArrayList<>(); + List inexpressibleTags = new ArrayList<>(); + for (Capability capability : declared) { + if (capability.reserved()) { + reservedTags.add(capability.name() + " (" + capability.tag() + ")"); + } else if (capability.inexpressible()) { + inexpressibleTags.add( + capability.name() + " (" + capability.tag() + "): " + capability.inexpressibleBecause()); + } + } + if (!reservedTags.isEmpty()) { + throw new IllegalArgumentException("capabilities() declares reserved " + reservedTags + + ", which no scenario in the suite carries. A reserved capability cannot be " + + "verified or contradicted by any result, so it must not be declared. Use " + + "Capability.declarable(), or Capability.declarableExcept(...) for " + + "\"everything except\" — EnumSet.allOf and EnumSet.complementOf pick reserved " + + "capabilities up on the way past."); + } + if (!inexpressibleTags.isEmpty()) { + throw new IllegalArgumentException("capabilities() declares " + inexpressibleTags + + ", which the Java SDK cannot express. This is not a reserved capability: the " + + "scenarios exist and are asked in languages whose API is wide enough, so they " + + "say nothing about your provider and everything about the SDK it is written " + + "against. No Java provider can satisfy them until the SDK changes, so none may " + + "claim them — and you do not have to know that: Capability.declarable() " + + "already leaves them out, and their scenarios are skipped with a reason that " + + "names the SDK rather than your provider. Remove them from capabilities()."); + } + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/CapabilityGate.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/CapabilityGate.java new file mode 100644 index 0000000000..a503fbbbd5 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/CapabilityGate.java @@ -0,0 +1,258 @@ +package dev.openfeature.contrib.tools.tck; + +import java.net.URI; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import org.opentest4j.TestAbortedException; + +/** + * Decides what a scenario's tags mean for this run: skip it, fail it, or let it go ahead. + * + *

One implementation, deliberately. This is the rule the whole suite rests on — a scenario + * skipped for an undeclared capability must be reported as skipped and never as passed — so the + * gate that produces the skip and the tests that prove the skip survives into the results have to + * be looking at the same code. Inlined into the step definitions, a self-test could only + * demonstrate that some abort becomes a skip, not that this abort does. + * + *

Aborting rather than failing is what makes the outcome a skip: {@link TestAbortedException} maps + * to {@code SKIPPED} in Cucumber's step results, which is what reaches the results. + * + *

The second rule here is the mirror of the first and fails rather than skips: a tag this + * implementation still calls {@linkplain Capability#reserved() reserved} must never reach a + * scenario. See {@link #requireNoExpiredReservation}. + * + *

The third produces a skip like the first but for a reason that has nothing to do with the + * provider: a capability this SDK {@linkplain Capability#inexpressible() cannot express}. Its skip + * reason is deliberately different from an undeclared capability's, because a report's reader has + * to be able to tell them apart. + * + *

The fourth is the second one's own direction reversed, and fails too: a canonical + * scenario carrying a tag this vocabulary does not know at all. See + * {@link #requireKnownVocabulary}. + */ +public final class CapabilityGate { + + private CapabilityGate() {} + + /** + * Applies both gate rules to a scenario about to run. + * + *

First {@link #requireNoExpiredReservation}, then the declaration check below. The order is + * not interchangeable and is fixed here rather than left to the caller: a reserved capability + * can never be declared, so a reserved tag examined second is always a skip for an undeclared + * capability and the expiry is never reported. Both passes are over the whole tag list for the + * same reason — a scenario tagged {@code @events @caching} against a provider that declares + * neither would otherwise abort on the first tag and never look at the second. + * + *

Tags that gate nothing are ignored, so a scenario with no capability tag is mandatory and + * always runs. + * + *

Two skips, and they do not say the same thing. The ordinary one names the + * provider, which did not declare the capability. The other names the SDK, which + * {@linkplain Capability#inexpressible() cannot express} it — reporting that as "the provider + * does not declare it" would read as a decision the provider took. It is checked before the + * declaration, which makes it the reason every time rather than only when the provider happens + * to have withheld the tag as well. + * + * @param tags the scenario's Gherkin tags, including the leading at-sign + * @param declared the capabilities the provider declares + * @throws IllegalStateException if a tag names a reserved capability + * @throws TestAbortedException if a tag gates an inexpressible or an undeclared capability + */ + public static void requireDeclared(Collection tags, Set declared) { + requireDeclared(null, tags, declared); + } + + /** + * Applies every gate rule to a scenario about to run, knowing where the scenario came from. + * + *

The overload {@link #requireDeclared(Collection, Set)} calls into this one with no source, + * which is every rule except {@link #requireKnownVocabulary} — that one is the only rule whose + * answer depends on whether the scenario is canonical, and it cannot be applied to a scenario + * of unknown origin without failing an adopter's own feature file for using its own tag. + * + * @param source the scenario's feature file, as Cucumber reports it, or {@code null} if unknown + * @param tags the scenario's Gherkin tags, including the leading at-sign + * @param declared the capabilities the provider declares + * @throws IllegalStateException if a tag names a reserved capability, or if a canonical + * scenario carries a tag this vocabulary does not know + * @throws TestAbortedException if a tag gates an inexpressible or an undeclared capability + */ + public static void requireDeclared(URI source, Collection tags, Set declared) { + requireNoExpiredReservation(tags); + requireKnownVocabulary(source, tags); + for (String tag : tags) { + Optional found = Capability.fromTag(tag); + if (!found.isPresent()) { + // Not a capability tag as far as this vocabulary is concerned, so it gates nothing + // here. Skipping it is right for an adopter's own tag under extensions/ and wrong + // for a canonical one, and the two are told apart by requireKnownVocabulary above + // rather than here — by the time this loop runs, an unknown canonical tag has + // already failed the scenario. + continue; + } + Capability capability = found.get(); + if (capability.inexpressible()) { + throw new TestAbortedException("Skipped: the Java SDK cannot express capability " + + capability.name() + " (tag " + tag + ") — " + capability.inexpressibleBecause() + + ". This scenario exists and is asked in languages whose API is wide enough, so " + + "this is not a reservation and not the provider under test declining: no Java " + + "provider can be asked it, and none may declare it."); + } + if (!declared.contains(capability)) { + throw new TestAbortedException("Skipped: provider does not declare capability " + capability.name() + + " (tag " + tag + "). Declared capabilities: " + declared); + } + } + } + + /** + * Fails the run if a canonical scenario carries a tag this vocabulary does not know. + * + *

{@link #requireNoExpiredReservation} run backwards. That one catches a tag this + * implementation knows and says nothing carries; this one catches a tag something carries and + * this implementation does not know. Both end in a scenario whose gating is wrong in a way no + * result reports, and this direction is the easier of the two to leave out — an unknown + * tag gates nothing, so its scenarios stay mandatory for every adopter. A suite that + * has not learned a new capability does not report a new capability; it silently keeps + * demanding the old behaviour, and the symptom is a provider that legitimately withholds the + * capability showing unexplained failures while every other provider stays green. Nothing in + * the results says why. All four reference implementations ignored an unknown tag rather than + * failing before Appendix F made this normative, and this package was one of them: the loop in + * {@link #requireDeclared} did nothing but {@code continue}. + * + *

Canonical scenarios only, and that restriction is not a weakening. The + * extension point exists so an adopter can add feature files under + * {@link ProviderTck#EXTENSIONS} with tags of its own, which this vocabulary is not supposed to + * know — failing those would make the extension point unusable, and {@code DeclarationApiTest} + * pins that a tag gating nothing is tolerated. What distinguishes them is the directory: + * {@link ProviderTck#FEATURES} holds the canonical set and nothing else, which is why + * {@code EXTENSIONS} is deliberately a different name rather than a subdirectory of it. A tag + * in there that resolves to nothing is a capability this implementation has not + * learned. + * + *

Checked at run time as well as in this artifact's own tests, and the run-time half is not + * redundant: {@code CanonicalTagCoverageTest} reads the assets packaged in this build, + * and an adopter can put a {@code gherkin/} directory on a classpath root that shadows the + * packaged one. Appendix F also requires the check to be in force where the scenarios execute, + * which the artifact's own test suite is not. + * + * @param source the scenario's feature file, as Cucumber reports it, or {@code null} if unknown + * @param tags the scenario's Gherkin tags, including the leading at-sign + * @throws IllegalStateException if the scenario is canonical and carries an unknown tag + */ + public static void requireKnownVocabulary(URI source, Collection tags) { + if (!isCanonical(source)) { + return; + } + List unknown = new ArrayList<>(); + for (String tag : tags) { + if (!Capability.fromTag(tag).isPresent()) { + unknown.add(tag); + } + } + if (unknown.isEmpty()) { + return; + } + throw new IllegalStateException("The canonical scenario at " + source + " carries " + unknown + + ", which this implementation's capability vocabulary does not know. An unknown tag " + + "gates nothing, so without this check the scenario would stay mandatory for every " + + "adopter — including one that legitimately cannot support the capability, which " + + "would see unexplained failures while every other provider stayed green, and " + + "nothing in the results would say why. The pinned specification revision has " + + "added a capability this package has not: add it to the Capability enum, beside " + + "the tag it was split from or grouped with, and say in its javadoc what declaring " + + "it claims. If instead this is a feature file of your own, move it under " + + ProviderTck.EXTENSIONS + "/ — " + ProviderTck.FEATURES + "/ is the canonical set " + + "and is checked against the vocabulary."); + } + + /** + * Whether a scenario came from the canonical set rather than from an adopter's extension. + * + *

Decided on the feature file's immediate parent directory being + * {@link ProviderTck#FEATURES}, over the URI Cucumber reports — {@code classpath:gherkin/ + * errors.feature} for the packaged assets, a {@code file:} URI when the features are read from + * a directory. Only the last two segments are looked at, so neither form needs special casing + * and a shadowing copy on another classpath root is still canonical, which is the point. + * + *

A {@code null} source is not canonical. It is what the two-argument + * {@link #requireDeclared(Collection, Set)} passes, and the callers that use it are tests + * asserting the other three rules over a bare tag list; treating an unknown origin as canonical + * would make those assert this rule by accident. + */ + private static boolean isCanonical(URI source) { + if (source == null) { + return false; + } + String path = source.toString().replace('\\', '/'); + int lastSeparator = path.lastIndexOf('/'); + if (lastSeparator < 0) { + return false; + } + String parent = path.substring(0, lastSeparator); + int start = Math.max(parent.lastIndexOf('/'), parent.lastIndexOf(':')) + 1; + return ProviderTck.FEATURES.equals(parent.substring(start)); + } + + /** + * Fails the run if a scenario carries the tag of a capability this suite still calls reserved. + * + *

The expiry check on {@link Capability#reserved()}, and the other half of + * {@link Capability#requireDeclarable}: that one refuses a declaration naming a reserved + * capability, this one a scenario carrying its tag. An + * {@linkplain Capability#inexpressible() inexpressible} capability has no equivalent and could + * not — a scenario carrying its tag is exactly what is expected, since other languages run it. + * + *

When the specification writes the scenarios a reservation was holding the name open for and + * this implementation has not followed, the two halves meet in the worst possible place: the + * scenario is skipped for a capability no adopter is permitted to claim — the + * unclaimable-capability failure + * Appendix + * F describes. Nothing else in the suite would notice: the report is well-formed, the run is + * green, and a capability-gated skip is explicitly not a gap. It has no local symptom at all, + * which is why it is checked rather than watched for — {@link Capability#TARGETING} was reserved + * until the {@code targeting-key-flag} scenarios arrived. + * + *

Refused rather than worked around: treating the tag as declarable here would let a run + * claim a capability against an implementation that does not know the tag exists, and the point + * of the check is that a human re-reads the reserved list against the specification. + * + *

The tags are the parsed ones. They come from + * {@link io.cucumber.java.Scenario#getSourceTagNames()}, which is the same parse the run itself + * is driven by, so this cannot disagree with the run about which tags a scenario carries — + * including tags inherited from the feature and tags on an {@code Examples} block. A check that + * scanned the feature files as text instead would be wrong on the day it was written: + * {@code gherkin/events.feature} names {@code @caching} inside a Gherkin {@code #} comment, + * explaining which scenarios are deliberately not covered yet, and a text scan would fail every + * adoption over a sentence. + * + * @param tags the scenario's Gherkin tags, including the leading at-sign + * @throws IllegalStateException if a tag names a reserved capability + */ + public static void requireNoExpiredReservation(Collection tags) { + List expired = new ArrayList<>(); + for (String tag : tags) { + Optional capability = Capability.fromTag(tag); + if (capability.isPresent() && capability.get().reserved()) { + expired.add(capability.get().name() + " (" + capability.get().tag() + ")"); + } + } + if (expired.isEmpty()) { + return; + } + throw new IllegalStateException("This scenario carries reserved " + expired + + ", so the scenarios that reservation was held open for now exist. A reserved " + + "capability cannot be declared — Capability.requireDeclarable refuses it — so " + + "without this check the scenario would be reported as skipped for a capability no " + + "adopter is permitted to claim, which is the unclaimable-capability failure " + + "Appendix F describes and which nothing else in this suite would notice. If the " + + "tag arrived with the canonical feature files, drop the reserved flag from that " + + "constant in Capability so an adoption can declare it and be held to it. If it " + + "arrived from a feature file of your own under extensions/, pick a tag of your " + + "own: a reserved tag gates nothing and cannot be declared."); + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ContainerizedProviderTckTest.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ContainerizedProviderTckTest.java new file mode 100644 index 0000000000..00f56f7cdb --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ContainerizedProviderTckTest.java @@ -0,0 +1,239 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.FeatureProvider; +import java.io.File; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +/** + * Base JUnit Platform Suite for providers that talk to an external backend. + * + *

Adds everything {@link ProviderTckTest} deliberately leaves out: the Docker Compose lifecycle, + * discovery of dynamically mapped host ports, and construction of an {@link HttpBackendControl} + * against the backend's control API. This is the base class for the overwhelming majority of + * providers. + * + *

The HTTP control API described in {@code openapi/control-api.yaml} is the normative contract + * here; a custom in-JVM {@link BackendControl} manipulating an external backend through a side + * channel bypasses it — see {@link BackendControl}. + * + *

Provider authors implement three methods, optionally a fourth, and override the defaults their + * stack needs. The Compose stack is started once, before the first scenario, and + * stopped after the last one; it is never restarted in between, and backend unavailability is + * always simulated inside the running stack through the control API. Appendix F says why. + * + *

{@code tools/tck/README.md} carries a worked adoption. + * + * @see ProviderTckTest + * @see HttpBackendControl + */ +public abstract class ContainerizedProviderTckTest extends ProviderTckTest { + + private static final Logger log = LoggerFactory.getLogger(ContainerizedProviderTckTest.class); + + private ComposeContainer compose; + private BackendEndpoint endpoint; + private HttpBackendControl control; + + // --------------------------------------------------------------------------------------- + // What a provider author supplies + // --------------------------------------------------------------------------------------- + + /** + * Returns the Docker Compose file describing the backend stack under test. + * + *

The path is resolved relative to the Maven module directory, so + * {@code new File("src/test/resources/tck/docker-compose.yaml")} is the idiomatic form. + * + *

The stack must not pin host ports — Docker assigns them dynamically and the TCK discovers + * them after startup. + * + * @return the Compose file describing the backend stack + */ + public abstract File composeFile(); + + /** + * Returns the container-internal ports on {@link #backendService()} that the provider connects + * to, so Testcontainers can expose and map them. + * + *

The control API port from {@link #controlPort()} is exposed automatically and does not + * need to be listed here. + * + * @return container-internal ports the provider connects to + */ + public abstract List backendPorts(); + + /** + * Creates the provider under test, configured against the running backend. + * + *

Called after the Compose stack is up and the control API has seeded the canonical flag + * set. The endpoint carries the dynamically mapped host ports, which is why this is a factory + * rather than a field: the ports do not exist until the stack has started. + * + *

The TCK owns the provider lifecycle from here. Do not call {@code setProvider} or + * {@code initialize} yourself. + * + * @param endpoint host and mapped ports of the running backend stack + * @return a configured, uninitialised provider + */ + public abstract FeatureProvider createProvider(BackendEndpoint endpoint); + + /** + * {@inheritDoc} + * + *

Delegates to {@link #createProvider(BackendEndpoint)} with the running stack's endpoint. + */ + @Override + public final FeatureProvider createProvider() { + return createProvider(endpoint()); + } + + /** + * {@inheritDoc} + * + *

Abstract here rather than defaulted: a provider with a real backend can always be pointed + * at a closed port, so there is no reason for one not to cover the initialisation-failure + * scenarios. + */ + @Override + public abstract FeatureProvider createUnavailableProvider(); + + /** + * Returns the Compose service name that hosts the control API and the backend the provider + * connects to. + * + * @return the Compose service name, {@code backend} by default + */ + public String backendService() { + return "backend"; + } + + /** + * Returns the container-internal port the control API listens on. + * + * @return the control API port, {@code 8080} by default + */ + public int controlPort() { + return 8080; + } + + /** + * Returns extra services and container-internal ports to expose, for stacks that contain more + * than the backend service. + * + *

Keys are Compose service names, values are container-internal ports. Resolve the mapped + * ports with {@link BackendEndpoint#port(String, int)}. + * + * @return additional services and ports to expose, empty by default + */ + public Map> additionalPorts() { + return Collections.emptyMap(); + } + + /** + * Returns the name of the backend configuration used to seed the canonical flag set. + * + *

Not to be confused with {@link ProviderTckHarness#configuration()}, which names the mode of + * the provider under test. This one is a name the backend understands, passed through + * to {@code POST /start?config=...}. + * + * @return the backend configuration name passed to {@code POST /start}, {@code default} by + * default + */ + public String backendConfiguration() { + return "default"; + } + + /** + * Returns how long to wait for the Compose stack and its control API to become reachable. + * + * @return the stack startup timeout, 60 seconds by default + */ + public Duration startupTimeout() { + return Duration.ofSeconds(60); + } + + // --------------------------------------------------------------------------------------- + // The lifecycle-agnostic contract, implemented in terms of the Compose stack + // --------------------------------------------------------------------------------------- + + /** + * {@inheritDoc} + * + *

Starts the Compose stack, resolves the control API's mapped port and waits for it to + * accept commands. + * + *

The await here is the only timing allowance the suite makes: a readiness check against the + * control API itself, bounded by {@link #startupTimeout()}, rather than a guess at how long a + * backend takes. Nothing sleeps after a control command — see {@link HttpBackendControl}. + */ + @Override + public final void startSuite() { + compose = startCompose(); + endpoint = new BackendEndpoint(compose, backendService()); + control = new HttpBackendControl( + "http://" + endpoint.host() + ":" + endpoint.port(controlPort()), backendConfiguration()); + control.awaitReady(startupTimeout()); + log.info("Control API ready at {}", control.baseUrl()); + } + + /** {@inheritDoc} */ + @Override + public final void stopSuite() { + if (compose != null) { + compose.stop(); + } + compose = null; + endpoint = null; + control = null; + } + + /** {@inheritDoc} */ + @Override + public final BackendControl backendControl() { + return control; + } + + /** + * Returns the host and mapped ports of the running stack. + * + * @return the backend endpoint + * @throws IllegalStateException if the stack has not been started + */ + protected final BackendEndpoint endpoint() { + if (endpoint == null) { + throw new IllegalStateException("The Compose stack has not been started yet."); + } + return endpoint; + } + + private ComposeContainer startCompose() { + File composeFile = composeFile(); + if (!composeFile.isFile()) { + throw new IllegalStateException("Compose file not found: " + composeFile.getAbsolutePath() + + ". ContainerizedProviderTckTest.composeFile() is resolved relative to the module directory."); + } + ComposeContainer stack = new ComposeContainer(composeFile); + + stack.withExposedService(backendService(), controlPort(), Wait.forListeningPort()); + for (Integer port : backendPorts()) { + stack.withExposedService(backendService(), port, Wait.forListeningPort()); + } + for (Map.Entry> service : additionalPorts().entrySet()) { + for (Integer port : service.getValue()) { + stack.withExposedService(service.getKey(), port, Wait.forListeningPort()); + } + } + stack.withStartupTimeout(startupTimeout()); + + log.info("Starting Compose stack {} (started once per suite, never restarted)", composeFile.getAbsolutePath()); + stack.start(); + return stack; + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ControlApi.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ControlApi.java new file mode 100644 index 0000000000..e2340ca065 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ControlApi.java @@ -0,0 +1,58 @@ +package dev.openfeature.contrib.tools.tck; + +import com.fasterxml.jackson.annotation.JsonValue; + +/** + * Which of the two control contracts a conformance run was conducted under. + * + *

Closed on purpose. The report schema's {@code backend.controlApi} is an enum of exactly these + * two values, so a {@code String} here would be wider than the thing it feeds: an implementor could + * answer {@code "HTTP"} and produce a document that fails validation with no local error. There is + * no third case to leave room for either. + * + * @see BackendControl#controlApi() + */ +public enum ControlApi { + + /** + * The normative control API: a real backend driven over the HTTP control endpoints in + * {@code openapi/control-api.yaml}. This is what makes a conformance claim portable. + */ + HTTP("http"), + + /** + * Control of a provider with no backend, exercised inside this JVM. + * + *

The narrow allowance for in-memory, environment-variable and file-based providers, where + * "the backend" is a data structure in the same process. A claim of {@code in-process} for a + * provider that does have a backend should be treated with suspicion. + */ + IN_PROCESS("in-process"); + + private final String wireValue; + + ControlApi(String wireValue) { + this.wireValue = wireValue; + } + + /** + * Returns the form this value takes in a conformance report and in the control API definition. + * + * @return {@code http} or {@code in-process} + */ + @JsonValue + public String wireValue() { + return wireValue; + } + + /** + * Returns the wire form, so that logs and failure messages quote the spelling a reader will see + * in the report rather than the enum constant. + * + * @return {@code http} or {@code in-process} + */ + @Override + public String toString() { + return wireValue; + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/FlagUnderTest.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/FlagUnderTest.java new file mode 100644 index 0000000000..609c348d3d --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/FlagUnderTest.java @@ -0,0 +1,57 @@ +package dev.openfeature.contrib.tools.tck; + +/** + * The flag a scenario is currently exercising: its key, its declared type, and the code default + * passed to the evaluation call. + * + *

The declared type is what makes the integer/float distinction testable. The TCK dispatches to + * {@code getIntegerDetails} or {@code getDoubleDetails} purely on this value, so a provider that + * silently widens an integer to a double is caught rather than accommodated. + */ +public final class FlagUnderTest { + + private final String key; + private final String type; + private final Object defaultValue; + + /** + * Creates a flag under test. + * + * @param key the flag key + * @param type the declared type, one of {@code Boolean}, {@code String}, {@code Integer}, + * {@code Float} or {@code Object} + * @param defaultValue the code default passed to the evaluation call + */ + public FlagUnderTest(String key, String type, Object defaultValue) { + this.key = key; + this.type = type; + this.defaultValue = defaultValue; + } + + /** + * Returns the flag key. + * + * @return the flag key + */ + public String key() { + return key; + } + + /** + * Returns the declared flag type. + * + * @return the declared flag type + */ + public String type() { + return type; + } + + /** + * Returns the code default passed to the evaluation call. + * + * @return the code default + */ + public Object defaultValue() { + return defaultValue; + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/HttpBackendControl.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/HttpBackendControl.java new file mode 100644 index 0000000000..770cbc2f6a --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/HttpBackendControl.java @@ -0,0 +1,253 @@ +package dev.openfeature.contrib.tools.tck; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * {@link BackendControl} backed by the standardised HTTP control API. + * + *

This is the normative implementation for every provider that talks to an external backend. It + * implements the contract in {@code openapi/control-api.yaml}, including the documented fallback + * for the optional {@code /reset} operation. It uses the JDK HTTP client so that adopting the TCK + * does not drag an HTTP library onto a provider's test classpath. + * + *

Every operation here manipulates the backend process or its flag state. None of them + * touch containers — that is the no-container-restart invariant, and it is the reason a provider + * built once at suite start stays valid for every scenario. + * + *

Constructed by {@link ContainerizedProviderTckTest} once the Compose stack is up and the + * control API host port is known. Provider authors do not build one themselves. + */ +public final class HttpBackendControl implements BackendControl { + + private static final Logger log = LoggerFactory.getLogger(HttpBackendControl.class); + + private final HttpClient http; + private final String baseUrl; + private final String backendConfiguration; + + /** + * Tri-state cache of whether the backend implements the optional {@code /reset} operation. + * {@code null} until the first {@link #reset()} call probes it. + */ + private Boolean resetSupported; + + /** + * Whether the backend was last known to be unreachable, so that the next + * {@link #prepareScenario()} knows whether {@code /reset} alone can restore the baseline. + */ + private boolean backendStopped; + + /** + * Creates a control client for a running backend. + * + *

There is no post-command settle, deliberately. A control call returns when + * the backend has acted, because that is what the control API promises, and + * Appendix + * F says why a suite must not add a pause of its own. If a step after a control call is + * racy, the defect is in the backend's control API. The promise is about the backend; + * how long the provider takes to notice is what {@code eventTimeout()} bounds. + * + * @param baseUrl the control API base URL, without a trailing slash + * @param backendConfiguration the backend configuration name defining the canonical baseline + */ + HttpBackendControl(String baseUrl, String backendConfiguration) { + this.baseUrl = baseUrl; + this.backendConfiguration = backendConfiguration; + this.http = + HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(); + } + + /** + * Returns the base URL of the control API, for diagnostics. + * + * @return the control API base URL + */ + public String baseUrl() { + return baseUrl; + } + + @Override + public String description() { + return "HTTP control API at " + baseUrl; + } + + /** + * {@inheritDoc} + * + *

Always {@link ControlApi#HTTP}: this is the normative control API, and a run conducted + * through it is the portable kind of conformance claim. + */ + @Override + public ControlApi controlApi() { + return ControlApi.HTTP; + } + + /** + * {@inheritDoc} + * + *

Prefers {@code POST /reset} when the backend is already running, because restoring the + * baseline without an availability blip means the previous scenario teardown cannot leak a + * spurious lifecycle event into the next scenario. When the previous scenario left the backend + * unreachable, {@code /reset} alone would not bring it back, so this falls through to + * {@code POST /start?config=...}. + */ + @Override + public void prepareScenario() { + if (backendStopped) { + start(); + } else { + reset(); + } + } + + /** + * {@inheritDoc} + * + *

Issues {@code POST /change}. + */ + @Override + public void changeFlag() { + post("/change"); + } + + /** + * {@inheritDoc} + * + *

Issues {@code POST /stop}, which makes the backend unreachable without stopping its + * container. + */ + @Override + public void disconnect() { + post("/stop"); + backendStopped = true; + } + + /** + * {@inheritDoc} + * + *

Issues {@code POST /start?config=...}, which also restores the baseline flag state. + */ + @Override + public void reconnect() { + start(); + } + + /** + * Waits until the control API accepts commands. + * + *

Probes the optional {@code GET /healthz}. A {@code 404} is a conformant answer meaning + * "not implemented", in which case readiness has already been established by the Testcontainers + * listening-port wait strategy and this returns immediately. + * + * @param timeout how long to keep probing + */ + public void awaitReady(Duration timeout) { + long deadline = System.nanoTime() + timeout.toNanos(); + RuntimeException last = null; + while (System.nanoTime() < deadline) { + try { + HttpResponse response = http.send( + HttpRequest.newBuilder(URI.create(baseUrl + "/healthz")) + .GET() + .timeout(Duration.ofSeconds(5)) + .build(), + HttpResponse.BodyHandlers.discarding()); + if (response.statusCode() == 200 || response.statusCode() == 404) { + return; + } + last = new IllegalStateException("control API not ready, HTTP " + response.statusCode()); + } catch (IOException e) { + last = new IllegalStateException("control API not reachable at " + baseUrl, e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted while waiting for the control API", e); + } + sleep(Duration.ofMillis(200)); + } + throw new IllegalStateException("control API at " + baseUrl + " did not become ready within " + timeout, last); + } + + /** + * Starts the backend with the configured backend configuration, seeding flag state to that + * configuration's baseline. + */ + private void start() { + post("/start?config=" + backendConfiguration); + backendStopped = false; + } + + /** + * Restores flag state to the seeded baseline without an availability blip. + * + *

{@code POST /reset} is optional. When the backend answers {@code 404} or {@code 501} the + * result is cached and every subsequent call falls back to {@code POST /start?config=...}, + * which resets state at the cost of a process restart. Both paths are conformant; see + * {@code openapi/control-api.yaml}. + */ + private void reset() { + if (Boolean.FALSE.equals(resetSupported)) { + start(); + return; + } + HttpResponse response = send("/reset"); + if (response.statusCode() == 404 || response.statusCode() == 501) { + if (resetSupported == null) { + log.info( + "Control API at {} does not implement POST /reset (HTTP {}); " + + "falling back to POST /start?config={} for scenario isolation.", + baseUrl, + response.statusCode(), + backendConfiguration); + } + resetSupported = false; + start(); + return; + } + expectSuccess("/reset", response); + resetSupported = true; + } + + private void post(String path) { + expectSuccess(path, send(path)); + } + + private HttpResponse send(String path) { + HttpRequest request = HttpRequest.newBuilder(URI.create(baseUrl + path)) + .POST(HttpRequest.BodyPublishers.noBody()) + .timeout(Duration.ofSeconds(30)) + .build(); + try { + return http.send(request, HttpResponse.BodyHandlers.discarding()); + } catch (IOException e) { + throw new IllegalStateException("control API call POST " + baseUrl + path + " failed", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted during control API call POST " + path, e); + } + } + + private void expectSuccess(String path, HttpResponse response) { + if (response.statusCode() != 200) { + throw new IllegalStateException( + "control API call POST " + baseUrl + path + " returned HTTP " + response.statusCode() + + ", expected 200. See openapi/control-api.yaml for the expected contract."); + } + } + + /** Pauses between two readiness probes. The only sleep left here, and it is a poll interval. */ + private static void sleep(Duration duration) { + try { + Thread.sleep(duration.toMillis()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted while waiting for the control API to become ready", e); + } + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/InProcessBackendControl.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/InProcessBackendControl.java new file mode 100644 index 0000000000..26ca4b9145 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/InProcessBackendControl.java @@ -0,0 +1,196 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.providers.memory.Flag; +import dev.openfeature.sdk.providers.memory.InMemoryProvider; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.util.HashMap; +import java.util.Map; + +/** + * {@link BackendControl} that manipulates the SDK's {@link InMemoryProvider} directly, with no + * backend, no container and no HTTP. + * + *

This exists so that providers with nothing to connect to — in-memory, environment-variable and + * file-based providers — can run the TCK. For those, "the backend" is a data structure in the same + * JVM: seeding flags is building a map, and changing one is + * {@link InMemoryProvider#updateFlag(String, Flag)}, which emits + * {@code PROVIDER_CONFIGURATION_CHANGED} through the provider's own event mechanism rather than + * through a simulated one. + * + *

This is not a shortcut for providers that do have a backend. Those use + * {@link HttpBackendControl} via {@link ContainerizedProviderTckTest}; {@link BackendControl} says + * why, and links the rule. + * + *

Connection control

+ * + *

{@link #disconnect()} and {@link #reconnect()} are not implemented, so they inherit the + * interface defaults and throw. An in-memory provider has no connection to lose, and a no-op would + * report {@code @stale} scenarios as passed. The harness instead leaves {@link Capability#STALE} and + * {@link Capability#UNAVAILABLE_INIT} undeclared, and those scenarios are skipped. + * + *

Ownership of the provider

+ * + *

This class both seeds the flags and creates the provider that serves them, because in-process + * they are the same object: {@link #changeFlag()} has to reach the live provider instance to emit + * an event from it. A harness therefore wires both of its factory methods to one instance: + * + * 

{@code
+ * private final InProcessBackendControl control = new InProcessBackendControl();
+ *
+ * @Override
+ * public BackendControl backendControl() {
+ *     return control;
+ * }
+ *
+ * @Override
+ * public FeatureProvider createProvider() {
+ *     return control.createProvider();
+ * }
+ * }
+ */ +public final class InProcessBackendControl implements BackendControl { + + /** + * The flag {@link #changeFlag()} mutates, as named by the control API and by the canonical flag + * definition. + * + *

The key is the one thing about this flag that is not read out of the definition, because + * {@code POST /change} in {@code openapi/control-api.yaml} names it too: the two have to agree, + * and a key discovered from the file could not be checked against the contract that uses it. Its + * variants are read, and {@link #changingFlag} rebuilds it from them. + */ + private static final String CHANGING_FLAG = "changing-flag"; + + /** + * The canonical flag set, never mutated after construction. + * + *

Decoded from the packaged {@code flags/canonical-flags.json} rather than restated here — + * see {@link CanonicalFlags} for why a transcription is the failure mode this guards against. + * + *

Scenario isolation depends on the map not being mutated: {@link InMemoryProvider} copies the + * map it is given, and {@code updateFlag} writes only to the provider's copy, so every provider + * handed out by {@link #createProvider()} starts from an untouched baseline. + */ + private final Map> baseline = CanonicalFlags.flagSet(); + + /** {@code changing-flag} as the definition ships it, the source of its variants. */ + private final Flag changingBaseline = requireChangingFlag(); + + /** The provider serving the current scenario, or {@code null} between scenarios. */ + private InMemoryProvider current; + + /** Which variant {@code changing-flag} currently resolves to. */ + private String changingVariant = changingBaseline.getDefaultVariant(); + + /** + * Creates the provider for the scenario about to run, seeded with the canonical flag set. + * + *

Each call returns a fresh instance over a fresh copy of the baseline, which is what makes + * {@link #prepareScenario()} nothing more than dropping the previous reference. + * + * @return a configured, uninitialised in-memory provider + */ + @SuppressFBWarnings( + value = "EI_EXPOSE_REP", + justification = "Handing out the live provider is the contract, not a leak: in-process " + + "the flag store and the provider are one object, and changeFlag() must reach " + + "the same instance the TCK registered in order to emit an event from it") + public InMemoryProvider createProvider() { + changingVariant = changingBaseline.getDefaultVariant(); + current = new InMemoryProvider(new HashMap<>(baseline)); + return current; + } + + @Override + public String description() { + return "in-process control of " + InMemoryProvider.class.getSimpleName(); + } + + /** + * {@inheritDoc} + * + *

Always {@link ControlApi#IN_PROCESS}, which is what this class exists for: there is no + * backend to drive over the normative HTTP endpoints, because "the backend" is a map in this + * JVM. + */ + @Override + public ControlApi controlApi() { + return ControlApi.IN_PROCESS; + } + + /** + * {@inheritDoc} + * + *

Drops the reference to the previous scenario's provider. That is the whole reset: the + * baseline map is never mutated, so the {@link #createProvider()} call that follows produces a + * provider already at the baseline. Clearing the reference rather than leaving it dangling + * means a scenario that manipulates flags without creating a provider fails with a clear + * message instead of mutating a provider that has already been shut down. + */ + @Override + public void prepareScenario() { + current = null; + } + + /** + * {@inheritDoc} + * + *

Flips {@code changing-flag} between its two variants through + * {@link InMemoryProvider#updateFlag(String, Flag)}, so the event the suite awaits is the + * provider's own {@code PROVIDER_CONFIGURATION_CHANGED} — carrying {@code changing-flag} in + * {@code flagsChanged} — and not a signal the TCK synthesised. + * + *

Alternating rather than assigning a fixed variant keeps repeated calls within one scenario + * meaningful; the suite asserts that the resolved value differs, not what it became. + */ + @Override + public void changeFlag() { + changingVariant = otherVariant(changingVariant); + requireProvider().updateFlag(CHANGING_FLAG, changingFlag(changingVariant)); + } + + /** + * Returns a variant of {@code changing-flag} other than the given one. + * + *

Read out of the definition rather than named here, so that renaming either variant in the + * spec cannot leave this switching between a name the file no longer defines and one it does. + */ + private String otherVariant(String resolved) { + for (String variant : changingBaseline.getVariants().keySet()) { + if (!variant.equals(resolved)) { + return variant; + } + } + throw new IllegalStateException("The canonical definition of '" + CHANGING_FLAG + "' has only the variant '" + + resolved + "'. changeFlag() has to switch to a different one, so the flag needs at least two."); + } + + /** Rebuilds {@code changing-flag} with a different variant as the one it resolves to. */ + private Flag changingFlag(String defaultVariant) { + return Flag.builder() + .variants(changingBaseline.getVariants()) + .defaultVariant(defaultVariant) + .disabled(changingBaseline.isDisabled()) + .build(); + } + + /** The canonical definition of {@code changing-flag}, which the suite cannot do without. */ + private Flag requireChangingFlag() { + Flag flag = baseline.get(CHANGING_FLAG); + if (flag == null) { + throw new IllegalStateException("The canonical flag definition " + CanonicalFlags.RESOURCE + + " does not define '" + CHANGING_FLAG + "', which is the flag POST /change mutates and the " + + "@configuration-change scenarios evaluate."); + } + return flag; + } + + private InMemoryProvider requireProvider() { + if (current == null) { + throw new IllegalStateException("No in-memory provider exists for this scenario. In-process backend " + + "control manipulates the provider itself, so the scenario must create one — with " + + "'Given a stable provider' — before any step that changes flag state."); + } + return current; + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/KnownDeviation.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/KnownDeviation.java new file mode 100644 index 0000000000..f82e8f0524 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/KnownDeviation.java @@ -0,0 +1,133 @@ +package dev.openfeature.contrib.tools.tck; + +import com.fasterxml.jackson.annotation.JsonInclude; + +/** + * An entry that says: this provider fails to do something it is required to do. + * + *

The requirement has to be a numbered {@code MUST}, or a rule the implementation bound itself to + * elsewhere — flagd's numeric-coercion ADR, say. Where the specification permits the + * choice, withholding the capability is the honest report and a deviation entry would + * assert a defect that does not exist. + * + *

An entry is legitimate in two shapes, and the first is preferred: declare the + * capability and let the scenario fail, or — only where the provider cannot attempt the behaviour at + * all — withhold it and let the scenarios skip. Withdrawing a capability in order to turn a + * failure into a skip is the failure mode this class exists to prevent. + * Appendix + * F states both shapes and the rule behind them. + * + *

An illustration from this module, because it is the one case where the same absence means two + * things. A provider with no streaming transport does not declare {@code @configuration-change} and + * owes nothing further; {@code MultiProviderTckTest} withholds the same tag because + * {@code MultiProvider} extends {@code EventProvider} and never subscribes to its children, so a + * child's {@code PROVIDER_CONFIGURATION_CHANGED} is swallowed — + * open-feature/java-sdk#1882. + * Only the second is something a reader has to be told. + * + *

{@link #summary} is required; {@link #issue} is optional — see {@link #tracked} and + * {@link #untracked}. {@link #capability} may be {@code null}, when the gap is against a mandatory, + * ungated scenario. It may not name a {@linkplain Capability#reserved() reserved} + * or {@linkplain Capability#inexpressible() inexpressible} capability: neither's scenarios were put + * to this provider, so a deviation would assert a fault nobody committed. + * + *

Declared by the provider author through {@link ProviderTckHarness#knownDeviations()}, because + * that is the only place that knows. Whatever reads the declaration — a conformance report, a build + * check, a human — is downstream of it and does not widen it. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public final class KnownDeviation { + + /** + * The capability tag the deviation concerns, or {@code null} when it maps to none. + * + *

Set whether the capability was declared and its scenario failed, or withheld and its + * scenarios skipped. The results say which happened; this says which capability it was about. + */ + public final String capability; + + /** Where the gap is tracked, or {@code null} when it is not tracked anywhere. */ + public final String issue; + + /** What the gap is, in a form someone comparing providers can use. Never {@code null}. */ + public final String summary; + + private KnownDeviation(Capability capability, String issue, String summary) { + requireDeviable(capability); + this.capability = capability == null ? null : capability.tag(); + this.issue = issue; + this.summary = summary; + } + + /** + * Refuses a deviation against a capability whose scenarios were never put to this provider. + * + *

The same rule as {@link Capability#requireDeclarable}, one step along: a deviation asserts + * that the provider fails something it is required to do, so it has to be about a question that + * was actually asked. Recording a deviation against a capability that could not be declared is + * the same claim by another route, and the more dangerous of the two, because a deviation reads + * as an admission of fault. The two cases are refused separately because they are different + * facts. + * + *

Checked when the deviation is constructed rather than when it is read, so an adopter is told + * at the point they wrote it and whether or not anything downstream ever reads the declaration. + * + * @param capability the capability the deviation names, or {@code null} + * @throws IllegalArgumentException if the capability is reserved or inexpressible + */ + private static void requireDeviable(Capability capability) { + if (capability == null) { + return; + } + if (capability.reserved()) { + throw new IllegalArgumentException("knownDeviations() records a deviation against reserved " + + capability.name() + " (" + capability.tag() + "), which no scenario in the suite " + + "carries. There is nothing to deviate from: no scenario was skipped for it and " + + "none failed. Use null for a gap against a mandatory, ungated scenario."); + } + if (capability.inexpressible()) { + throw new IllegalArgumentException("knownDeviations() records a deviation against " + + capability.name() + " (" + capability.tag() + "), which the Java SDK cannot " + + "express: " + capability.inexpressibleBecause() + ". This is not a reserved " + + "capability — the scenarios exist and are asked in languages whose API is wide " + + "enough — but they were never put to your provider, so a deviation here asserts " + + "a defect that could not have been observed and that nobody could fix. The " + + "scenario's skip already says the SDK is the limit."); + } + } + + /** + * Records a deviation that is tracked somewhere. The preferred form. + * + * @param capability the capability the gap is about — declared and failing, or withheld and + * skipped — or {@code null} when the gap belongs to no capability. Must not be + * {@linkplain Capability#reserved() reserved} or + * {@linkplain Capability#inexpressible() inexpressible} + * @param issue a URI where the gap is tracked + * @param summary what the gap is; required + * @return the deviation, ready to declare + * @throws IllegalArgumentException if the capability is reserved or inexpressible + */ + public static KnownDeviation tracked(Capability capability, String issue, String summary) { + return new KnownDeviation(capability, issue, summary); + } + + /** + * Records a deviation that is not tracked anywhere yet. + * + *

Worth declaring even so. Naming the defect is what separates it from a capability the + * provider chose not to offer, and a declaration that merely omits the tag cannot say which of + * the two happened. Prefer {@link #tracked} as soon as there is an issue to point at. + * + * @param capability the capability the gap is about — declared and failing, or withheld and + * skipped — or {@code null} when the gap belongs to no capability. Must not be + * {@linkplain Capability#reserved() reserved} or + * {@linkplain Capability#inexpressible() inexpressible} + * @param summary what the gap is; required + * @return the deviation, ready to declare + * @throws IllegalArgumentException if the capability is reserved or inexpressible + */ + public static KnownDeviation untracked(Capability capability, String summary) { + return new KnownDeviation(capability, null, summary); + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderEventRecord.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderEventRecord.java new file mode 100644 index 0000000000..6b240ed54b --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderEventRecord.java @@ -0,0 +1,47 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.EventDetails; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +/** + * A provider event observed by a scenario's event handler, tagged with the Gherkin word that + * registered the handler ({@code ready}, {@code error}, {@code stale}, {@code change}). + */ +@SuppressFBWarnings( + value = "EI_EXPOSE_REP", + justification = "The SDK's event payload is held and handed on as-is; copying it would " + + "hide exactly the object under assertion") +public final class ProviderEventRecord { + + private final String type; + private final EventDetails details; + + /** + * Records an observed event. + * + * @param type the Gherkin event word the handler was registered under + * @param details the event payload delivered by the SDK + */ + public ProviderEventRecord(String type, EventDetails details) { + this.type = type; + this.details = details; + } + + /** + * Returns the Gherkin event word. + * + * @return the event type word + */ + public String type() { + return type; + } + + /** + * Returns the event payload delivered by the SDK. + * + * @return the event details + */ + public EventDetails details() { + return details; + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTck.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTck.java new file mode 100644 index 0000000000..a09508905e --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTck.java @@ -0,0 +1,106 @@ +package dev.openfeature.contrib.tools.tck; + +/** + * The values {@link ProviderTckTest} configures Cucumber with, as compile-time constants. + * + *

Every one of these is already implied by the suite's annotations. They are named here because + * an annotation value has to be a compile-time constant, so an adopter who adds a + * {@code @ConfigurationParameter} of their own cannot compute one — they would otherwise have to + * restate our package name, our resource directory or our object factory as a string literal, and a + * literal copy of someone else's configuration is a copy that goes stale without anything failing. + * Annotation values permit constant concatenation, so {@code ProviderTck.ALL_GLUE + ",com.vendor.steps"} + * is legal where a method call is not. + * + *

A class of its own rather than constants on the suite, because things that are not suites read + * them and because putting them on {@link ProviderTckTest} would inherit the whole namespace into + * every adopter's suite class. + * + *

Nothing here is a setting. Changing what the suite passes to Cucumber means changing the + * annotations on {@link ProviderTckTest}; these constants follow that, they do not drive it. + */ +public final class ProviderTck { + + /** + * Classpath directory holding the canonical feature files, packaged inside this JAR. + * + *

Reserved for the conformance suite. A feature file an adopter adds here does not extend the + * canonical set, and one that collides with a canonical file name silently replaces it — see + * {@link #EXTENSIONS}. + * + *

Named for the directory the assets have in Appendix F rather than for Cucumber's habit of + * calling them features: a consumer joining results from several languages keys on the path a + * canonical feature has relative to the asset directory, so the directory a runner reports has + * to be this one. Cucumber's {@code classpath:} scheme in front of it is the runner's and is + * compared past, not stripped. + */ + public static final String FEATURES = "gherkin"; + + /** + * Classpath directory an adopter puts their own feature files in. + * + *

Deliberately not a subdirectory of {@link #FEATURES}, and deliberately a different name. + * The same directory name in two classpath roots is scanned additively, but the same directory + * and file name is not: one root wins and the other file is never read, with no warning. + * An adopter who dropped {@code gherkin/errors.feature} beside ours would therefore replace a + * canonical file with their own and watch the suite go green having run theirs — the worst + * outcome available to a conformance suite. Two distinct directories, {@code gherkin/} and + * {@code extensions/}, make that collision impossible to reach by accident. + * + *

{@code extensions/} is also the prefix Appendix F reserves for exactly this, so the path a + * runner reports for an adopter's scenario is one any consumer can tell apart from a canonical + * one without knowing anything about this implementation. + * + *

Shipped in this JAR containing only a README, because + * {@link org.junit.platform.suite.api.SelectClasspathResource} on a resource that exists nowhere + * on the classpath is a discovery error rather than an empty selection. Cucumber ignores files + * that are not {@code .feature}, so the README costs nothing. + */ + public static final String EXTENSIONS = "extensions"; + + /** Package holding the canonical step definitions. */ + public static final String GLUE = "dev.openfeature.contrib.tools.tck.steps"; + + /** + * Package an adopter puts their own step definitions in. + * + *

Outside this artifact's package namespace on purpose: it is the adopter's package, not + * ours, and it lives in their source tree. A glue package that does not exist is tolerated + * silently by Cucumber, so an adopter who writes no extensions pays nothing for it being on the + * glue path. + */ + public static final String EXTENSION_GLUE = "openfeature.tck.extensions"; + + /** + * The glue path the suite runs with: the canonical steps and the extension package. + * + *

Concatenate to add more, as {@code ProviderTck.ALL_GLUE + ",com.vendor.steps"}. Note that + * an adopter adding a package this way has to keep the canonical one, or every canonical step + * becomes undefined. + */ + public static final String ALL_GLUE = GLUE + "," + EXTENSION_GLUE; + + /** + * The Cucumber plugins the suite registers. + * + *

Just Cucumber's own summary. The suite runs the scenarios and reports them through JUnit; + * turning a run into a machine-readable conformance report is a separate concern that registers + * its own plugin here. + */ + public static final String PLUGINS = "summary"; + + /** + * Whether scenarios may run in parallel: never. + * + *

Backend state is global to the suite, so concurrent scenarios corrupt each other. The suite + * pins this where it overrides a consuming module's {@code junit-platform.properties}. + */ + public static final String PARALLEL_EXECUTION_ENABLED = "false"; + + /** Execution mode for the scenarios of one feature, matching {@link #PARALLEL_EXECUTION_ENABLED}. */ + public static final String FEATURE_EXECUTION_MODE = "same_thread"; + + /** Object factory that injects {@link TckState} into every step class. */ + public static final String OBJECT_FACTORY = "io.cucumber.picocontainer.PicoFactory"; + + private ProviderTck() {} +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTckHarness.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTckHarness.java new file mode 100644 index 0000000000..6a2636c0d6 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTckHarness.java @@ -0,0 +1,197 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.FeatureProvider; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +/** + * The lifecycle-agnostic contract a provider author implements to run the OpenFeature Provider TCK. + * + *

Two methods have no default: what provider to test, and what manipulates the backend it reads + * from. Everything else is a convention with a working default. Nothing here mentions containers, + * ports or HTTP — that belongs to {@link ContainerizedProviderTckTest}, which implements this + * interface in terms of a Compose stack. + * + *

Which base class to extend: + * + *

    + *
  • Your provider talks to an external backend — extend {@link ContainerizedProviderTckTest}. + *
  • Your provider has no backend (in-memory, environment variables, a local file) — extend + * {@link ProviderTckTest} directly and supply an in-process {@link BackendControl}. + *
+ * + *

Implementations are discovered through the executing JUnit suite, and through + * {@link java.util.ServiceLoader} as a fallback. Extend one of the two base classes — each is both + * the JUnit suite and the harness — and no registration is needed. + * + *

{@code tools/tck/README.md} carries a worked adoption for each of the two shapes. + * + * @see ProviderTckTest + * @see ContainerizedProviderTckTest + */ +public interface ProviderTckHarness { + + /** + * Creates the provider under test, configured against a backend that is already running and + * seeded with the canonical flag set. + * + *

Called once per scenario. This is a factory rather than a field because a provider cannot + * always be configured before the suite starts — a Compose stack's host ports do not exist + * until it is up — and because each scenario gets its own provider instance. + * + *

The TCK owns the provider lifecycle from here: it registers the provider with the + * OpenFeature API under a scenario-scoped domain, waits for it to become ready, and shuts it + * down afterwards. Do not call {@code setProvider} or {@code initialize} yourself. + * + * @return a configured, uninitialised provider + */ + FeatureProvider createProvider(); + + /** + * Returns the seam through which the TCK manipulates the backend. + * + *

Called after {@link #startSuite()}, so an implementation may build it there and return the + * same instance on every call. It must not be {@code null}. + * + * @return the backend control for this suite + */ + BackendControl backendControl(); + + /** + * Creates a provider pointed at a backend that does not exist. + * + *

Point this at a closed port on localhost, with a short connection deadline: the scenario + * allows a bounded time for {@code PROVIDER_ERROR} to arrive, and a provider with a 30-second + * connect timeout will not make it. Do not point it at the backend under test — that must stay + * up, and simulated outages belong to {@link BackendControl}. + * + *

Defaults to throwing, because a provider with no backend has no way to be unreachable. + * Such a harness leaves {@link Capability#UNAVAILABLE_INIT} undeclared, so the default is never + * reached; reaching it means a capability was declared that the harness cannot back up. + * + * @return a configured provider that cannot reach a backend + */ + default FeatureProvider createUnavailableProvider() { + throw new UnsupportedOperationException(getClass().getName() + " does not implement " + + "createUnavailableProvider(). This is a test-configuration bug rather than a provider " + + "defect: an @unavailable scenario ran, so the harness declared " + + "Capability.UNAVAILABLE_INIT without supplying a provider that cannot reach its " + + "backend. Remove that capability, or implement this method."); + } + + /** + * Declares which optional parts of the provider contract this provider supports. + * + *

Scenarios tagged with a capability that is not in this set are reported as + * skipped. They are never silently passed. + * + *

Defaults to every {@linkplain Capability#declarable() declarable} capability. Narrow it + * rather than widening it: start from the default, run the suite, and remove only what your + * provider genuinely cannot do — {@link Capability#declarableExcept} is the idiomatic way to say + * "everything except". + * + *

Read Appendix F's + * rules + * for declaring before narrowing this. They are what makes two reports comparable, + * and the two that are most often got wrong in opposite directions are that the unit of the + * decision is the scenario rather than the tag, and that whether your provider owes an + * answer at all is the question that comes first. + * + *

Remove only what your provider cannot do. What no Java provider can do is already + * gone — see {@link Capability#LARGE_INTEGERS} — and neither that nor a + * {@linkplain Capability#reserved() reserved} capability may be declared, so do not build the + * set with {@code EnumSet.allOf} or {@code EnumSet.complementOf}: both include them and + * declaring one fails the run. + * + * @return the capabilities this provider supports + */ + default Set capabilities() { + return Capability.declarable(); + } + + /** + * Declares gaps this provider is known to have against parts of the contract the specification + * does not treat as optional. + * + *

Declared so that a consumer can tell a design decision from a defect. The TCK cannot tell + * them apart from the outside: a capability the provider chose not to offer and one it cannot + * honour are the same absence, and only the provider author knows which happened. + * + *

Empty by default, which is silence rather than a claim. See {@link KnownDeviation} for what + * counts as a requirement to deviate from and which of the two legitimate shapes to reach for. + * + * @return the deviations this provider acknowledges, empty by default + */ + default List knownDeviations() { + return Collections.emptyList(); + } + + /** + * Returns the name of the provider configuration this suite exercises. + * + *

The provider's configuration rather than its identity. The identity is what the provider + * says through its own metadata; this is which of its modes was tested, and a provider with two + * materially different modes — flagd's RPC and in-process resolvers, say — produces two runs + * that are not interchangeable and must not be labelled the same. + * + *

Derived from the suite class name by default: {@code MyProviderInProcessTest} becomes + * {@code my-provider-in-process}. Override it when that does not read well, and check it when + * the suite lives in a package that already names the provider — {@code InProcessTest} derives + * {@code in-process}, which does not say whose. + * + * @return a short name for this configuration + */ + default String configuration() { + return ReportNames.configurationOf(getClass()); + } + + /** + * Prepares whatever must exist before the first scenario — a container stack, a temporary + * directory, a local server. + * + *

Called once, before any scenario, and always paired with {@link #stopSuite()}. Defaults to + * doing nothing, which is right for a harness whose backend is a data structure in this JVM. + * + *

{@link #backendControl()} is called immediately afterwards, so this is where to build it + * if it needs something that only exists once the suite has started. + */ + default void startSuite() { + // Nothing to start by default. + } + + /** + * Releases whatever {@link #startSuite()} created. + * + *

Called once, after the last scenario, and also if suite startup fails partway through, so + * it must tolerate being called when startup did not complete. + */ + default void stopSuite() { + // Nothing to stop by default. + } + + /** + * Returns how long to wait for a provider event to arrive. + * + *

The single most important knob for a provider author, because providers observe backend + * changes on wildly different timescales — a streaming provider in milliseconds, one that polls + * every 30 seconds in most of a poll interval. Set it to comfortably exceed your worst-case + * detection latency, or the suite reports timeouts that are really just impatience. A scenario + * can tighten it with the explicit {@code within {int}ms} step, which always wins. + * + * @return the default event await timeout, 12 seconds by default + */ + default Duration eventTimeout() { + return Duration.ofSeconds(12); + } + + /** + * Returns how long to wait for a provider to reach {@code READY} during initialisation. + * + * @return the readiness timeout, 30 seconds by default + */ + default Duration readyTimeout() { + return Duration.ofSeconds(30); + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTckTest.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTckTest.java new file mode 100644 index 0000000000..7773f7e525 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTckTest.java @@ -0,0 +1,75 @@ +package dev.openfeature.contrib.tools.tck; + +import io.cucumber.junit.platform.engine.Constants; +import org.junit.platform.suite.api.ConfigurationParameter; +import org.junit.platform.suite.api.IncludeEngines; +import org.junit.platform.suite.api.SelectClasspathResource; +import org.junit.platform.suite.api.Suite; + +/** + * Base JUnit Platform Suite for the OpenFeature Provider TCK. + * + *

Carries all Cucumber runner configuration so that a provider author writes no test + * infrastructure at all. The canonical feature files are packaged inside this JAR and selected from + * the classpath, so consumers need no git submodule of their own. + * + *

Which base class to extend

+ * + *

This one is lifecycle-agnostic: it starts nothing and knows nothing about how the backend is + * reached. Extend it directly when your provider has no backend — an in-memory, + * environment-variable or file-based provider — and supply an in-process {@link BackendControl} + * such as {@link InProcessBackendControl}. + * + *

When your provider talks to an external backend, extend {@link ContainerizedProviderTckTest} + * instead. In-process control is for backend-less providers only — see {@link BackendControl}. + * + *

Serial execution

+ * + *

Scenarios run serially, and this class enforces that rather than merely asking + * for it: it pins {@code cucumber.execution.parallel.enabled=false} here, where it overrides any + * {@code junit-platform.properties} the consuming module ships. Several providers already enable + * Cucumber parallelism for their own suites, and inheriting that setting silently breaks the TCK — + * backend state is global to the suite, so the failure looks like a flaky provider. + * + *

This class carries no lifecycle code of its own. Provider registration, event awaiting and + * backend manipulation are owned by the step definitions in + * {@code dev.openfeature.contrib.tools.tck.steps}, which reach the harness and its + * {@link BackendControl} through {@link TckRuntime}. + * + *

Adding your own scenarios

+ * + *

A provider with features of its own — flagd's {@code fractional} targeting, a vendor's + * proprietary evaluation mode — puts feature files in {@code src/test/resources/extensions/} and + * step definitions in the package {@code openfeature.tck.extensions}, and writes no annotations. + * Both are selected here, so the extra scenarios run inside this suite: same backend lifecycle, same + * {@link BackendControl}. The alternative, a second suite of one's own, is a second backend + * lifecycle to start and a second set of runner configuration to keep in step with this one. + * + *

The extension directory must not be {@code gherkin/} nor a subdirectory of it — see + * {@link ProviderTck#EXTENSIONS} for the classpath collision that rules out. It is shipped inside + * this JAR holding nothing but a README, because {@link SelectClasspathResource} on a resource that + * exists on no classpath root is a discovery error rather than an empty selection. + * + *

Every value these annotations carry is named in {@link ProviderTck}. An adopter who does write a + * {@code @ConfigurationParameter} of their own composes from those constants — + * {@code ProviderTck.ALL_GLUE + ",com.vendor.steps"} — rather than restating this configuration as a + * string literal that nothing would keep in step. + * + * @see ProviderTckHarness + * @see ContainerizedProviderTckTest + * @see ProviderTck + */ +@Suite +@IncludeEngines("cucumber") +@SelectClasspathResource(ProviderTck.FEATURES) +@SelectClasspathResource(ProviderTck.EXTENSIONS) +@ConfigurationParameter(key = Constants.PLUGIN_PROPERTY_NAME, value = ProviderTck.PLUGINS) +@ConfigurationParameter( + key = Constants.PARALLEL_EXECUTION_ENABLED_PROPERTY_NAME, + value = ProviderTck.PARALLEL_EXECUTION_ENABLED) +@ConfigurationParameter( + key = Constants.EXECUTION_MODE_FEATURE_PROPERTY_NAME, + value = ProviderTck.FEATURE_EXECUTION_MODE) +@ConfigurationParameter(key = Constants.GLUE_PROPERTY_NAME, value = ProviderTck.ALL_GLUE) +@ConfigurationParameter(key = Constants.OBJECT_FACTORY_PROPERTY_NAME, value = ProviderTck.OBJECT_FACTORY) +public abstract class ProviderTckTest implements ProviderTckHarness {} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ReportNames.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ReportNames.java new file mode 100644 index 0000000000..250d97c4bc --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ReportNames.java @@ -0,0 +1,48 @@ +package dev.openfeature.contrib.tools.tck; + +import java.util.Locale; + +/** + * Derives the names a run of the suite is identified by. + * + *

Kept apart from any one consumer so that the default + * {@link ProviderTckHarness#configuration()} and anything that files a run's output under that name + * agree on one derivation. + */ +final class ReportNames { + + /** Suffixes a suite class name carries for JUnit's benefit rather than the report's. */ + private static final String[] SUITE_SUFFIXES = {"TckTest", "TCKTest", "TckSuite", "Test", "IT"}; + + /** Used when a name sanitises away to nothing, which an anonymous class manages. */ + private static final String FALLBACK = "tck"; + + private ReportNames() {} + + /** + * Derives a configuration name from a suite class. + * + *

{@code MyProviderInProcessTest} becomes {@code my-provider-in-process}: the suffix that + * exists only so JUnit picks the class up is dropped, and the rest is hyphenated. A provider + * whose modes do not read well this way overrides {@link ProviderTckHarness#configuration()} and + * says so directly — as one whose suite sits in a package that already names it should, since a + * class called {@code InProcessTest} derives {@code in-process} and a report is read by someone + * who cannot see which package it came from. + * + * @param suite the concrete suite class + * @return a hyphenated, lower-case configuration name + */ + static String configurationOf(Class suite) { + String simple = suite.getSimpleName(); + for (String suffix : SUITE_SUFFIXES) { + if (simple.length() > suffix.length() && simple.endsWith(suffix)) { + simple = simple.substring(0, simple.length() - suffix.length()); + break; + } + } + String hyphenated = simple.replaceAll("([a-z0-9])([A-Z])", "$1-$2") + .replaceAll("([A-Z]+)([A-Z][a-z])", "$1-$2") + .toLowerCase(Locale.ROOT); + return hyphenated.isEmpty() ? FALLBACK : hyphenated; + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/TckRuntime.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/TckRuntime.java new file mode 100644 index 0000000000..613adf7a87 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/TckRuntime.java @@ -0,0 +1,180 @@ +package dev.openfeature.contrib.tools.tck; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.ServiceLoader; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Suite-scoped runtime: discovers the provider's harness, drives its suite lifecycle, and exposes + * its {@link BackendControl} to the step definitions. + * + *

This class knows nothing about containers, ports or transports. Whatever must exist before the + * first scenario is created by {@link ProviderTckHarness#startSuite()} and released by + * {@link ProviderTckHarness#stopSuite()} — a Compose stack for + * {@link ContainerizedProviderTckTest}, nothing at all for a harness whose backend is a data + * structure in this JVM. + * + *

The lifecycle runs once: started before the first scenario, stopped after the + * last one, never cycled in between. Scenario isolation is achieved through + * {@link BackendControl#prepareScenario()} instead. + * + *

State is static because Cucumber's {@code @BeforeAll} / {@code @AfterAll} hooks are static and + * the runtime must outlive individual scenarios. Consequently only one TCK suite may run per JVM + * fork at a time. + */ +public final class TckRuntime { + + private static final Logger log = LoggerFactory.getLogger(TckRuntime.class); + + /** System property selecting a harness by simple class name when several are registered. */ + public static final String HARNESS_SELECTOR_PROPERTY = "openfeature.tck.harness"; + + private static TckRuntime instance; + + private final ProviderTckHarness harness; + private final BackendControl backendControl; + + private TckRuntime(ProviderTckHarness harness, BackendControl backendControl) { + this.harness = harness; + this.backendControl = backendControl; + } + + /** + * Starts the suite lifecycle if it is not already running, and returns the shared runtime. + * + * @return the suite-scoped runtime + */ + public static synchronized TckRuntime startIfNeeded() { + if (instance != null) { + return instance; + } + ProviderTckHarness harness = discoverHarness(); + log.info("Provider TCK harness: {}", harness.getClass().getName()); + + // Checked before the suite lifecycle starts, rather than only where the declaration + // reaches the report: an adopter should not wait for Docker to be told about a + // one-line mistake in capabilities(). + Capability.requireDeclarable(harness.capabilities()); + + harness.startSuite(); + try { + BackendControl control = harness.backendControl(); + if (control == null) { + throw new IllegalStateException(harness.getClass().getName() + + ".backendControl() returned null. Every harness must supply the seam through " + + "which the TCK manipulates the backend — HttpBackendControl for an external " + + "backend, InProcessBackendControl for a provider that has none."); + } + log.info("Backend control: {}", control.description()); + instance = new TckRuntime(harness, control); + } catch (RuntimeException e) { + // startSuite() may have allocated a container stack before this failed. + harness.stopSuite(); + throw e; + } + return instance; + } + + /** + * Runs the harness's suite teardown and releases the shared runtime. + */ + public static synchronized void stop() { + if (instance != null) { + ProviderTckHarness harness = instance.harness; + instance = null; + harness.stopSuite(); + } + } + + /** + * Returns the running runtime. + * + * @return the suite-scoped runtime + * @throws IllegalStateException if the suite has not been started + */ + public static synchronized TckRuntime get() { + if (instance == null) { + throw new IllegalStateException("TCK runtime has not been started"); + } + return instance; + } + + /** + * Returns the provider author's harness. + * + * @return the discovered harness + */ + public ProviderTckHarness harness() { + return harness; + } + + /** + * Returns the seam through which the TCK manipulates the backend. + * + * @return the backend control for this suite + */ + public BackendControl backendControl() { + return backendControl; + } + + /** + * Finds the harness for the suite that is currently executing. + * + *

Primary mechanism: the executing suite class itself, reported by {@link TckSuiteListener}. + * A suite class implements {@link ProviderTckHarness}, so a provider with several transports + * writes one suite class per transport and needs no registration, no system property and no + * build configuration to keep them apart. + * + *

Fallback: {@link java.util.ServiceLoader}, for setups where the launcher does not + * auto-register listeners. That path cannot distinguish between several registered harnesses, + * so it accepts exactly one unless {@link #HARNESS_SELECTOR_PROPERTY} names which to use. + */ + private static ProviderTckHarness discoverHarness() { + Optional> suite = TckSuiteListener.currentSuite(); + if (suite.isPresent()) { + return instantiate(suite.get()); + } + + List found = new ArrayList<>(); + ServiceLoader.load(ProviderTckHarness.class).forEach(found::add); + + if (found.isEmpty()) { + throw new IllegalStateException("No ProviderTckHarness found. Write a test class extending " + + "ContainerizedProviderTckTest (external backend) or ProviderTckTest (no backend); " + + "it is both the JUnit suite and the harness."); + } + if (found.size() == 1) { + return found.get(0); + } + + String selector = System.getProperty(HARNESS_SELECTOR_PROPERTY); + if (selector == null) { + throw new IllegalStateException("Several ProviderTckHarness implementations are registered (" + + found.stream().map(h -> h.getClass().getSimpleName()).collect(Collectors.joining(", ")) + + ") and the executing suite could not be determined, which normally means the JUnit " + + "Platform did not auto-register TckSuiteListener. Select one with -D" + + HARNESS_SELECTOR_PROPERTY + "=."); + } + return found.stream() + .filter(h -> h.getClass().getSimpleName().equals(selector)) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No registered ProviderTckHarness named '" + selector + + "'. Registered: " + + found.stream().map(h -> h.getClass().getSimpleName()).collect(Collectors.joining(", ")))); + } + + private static ProviderTckHarness instantiate(Class suite) { + try { + return suite.getDeclaredConstructor().newInstance(); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException( + suite.getName() + " could not be instantiated. A TCK suite class needs a public no-argument " + + "constructor, because the TCK creates one to read its configuration.", + e); + } + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/TckState.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/TckState.java new file mode 100644 index 0000000000..e6d0637a9d --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/TckState.java @@ -0,0 +1,77 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.Client; +import dev.openfeature.sdk.FeatureProvider; +import dev.openfeature.sdk.FlagEvaluationDetails; +import dev.openfeature.sdk.MutableContext; +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.ConcurrentLinkedQueue; + +/** + * Scenario-scoped mutable state, injected into every step definition class by PicoContainer. + * + *

One instance per scenario. Anything that must survive across scenarios — the Compose stack, + * the control API client, the discovered harness — lives in {@link TckRuntime} instead. + */ +public class TckState { + + /** Client bound to the domain the provider under test is registered under. */ + public Client client; + + /** The provider under test. */ + public FeatureProvider provider; + + /** Scenario-scoped OpenFeature domain, so scenarios cannot see each other's providers. */ + public String domain; + + /** The flag the current scenario is exercising. */ + public FlagUnderTest flag; + + /** Evaluation context accumulated by the context steps. */ + public MutableContext context = new MutableContext(); + + /** Result of the most recent evaluation. */ + public FlagEvaluationDetails evaluation; + + /** + * A previously resolved value, captured so a later evaluation can be asserted to differ. + * + *

Used by the configuration-change scenario. Asserting "the value changed" rather than "the + * value is now X" keeps the scenario portable: the control API only requires that + * {@code POST /change} changes the resolved value of {@code changing-flag}, not which concrete + * value it changes to. + */ + public Object rememberedValue; + + /** + * Any exception thrown out of the most recent call the scenario made on the provider, with + * {@link #thrownBy} naming the call. + * + *

Evaluation, shutdown and re-initialisation all record here rather than propagate. The SDK + * contract is that typed evaluation never throws — errors surface as an error code and the code + * default — and the lifecycle scenarios make the same demand of a repeated {@code shutdown()} + * and of an {@code initialize()} against a reachable backend. One slot, asserted by one step, + * {@code no exception should have been thrown}, so a scenario states the expectation explicitly + * instead of a thrown exception merely showing up as a step failure. + */ + public Exception thrown; + + /** The call {@link #thrown} came out of, for the failure message; {@code null} when none did. */ + public String thrownBy; + + /** + * How long the most recent direct {@code shutdown()} call took, or {@code null} before one was + * made in this scenario. + * + *

Recorded so a scenario can assert that shutdown against a backend that will never answer + * returns promptly instead of blocking on a graceful close. + */ + public Duration shutdownDuration; + + /** Events observed by handlers registered in this scenario. */ + public final ConcurrentLinkedQueue events = new ConcurrentLinkedQueue<>(); + + /** The event most recently matched by an await step. */ + public Optional lastEvent = Optional.empty(); +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/TckSuiteListener.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/TckSuiteListener.java new file mode 100644 index 0000000000..8116deff05 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/TckSuiteListener.java @@ -0,0 +1,92 @@ +package dev.openfeature.contrib.tools.tck; + +import java.lang.reflect.Modifier; +import java.util.Optional; +import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.engine.support.descriptor.ClassSource; +import org.junit.platform.launcher.TestExecutionListener; +import org.junit.platform.launcher.TestIdentifier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tracks which TCK suite is currently executing, so the step definitions can find its harness. + * + *

The problem this solves: Cucumber's {@code @BeforeAll} is static and carries no information + * about which suite triggered it. A provider with more than one transport — flagd has RPC and + * in-process — therefore has no way to tell the glue which of its harnesses to use. Discovering + * harnesses through {@link java.util.ServiceLoader} alone makes that ambiguous the moment a second + * one is registered, and resolving it with a system property would push a separate Surefire + * execution per mode onto every adopter's POM. + * + *

Instead: a concrete suite class is a {@link ProviderTckHarness}, and the JUnit + * Platform tells us which one is running. This listener watches for a container whose source is a + * concrete class implementing the SPI and records it for the duration of that suite's execution. + * Adding a second mode is then a second class and nothing else — no registration file, no system + * property, no build configuration. + * + *

Registered through {@code META-INF/services/org.junit.platform.launcher.TestExecutionListener} + * inside this JAR, so it is picked up automatically by Surefire, Gradle and IDEs. It ignores every + * container that is not a TCK suite, so it is inert in builds that do not use the TCK. + */ +public class TckSuiteListener implements TestExecutionListener { + + private static final Logger log = LoggerFactory.getLogger(TckSuiteListener.class); + + private static volatile Class current; + + @Override + public void executionStarted(TestIdentifier testIdentifier) { + harnessClassOf(testIdentifier).ifPresent(suite -> { + current = suite; + log.debug("TCK suite started: {}", suite.getName()); + }); + } + + @Override + public void executionFinished(TestIdentifier testIdentifier, TestExecutionResult result) { + harnessClassOf(testIdentifier).ifPresent(suite -> { + if (suite.equals(current)) { + current = null; + } + }); + } + + /** + * Returns the suite class currently executing, if it is a TCK suite. + * + * @return the executing suite class, or empty when none is running or the listener was not + * registered + */ + static Optional> currentSuite() { + return Optional.ofNullable(current); + } + + /** + * Returns the TCK suite class a test identifier stands for, if it is one. + * + * @param testIdentifier the identifier to inspect + * @return the concrete suite class, or empty when the identifier is not a TCK suite + */ + static Optional> harnessClassOf(TestIdentifier testIdentifier) { + return testIdentifier + .getSource() + .filter(ClassSource.class::isInstance) + .map(ClassSource.class::cast) + .flatMap(TckSuiteListener::loadClass) + .filter(ProviderTckHarness.class::isAssignableFrom) + .filter(candidate -> !Modifier.isAbstract(candidate.getModifiers())) + .map(candidate -> candidate.asSubclass(ProviderTckHarness.class)); + } + + private static Optional> loadClass(ClassSource source) { + try { + // ClassSource resolves the class lazily and throws when it cannot be loaded — which is + // routine for sources belonging to other engines, so it must not fail the run. + return Optional.of(source.getJavaClass()); + } catch (RuntimeException e) { + log.trace("Ignoring unloadable class source {}", source, e); + return Optional.empty(); + } + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/TckValues.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/TckValues.java new file mode 100644 index 0000000000..79bc477cca --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/TckValues.java @@ -0,0 +1,70 @@ +package dev.openfeature.contrib.tools.tck; + +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.openfeature.sdk.Value; +import java.io.IOException; + +/** + * Converts the string values written in feature files into the typed Java values the SDK expects. + * + *

Gherkin has no type system — every cell in an Examples table is a string. The declared flag + * type in the step is therefore the only thing that distinguishes an integer flag from a float + * flag, and this class is where that distinction is made real. {@code Integer} produces an + * {@link Integer}; {@code Float} produces a {@link Double}. Nothing widens one into the other. + */ +public final class TckValues { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private TckValues() {} + + /** + * Converts a feature-file string to a typed value. + * + * @param value the raw string from the feature file; the literal {@code null} yields + * {@code null} + * @param type the declared type, one of {@code Boolean}, {@code String}, {@code Integer}, + * {@code Float} or {@code Object} + * @return the converted value + */ + public static Object convert(String value, String type) { + if ("null".equals(value)) { + return null; + } + switch (type) { + case "Boolean": + return Boolean.parseBoolean(value); + case "String": + return value; + case "Integer": + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "'" + value + "' is not an Integer the Java SDK can ask for: " + + "Client.getIntegerDetails takes a 32-bit Integer. The canonical scenario " + + "that needs more than 2^31 - 1 carries @large-integers, and cannot reach " + + "here: Capability.LARGE_INTEGERS is inexpressible in Java, so no provider " + + "can declare it and CapabilityGate skips the scenario. Reaching this means " + + "an extension feature file of your own asked for a value outside the " + + "accessor's range, which the Java SDK has no way to request.", + e); + } + case "Float": + return Double.parseDouble(value); + case "Object": + return toValue(value); + default: + throw new IllegalArgumentException("Unknown flag type '" + type + + "'. Supported types are Boolean, String, Integer, Float and Object."); + } + } + + private static Value toValue(String json) { + try { + return Value.objectToValue(MAPPER.readValue(json, Object.class)); + } catch (IOException e) { + throw new IllegalArgumentException("Could not parse '" + json + "' as an Object flag value", e); + } + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/steps/AbstractSteps.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/steps/AbstractSteps.java new file mode 100644 index 0000000000..46434a3ca6 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/steps/AbstractSteps.java @@ -0,0 +1,45 @@ +package dev.openfeature.contrib.tools.tck.steps; + +import dev.openfeature.contrib.tools.tck.BackendControl; +import dev.openfeature.contrib.tools.tck.ProviderTckHarness; +import dev.openfeature.contrib.tools.tck.TckRuntime; +import dev.openfeature.contrib.tools.tck.TckState; + +/** + * Base for the TCK step definition classes. + * + *

Holds the PicoContainer-injected scenario state and gives subclasses the only two collaborators + * a step is allowed to reach: the provider author's harness, and the {@link BackendControl} that + * manipulates the backend. + * + *

Deliberately no accessor for the runtime itself. Steps must not know whether the backend is a + * container reached over HTTP or a map in this JVM — that is exactly what {@link BackendControl} + * exists to hide, and it is what lets the same Gherkin run in both modes. + */ +public abstract class AbstractSteps { + + /** Scenario-scoped state, shared across all step classes in a scenario. */ + protected final TckState state; + + protected AbstractSteps(TckState state) { + this.state = state; + } + + /** + * Returns the provider author's harness. + * + * @return the discovered harness + */ + protected ProviderTckHarness harness() { + return TckRuntime.get().harness(); + } + + /** + * Returns the seam through which the backend is manipulated. + * + * @return the backend control for this suite + */ + protected BackendControl backend() { + return TckRuntime.get().backendControl(); + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/steps/ContextSteps.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/steps/ContextSteps.java new file mode 100644 index 0000000000..8bb74b0571 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/steps/ContextSteps.java @@ -0,0 +1,78 @@ +package dev.openfeature.contrib.tools.tck.steps; + +import dev.openfeature.contrib.tools.tck.TckState; +import dev.openfeature.sdk.MutableStructure; +import io.cucumber.java.en.Given; + +/** + * Steps that build the evaluation context passed to the evaluation call. + * + *

Step vocabulary is inherited verbatim from the flagd test harness. + * + *

The context accumulated here is passed to every evaluation — {@code FlagSteps} hands + * {@code state.context} to the typed accessor it dispatches on — so the {@code @targeting} + * scenarios observe passthrough of the targeting key directly: {@code targeting-key-flag} resolves + * to a different value for a matching context, so a provider that drops the context is caught by + * the resolved value itself. + * + *

What is still not asserted is that the whole context reached the backend intact. A + * provider that forwards the targeting key and silently discards every other attribute passes. + * Closing that needs either an echo operation on the control API — something like + * {@code GET /last-evaluation} returning the request the backend last received — or a canonical flag + * whose rule keys on a custom attribute. That remains a known gap. + */ +public class ContextSteps extends AbstractSteps { + + public ContextSteps(TckState state) { + super(state); + } + + /** + * Adds a typed entry to the evaluation context. + * + * @param key the context key + * @param type one of {@code Boolean}, {@code String}, {@code Integer} or {@code Float} + * @param value the value, as written in the feature file + */ + @Given("a context containing a key {string}, with type {string} and with value {string}") + public void contextContainingKeyWithTypeAndValue(String key, String type, String value) { + switch (type) { + case "Boolean": + state.context.add(key, Boolean.parseBoolean(value)); + break; + case "Integer": + state.context.add(key, Integer.parseInt(value)); + break; + case "Float": + state.context.add(key, Double.parseDouble(value)); + break; + case "String": + state.context.add(key, value); + break; + default: + throw new IllegalArgumentException("Unknown context value type '" + type + "'"); + } + } + + /** + * Sets the targeting key on the evaluation context. + * + * @param targetingKey the targeting key + */ + @Given("a context containing a targeting key with value {string}") + public void contextContainingTargetingKey(String targetingKey) { + state.context.setTargetingKey(targetingKey); + } + + /** + * Adds a nested structure entry to the evaluation context. + * + * @param outerKey the outer key + * @param innerKey the key inside the nested structure + * @param value the string value stored under the inner key + */ + @Given("a context containing a nested property with outer key {string} and inner key {string}, with value {string}") + public void contextContainingNestedProperty(String outerKey, String innerKey, String value) { + state.context.add(outerKey, new MutableStructure().add(innerKey, value)); + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/steps/EventSteps.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/steps/EventSteps.java new file mode 100644 index 0000000000..d7434caa6a --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/steps/EventSteps.java @@ -0,0 +1,119 @@ +package dev.openfeature.contrib.tools.tck.steps; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.awaitility.Awaitility.await; + +import dev.openfeature.contrib.tools.tck.ProviderEventRecord; +import dev.openfeature.contrib.tools.tck.ProviderTckHarness; +import dev.openfeature.contrib.tools.tck.TckState; +import dev.openfeature.sdk.ProviderEvent; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Registration of provider event handlers and awaiting the events they observe. + * + *

Step vocabulary is inherited verbatim from the flagd test harness. The one behavioural change + * is that the default await timeout comes from {@link ProviderTckHarness#eventTimeout()} instead of + * being a hard-coded constant, because how fast a provider notices a backend change differs by + * orders of magnitude between streaming and polling transports. + */ +public class EventSteps extends AbstractSteps { + + private static final Logger log = LoggerFactory.getLogger(EventSteps.class); + + public EventSteps(TckState state) { + super(state); + } + + /** + * Registers a handler for one kind of provider event. + * + * @param eventType one of {@code ready}, {@code error}, {@code stale} or {@code change} + */ + @Given("a {} event handler") + public void registerEventHandler(String eventType) { + state.client.on(mapEventType(eventType), details -> { + log.info("{} event observed", eventType); + state.events.add(new ProviderEventRecord(eventType, details)); + }); + } + + /** + * Awaits an event of the given kind, using the provider's configured timeout. + * + * @param eventType the event kind + */ + @When("a {} event was fired") + public void eventWasFired(String eventType) { + awaitEvent(eventType, harness().eventTimeout().toMillis()); + } + + /** + * Awaits an event of the given kind, using the provider's configured timeout. + * + * @param eventType the event kind + */ + @Then("the {} event handler should have been executed") + public void theEventHandlerShouldHaveBeenExecuted(String eventType) { + awaitEvent(eventType, harness().eventTimeout().toMillis()); + } + + /** + * Awaits an event of the given kind within an explicit deadline. + * + *

Use this where the deadline is part of what the scenario asserts — for instance that a + * provider initialised against a dead backend reports the failure promptly rather than hanging. + * The explicit value always wins over {@link ProviderTckHarness#eventTimeout()}. + * + * @param eventType the event kind + * @param milliseconds the deadline + */ + @Then("the {} event handler should have been executed within {int}ms") + public void theEventHandlerShouldHaveBeenExecutedWithin(String eventType, int milliseconds) { + awaitEvent(eventType, milliseconds); + } + + private void awaitEvent(String eventType, long milliseconds) { + log.info("Awaiting {} event (timeout {}ms)", eventType, milliseconds); + await().alias("provider event " + eventType) + .atMost(milliseconds, MILLISECONDS) + .pollInterval(10, MILLISECONDS) + .until(() -> + state.events.stream().anyMatch(event -> event.type().equals(eventType))); + + // Drain up to and including the first match. Without this, a READY recorded before a + // disconnect would satisfy a later assertion expecting a *new* READY after reconnect, + // and the reconnect scenarios would pass without the provider ever reconnecting. + // Events that arrived after the match are preserved for subsequent steps. + ProviderEventRecord matched = null; + while (!state.events.isEmpty()) { + ProviderEventRecord head = state.events.poll(); + if (head != null && head.type().equals(eventType)) { + matched = head; + break; + } + } + state.lastEvent = Optional.ofNullable(matched); + } + + private static ProviderEvent mapEventType(String eventType) { + switch (eventType) { + case "ready": + return ProviderEvent.PROVIDER_READY; + case "error": + return ProviderEvent.PROVIDER_ERROR; + case "stale": + return ProviderEvent.PROVIDER_STALE; + case "change": + return ProviderEvent.PROVIDER_CONFIGURATION_CHANGED; + default: + throw new IllegalArgumentException( + "Unknown event type '" + eventType + "'. The TCK recognises ready, error, stale and change."); + } + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/steps/FlagSteps.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/steps/FlagSteps.java new file mode 100644 index 0000000000..c6ce88af67 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/steps/FlagSteps.java @@ -0,0 +1,275 @@ +package dev.openfeature.contrib.tools.tck.steps; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.openfeature.contrib.tools.tck.FlagUnderTest; +import dev.openfeature.contrib.tools.tck.ProviderEventRecord; +import dev.openfeature.contrib.tools.tck.TckState; +import dev.openfeature.contrib.tools.tck.TckValues; +import dev.openfeature.sdk.ErrorCode; +import dev.openfeature.sdk.Structure; +import dev.openfeature.sdk.Value; +import io.cucumber.datatable.DataTable; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Flag evaluation steps and assertions on the resulting resolution details. + * + *

Step vocabulary is inherited verbatim from the flagd test harness, which was already + * provider-neutral here. + */ +public class FlagSteps extends AbstractSteps { + + private static final Logger log = LoggerFactory.getLogger(FlagSteps.class); + + public FlagSteps(TckState state) { + super(state); + } + + /** + * Declares the flag the scenario will evaluate. + * + * @param type the declared type: {@code Boolean}, {@code String}, {@code Integer}, + * {@code Float} or {@code Object} + * @param key the flag key + * @param defaultValue the code default, as written in the feature file + */ + @Given("a {}-flag with key {string} and a default value {string}") + public void flagWithKeyAndDefaultValue(String type, String key, String defaultValue) { + state.flag = new FlagUnderTest(key, type, TckValues.convert(defaultValue, type)); + } + + /** + * Evaluates the declared flag through the typed API matching its declared type. + * + *

Dispatch is on the declared type alone, which is what makes the integer/float distinction + * observable: an {@code Integer} flag goes through {@code getIntegerDetails} and a {@code Float} + * flag through {@code getDoubleDetails}, with no widening in between. A provider that returns a + * double for an integer flag fails here rather than being quietly accommodated. + * + *

Exceptions are recorded rather than propagated. The SDK contract is that typed evaluation + * never throws — errors surface as an error code plus the code default — so + * {@code no exception should have been thrown} can assert that explicitly instead of the + * scenario merely erroring out. + */ + @When("the flag was evaluated with details") + public void theFlagWasEvaluatedWithDetails() { + FlagUnderTest flag = state.flag; + try { + switch (flag.type()) { + case "Boolean": + state.evaluation = + state.client.getBooleanDetails(flag.key(), (Boolean) flag.defaultValue(), state.context); + break; + case "String": + state.evaluation = + state.client.getStringDetails(flag.key(), (String) flag.defaultValue(), state.context); + break; + case "Integer": + state.evaluation = + state.client.getIntegerDetails(flag.key(), (Integer) flag.defaultValue(), state.context); + break; + case "Float": + state.evaluation = + state.client.getDoubleDetails(flag.key(), (Double) flag.defaultValue(), state.context); + break; + case "Object": + state.evaluation = + state.client.getObjectDetails(flag.key(), (Value) flag.defaultValue(), state.context); + break; + default: + throw new IllegalArgumentException("Unknown flag type '" + flag.type() + "'"); + } + } catch (RuntimeException e) { + log.warn("Evaluation of '{}' threw, which violates the SDK contract", flag.key(), e); + state.thrown = e; + state.thrownBy = "evaluation of '" + flag.key() + "'"; + } + } + + /** + * Asserts the resolved value, converted according to the flag's declared type. + * + * @param value the expected value, as written in the feature file + */ + @Then("the resolved details value should be \"{}\"") + public void theResolvedDetailsValueShouldBe(String value) { + requireEvaluation(); + if (state.evaluation.getErrorCode() != null) { + log.info( + "Evaluation of '{}' carries error code {}: {}", + state.flag.key(), + state.evaluation.getErrorCode(), + state.evaluation.getErrorMessage()); + } + assertThat(state.evaluation.getValue()).isEqualTo(TckValues.convert(value, state.flag.type())); + } + + /** + * Asserts the resolution reason. + * + * @param reason the expected reason + */ + @Then("the reason should be {string}") + public void theReasonShouldBe(String reason) { + requireEvaluation(); + assertThat(state.evaluation.getReason()).isEqualTo(reason); + } + + /** + * Asserts the resolved variant. + * + * @param variant the expected variant + */ + @Then("the variant should be {string}") + public void theVariantShouldBe(String variant) { + requireEvaluation(); + assertThat(state.evaluation.getVariant()).isEqualTo(variant); + } + + /** + * Asserts the error code, where an empty string means no error. + * + * @param errorCode the expected {@link ErrorCode} name, or an empty string + */ + @Then("the error-code should be {string}") + public void theErrorCodeShouldBe(String errorCode) { + requireEvaluation(); + if (errorCode == null || errorCode.isEmpty()) { + assertThat(state.evaluation.getErrorCode()).isNull(); + } else { + assertThat(state.evaluation.getErrorCode()).isEqualTo(ErrorCode.valueOf(errorCode)); + } + } + + /** + * Captures the current resolved value so a later evaluation can be asserted to differ. + * + *

Added by the TCK, for the configuration-change scenario. The control API only requires + * that {@code POST /change} changes the resolved value of {@code changing-flag}; which concrete + * value it changes to is vendor-defined. Asserting a delta rather than an absolute keeps the + * scenario portable and independent of how many times it has run against the same stack. + */ + @When("the resolved value is remembered") + public void theResolvedValueIsRemembered() { + requireEvaluation(); + state.rememberedValue = state.evaluation.getValue(); + } + + /** + * Asserts that re-evaluation produced a different value than the remembered one. + */ + @Then("the resolved details value should have changed") + public void theResolvedDetailsValueShouldHaveChanged() { + requireEvaluation(); + assertThat(state.evaluation.getValue()) + .withFailMessage( + "Expected the value of '%s' to differ after the configuration change, " + + "but it is still %s. The provider signalled the change but did not apply it.", + state.flag.key(), state.rememberedValue) + .isNotEqualTo(state.rememberedValue); + } + + /** + * Asserts that a resolved structure contains the given entries. + * + *

Table columns are {@code key}, {@code type} and {@code value}, mirroring the shape of the + * flagd harness's metadata table. Asserting individual entries rather than a whole JSON blob + * keeps the step readable and avoids quoting a JSON document inside a Gherkin cell. + * + * @param expected a table of expected entries + */ + @Then("the resolved object value should contain") + public void theResolvedObjectValueShouldContain(DataTable expected) { + requireEvaluation(); + assertThat(state.evaluation.getValue()) + .as("resolved value of '%s' is a structure", state.flag.key()) + .isInstanceOf(Value.class); + Structure structure = ((Value) state.evaluation.getValue()).asStructure(); + assertThat(structure) + .as("resolved value of '%s' is a structure", state.flag.key()) + .isNotNull(); + + for (Map row : expected.asMaps()) { + String key = row.get("key"); + Value actual = structure.getValue(key); + assertThat(actual).as("structure entry '%s'", key).isNotNull(); + + Object expectedValue = TckValues.convert(row.get("value"), row.get("type")); + Object actualValue = actual.asObject(); + + // Numbers nested inside a structure are compared by value rather than by Java type. + // Structures arrive as JSON, and JSON has a single number type — whether 100 comes + // back as an Integer or a Double is an artefact of the provider's JSON library, not + // an observable part of the provider contract. The integer/float distinction that + // *is* part of the contract applies to top-level typed evaluation, and is asserted + // by the dedicated scenarios in evaluation.feature and errors.feature. + if (expectedValue instanceof Number && actualValue instanceof Number) { + assertThat(((Number) actualValue).doubleValue()) + .as("structure entry '%s'", key) + .isEqualTo(((Number) expectedValue).doubleValue()); + } else { + assertThat(actualValue).as("structure entry '%s'", key).isEqualTo(expectedValue); + } + } + } + + /** + * Asserts that no error message accompanies the evaluation. + * + *

Requirement 2.3.2: a provider that reports a value and an error message is sending + * two contradictory signals, and an application reading the message believes the wrong one. + * Every success path asserts this alongside the empty error code. + */ + @Then("the error message should be empty") + public void theErrorMessageShouldBeEmpty() { + requireEvaluation(); + assertThat(state.evaluation.getErrorMessage()) + .as("error message of a successful evaluation of '%s'", state.flag.key()) + .isNullOrEmpty(); + } + + /** + * Asserts that every call the scenario made on the provider returned normally. + * + *

Added by the TCK. The spec requires typed evaluation to absorb every error into the + * returned details, so an error scenario must prove both halves: the right error code, and no + * exception escaping to the caller. The lifecycle scenarios reuse it for a repeated + * {@code shutdown()} and for {@code initialize()} against a reachable backend, which record into + * the same slot as an evaluation does. + */ + @Then("no exception should have been thrown") + public void noExceptionShouldHaveBeenThrown() { + assertThat(state.thrown) + .withFailMessage( + "%s threw %s, but the scenario expected it to return normally: typed evaluation " + + "must never throw (errors belong in the resolution details), a repeated " + + "shutdown must have no further effect, and initialisation against a " + + "reachable backend must succeed.", + state.thrownBy, state.thrown) + .isNull(); + } + + /** + * Asserts the flag under test appears in the payload of the most recently matched event. + */ + @Then("the flag should be part of the event payload") + public void theFlagShouldBePartOfTheEventPayload() { + ProviderEventRecord event = state.lastEvent.orElseThrow( + () -> new AssertionError("No event has been matched yet; await an event before asserting its payload")); + assertThat(event.details().getFlagsChanged()).contains(state.flag.key()); + } + + private void requireEvaluation() { + if (state.evaluation == null) { + throw new AssertionError("No evaluation has been performed. " + + "Did the scenario forget 'When the flag was evaluated with details'?" + + (state.thrown == null ? "" : " " + state.thrownBy + " threw: " + state.thrown)); + } + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/steps/ProviderSteps.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/steps/ProviderSteps.java new file mode 100644 index 0000000000..3456e5b607 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/steps/ProviderSteps.java @@ -0,0 +1,322 @@ +package dev.openfeature.contrib.tools.tck.steps; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +import dev.openfeature.contrib.tools.tck.Capability; +import dev.openfeature.contrib.tools.tck.CapabilityGate; +import dev.openfeature.contrib.tools.tck.ProviderTck; +import dev.openfeature.contrib.tools.tck.ProviderTckHarness; +import dev.openfeature.contrib.tools.tck.TckRuntime; +import dev.openfeature.contrib.tools.tck.TckState; +import dev.openfeature.sdk.FeatureProvider; +import dev.openfeature.sdk.Metadata; +import dev.openfeature.sdk.NoOpProvider; +import dev.openfeature.sdk.OpenFeatureAPI; +import dev.openfeature.sdk.ProviderState; +import io.cucumber.java.After; +import io.cucumber.java.AfterAll; +import io.cucumber.java.Before; +import io.cucumber.java.BeforeAll; +import io.cucumber.java.Scenario; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; +import java.time.Duration; +import java.util.UUID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Lifecycle and backend-control steps: bringing the suite up, gating scenarios on declared + * capabilities, creating and registering the provider under test, shutting it down and initialising + * it again, and simulating backend outages. + * + *

Every step that touches the backend goes through {@link #backend()}. Nothing here knows whether + * that is a container driven over HTTP or an in-memory provider manipulated directly, which is what + * lets one set of feature files cover both. + * + *

Step vocabulary is inherited from the flagd test harness so that existing feature files port + * with a near-zero diff. The only change is dropping the word {@code flagd} from the provider setup + * step: {@code a stable flagd provider} becomes {@code a stable provider}. + */ +public class ProviderSteps extends AbstractSteps { + + private static final Logger log = LoggerFactory.getLogger(ProviderSteps.class); + + public ProviderSteps(TckState state) { + super(state); + } + + /** + * Runs the harness's suite startup once, before the first scenario. + */ + @BeforeAll + public static void beforeAll() { + TckRuntime.startIfNeeded(); + } + + /** + * Runs the harness's suite teardown after the last scenario. + */ + @AfterAll + public static void afterAll() { + TckRuntime.stop(); + } + + /** + * Skips scenarios that exercise a capability the provider did not declare. + * + *

Aborting rather than failing means the scenario is reported as skipped by the + * JUnit Platform. That distinction is the whole point: a provider that does not support + * configuration-change events should see those scenarios visibly excluded, never silently + * green. + * + *

This is also how a backend with no connection to lose stays honest. A harness whose + * {@link dev.openfeature.contrib.tools.tck.BackendControl} cannot simulate an outage + * leaves {@link Capability#STALE} and {@link Capability#UNAVAILABLE_INIT} undeclared, and the + * scenarios needing them are skipped here — before any step can reach an unsupported operation. + * + *

Two tags are failed rather than skipped, and both are failures of this suite + * rather than of the provider. A {@linkplain Capability#reserved() reserved} one cannot be + * declared, so it could only ever produce a skip nobody is able to clear — see + * {@link CapabilityGate#requireNoExpiredReservation}. A tag on a canonical scenario that this + * vocabulary does not know at all gates nothing, so it would leave its scenario mandatory for + * every adopter — see {@link CapabilityGate#requireKnownVocabulary}. + * + *

The scenario's URI is passed along with its tags, and only the second of those two checks + * reads it: an adopter's feature file under {@link ProviderTck#EXTENSIONS} is expected to carry + * tags this vocabulary does not know, and {@link ProviderTck#FEATURES} is expected not to. + * + * @param scenario the scenario about to run + */ + @Before(order = 0) + public void gateOnCapabilities(Scenario scenario) { + CapabilityGate.requireDeclared( + scenario.getUri(), scenario.getSourceTagNames(), harness().capabilities()); + } + + /** + * Restores the backend to the state every scenario starts from. + * + *

Scenario isolation is achieved here and nowhere else — never by restarting containers, and + * never by relying on scenarios happening not to interfere. + */ + @Before(order = 10) + public void prepareBackend() { + backend().prepareScenario(); + } + + /** + * Tears the provider down without disturbing the backend. + * + *

Replaces the domain's provider with a {@link NoOpProvider} through the SDK lifecycle rather + * than calling {@code shutdown()} directly, because only the former makes the SDK detach the + * event provider and shut down its emitter executor. Skipping this leaks an emitter thread per + * scenario and lets events from a finished scenario surface in the next one. + */ + @After + public void tearDown() { + if (state.domain != null) { + OpenFeatureAPI.getInstance().setProvider(state.domain, new NoOpProvider()); + } + } + + /** + * Creates the provider under test and registers it under a scenario-scoped domain. + * + *

Two provider flavours are recognised. A {@code stable} provider is built by the harness + * against the running backend and registered with {@code setProviderAndWait}, so the step does + * not return until the provider is ready. An {@code unavailable} provider points at a dead + * backend and is registered with {@code setProvider}, deliberately without waiting — the + * scenario's whole point is that readiness never arrives. + * + * @param flavour either {@code stable} or {@code unavailable} + */ + @Given("a {} provider") + public void createProvider(String flavour) { + ProviderTckHarness harness = harness(); + FeatureProvider provider; + boolean waitForReady; + + switch (flavour) { + case "stable": + provider = harness.createProvider(); + waitForReady = true; + break; + case "unavailable": + provider = harness.createUnavailableProvider(); + waitForReady = false; + break; + default: + throw new IllegalArgumentException( + "Unknown provider flavour '" + flavour + "'. The TCK recognises 'stable' and 'unavailable'."); + } + + String domain = "tck-" + UUID.randomUUID(); + OpenFeatureAPI api = OpenFeatureAPI.getInstance(); + if (waitForReady) { + api.setProviderAndWait(domain, provider); + } else { + api.setProvider(domain, provider); + } + + state.provider = provider; + state.domain = domain; + state.client = api.getClient(domain); + log.info( + "Registered {} provider {} under domain {}", + flavour, + provider.getMetadata().getName(), + domain); + } + + /** + * Asserts that the provider under test identifies itself by name. + * + *

Requirement 2.1.1. Too small to test, until a conformance report keyed on the provider's + * metadata name turned an empty name into a report nobody can attribute. + */ + @Then("the provider metadata name should not be empty") + public void theProviderMetadataNameShouldNotBeEmpty() { + Metadata metadata = requireProvider().getMetadata(); + assertThat(metadata).as("provider metadata").isNotNull(); + assertThat(metadata.getName()).as("provider metadata name").isNotBlank(); + } + + /** + * Shuts the provider under test down by calling its own {@code shutdown()} directly. + * + *

Directly, and not by replacing it through the SDK. {@code setProvider} would shut the old + * provider down too, but wrapping that in a scenario tests the SDK's bookkeeping as much as the + * provider's, and Appendix B already covers the SDK. The provider stays registered and the SDK is + * not told, which is what lets {@code the provider is initialized again} be observed through the + * same client afterwards. + * + *

Timed, because one scenario asserts that shutdown against a backend that will never answer + * returns at all rather than blocking on a graceful close. Exceptions are recorded rather than + * propagated, exactly as an evaluation's are, so that {@code no exception should have been + * thrown} covers the double-shutdown case explicitly. + */ + @When("the provider is shut down") + public void theProviderIsShutDown() { + FeatureProvider provider = requireProvider(); + long started = System.nanoTime(); + try { + provider.shutdown(); + } catch (RuntimeException e) { + log.warn("shutdown() of provider {} threw", provider.getMetadata().getName(), e); + state.thrown = e; + state.thrownBy = "shutdown()"; + } finally { + state.shutdownDuration = Duration.ofNanos(System.nanoTime() - started); + } + } + + /** + * Initialises the provider under test again by calling its own {@code initialize()} directly. + * + *

The one scenario using this step is gated on + * {@link dev.openfeature.contrib.tools.tck.Capability#REINITIALIZATION}, which says why reuse is + * permitted rather than required. The SDK still holds the provider as {@code READY}, because it + * was never told about the shutdown, so the evaluation that follows this step reaches the + * re-initialised provider through the scenario's client with nothing in between. + * + *

The scenario's evaluation context is passed, which is empty unless a context step added to + * it. Exceptions are recorded rather than propagated, the same way an evaluation's are. + */ + @When("the provider is initialized again") + public void theProviderIsInitializedAgain() { + FeatureProvider provider = requireProvider(); + try { + provider.initialize(state.context); + } catch (Exception e) { + log.warn( + "initialize() of provider {} threw after shutdown", + provider.getMetadata().getName(), + e); + state.thrown = e; + state.thrownBy = "initialize()"; + } + } + + /** + * Asserts that the most recent {@code the provider is shut down} returned within a bound. + * + *

A shutdown that waits for a graceful close of a connection that will never answer hangs the + * host application's own shutdown. The bound in the feature file is generous; what is asserted + * is that shutdown returns at all rather than blocking on the backend. + * + * @param milliseconds the bound + */ + @Then("the shutdown should have completed within {int}ms") + public void theShutdownShouldHaveCompletedWithin(int milliseconds) { + assertThat(state.shutdownDuration) + .as("a shutdown was recorded; did the scenario forget 'When the provider is shut down'?") + .isNotNull(); + assertThat(state.shutdownDuration.toMillis()) + .withFailMessage( + "shutdown() took %dms, over the %dms bound. A shutdown must not block on a backend " + + "that will never answer; release what initialisation acquired and return.", + state.shutdownDuration.toMillis(), milliseconds) + .isLessThanOrEqualTo(milliseconds); + } + + /** + * Makes the backend unreachable for the rest of the scenario. + */ + @When("the connection is lost") + public void theConnectionIsLost() { + backend().disconnect(); + } + + /** + * Brings the backend back after {@code the connection is lost}. + * + *

Added by the TCK. The flagd harness only has a self-healing + * {@code the connection is lost for {int}s} form, which cannot express "assert the provider is + * stale, and only then reconnect" — the reconnect races the assertion. Splitting the outage + * into an explicit start and end makes the stale-then-ready transition deterministic, and it is + * why no shipped scenario reaches {@code POST /restart} and why this suite binds no step to it. + */ + @When("the connection is restored") + public void theConnectionIsRestored() { + backend().reconnect(); + } + + /** + * Mutates flag configuration so a conforming provider observes a configuration change. + */ + @When("the flag was modified") + public void theFlagWasModified() { + backend().changeFlag(); + } + + /** + * Asserts the provider settles into the expected lifecycle state. + * + *

Awaits rather than asserting immediately. State transitions are asynchronous in every + * provider, and how quickly one notices a backend change varies by orders of magnitude between + * streaming and polling transports, so the timeout comes from + * {@link ProviderTckHarness#readyTimeout()}. + * + * @param expected the expected {@link ProviderState}, case-insensitive + */ + @Then("the client should be in {} state") + public void theClientShouldBeInState(String expected) { + ProviderState target = ProviderState.valueOf(expected.toUpperCase()); + await().alias("provider state " + target) + .atMost(harness().readyTimeout().toMillis(), MILLISECONDS) + .pollInterval(10, MILLISECONDS) + .until(() -> state.client.getProviderState() == target); + } + + private FeatureProvider requireProvider() { + if (state.provider == null) { + throw new AssertionError("No provider has been created. " + + "Did the scenario forget 'Given a stable provider' or 'Given a unavailable provider'?"); + } + return state.provider; + } +} diff --git a/tools/tck/src/main/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener b/tools/tck/src/main/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener new file mode 100644 index 0000000000..1fdb1d3e74 --- /dev/null +++ b/tools/tck/src/main/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener @@ -0,0 +1 @@ +dev.openfeature.contrib.tools.tck.TckSuiteListener diff --git a/tools/tck/src/main/resources/extensions/README.md b/tools/tck/src/main/resources/extensions/README.md new file mode 100644 index 0000000000..60f3c7a890 --- /dev/null +++ b/tools/tck/src/main/resources/extensions/README.md @@ -0,0 +1,46 @@ +# Provider TCK extension point + +Feature files placed on the classpath under `extensions/` run inside the TCK suite, alongside +the canonical conformance scenarios. + +This file is here so that the directory exists on the classpath even when nobody has extended +anything. `ProviderTckTest` selects `extensions` unconditionally, and a classpath resource +selector naming a resource that exists on no classpath root is a discovery error rather than an +empty selection. Cucumber ignores files that are not `.feature`, so the README itself is never read +as a scenario. + +## Adding scenarios + +Write nothing but the two files: + +``` +src/test/resources/extensions/fractional.feature +src/test/java/openfeature/tck/extensions/FractionalSteps.java // package openfeature.tck.extensions +``` + +No annotations, no second suite, no runner configuration. The scenarios are discovered by the same +suite as the canonical set, so they share its backend lifecycle: one `@BeforeAll`, one backend, the +same `BackendControl`. + +Your step classes may take `dev.openfeature.contrib.tools.tck.TckState` as a constructor +argument to reach the client and the last evaluation, exactly as the canonical steps do, and +`TckRuntime.get()` for the backend control and the backend endpoint. + +## Why this directory rather than `gherkin/` + +Two classpath roots containing the same directory are scanned additively. Two containing the same +directory *and* the same file name are not: one silently wins. A feature file added to `gherkin/` +under a canonical name would therefore replace a canonical file, and the suite would report success +having run the replacement. `gherkin/` and `extensions/` being two distinct directories means that +collision cannot be reached by accident. + +`gherkin/` is the canonical set and belongs to the specification. Extensions are yours. Both names +are Appendix F's: a canonical feature is identified by its path relative to the spec's asset +directory, and `extensions/` is the prefix reserved for yours. A consumer reading the results of a +run tells the two apart by that prefix, in any language. + +## What extensions are not + +An extension scenario is not conformance. It does not appear in the canonical set, it cannot make +the canonical set smaller, and a conformance claim is not a claim about it. If a scenario is +portable across providers it belongs in `gherkin/` — send it to the TCK. diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/CanonicalAssetDigestTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/CanonicalAssetDigestTest.java new file mode 100644 index 0000000000..0d20a4bd90 --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/CanonicalAssetDigestTest.java @@ -0,0 +1,210 @@ +package dev.openfeature.contrib.tools.tck; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.CodeSource; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * The canonical assets in this build are the ones the pinned specification revision defines. + * + *

They are copied out of the {@code spec} submodule at {@code generate-resources}, and the + * submodule's gitlink and its working tree are moved by different commands: a + * rebase, a branch switch or a checkout moves the first, and only {@code git submodule update} + * moves the second. Build in between and the copy step packages the previous pin's assets under + * the new pin's name. Nothing about the result looks wrong — the old feature files agree with each + * other and with the old flag set, so the suite runs, passes, and reports numbers that describe a + * revision nobody asked for. That is not hypothetical; one language's suite did exactly this for a + * whole adoption run and the only trace was that its totals matched the previous pass exactly. + * + *

{@link CanonicalTagCoverageTest} catches one symptom of it — a declarable capability whose + * scenarios have not arrived — and catches it with a better message than this test could give. It + * only fires for that symptom, though. A pin that changes the wording of a scenario, the value of a + * canonical flag, or a field of the control API moves nothing it looks at, and those are the + * changes most pins actually make: the re-pin this test was written for changed two + * {@code $comment} blocks in {@code canonical-flags.json} and not one scenario. + * + *

So this asserts the assets themselves, by digest. It is deliberately blunt: it cannot say + * what differs, only that what was packaged is not what the pin names. The build step it + * backs up is in {@code pom.xml}, and the division between them is the point — the build moves the + * checkout and empties the copy's target directory, and this fails the build if the assets are + * nevertheless wrong. Every route to stale assets ends here, including the one where the checkout + * was skipped by hand. + * + *

Updating the pin

+ * + *

Three things move together, in one commit: + * + *

    + *
  1. the {@code spec} submodule gitlink — {@code git -C tools/tck/spec checkout }; + *
  2. {@link #PINNED_REVISION} below; + *
  3. {@link #PINNED_DIGEST} below, which this test prints when it fails. + *
+ * + *

On the report branch a fourth follows: {@code tck.spec.revision} in the POM, which is what a + * conformance report names as the source of its scenarios. That one is checked against + * {@link #PINNED_REVISION} where it is read back, so it cannot be forgotten. + * + *

What the digest is over

+ * + *

All three asset trees, not just the Gherkin. The feature files are meaningless without the + * flag set they evaluate and the control API that produces their outages, and it is the flag set + * that a Gherkin-only digest would have missed here. + * + *

Line endings are normalised out of it. The assets are checked out through git, so a Windows + * clone with {@code core.autocrlf=true} holds bytes a Linux one does not, and a digest that + * disagreed with itself across platforms would be turned off within a week. Nothing else is + * normalised: trailing whitespace, ordering and encoding are all part of what is pinned. + */ +class CanonicalAssetDigestTest { + + /** + * The open-feature/spec commit the packaged assets come from. + * + *

Must equal {@code git -C tools/tck/spec rev-parse HEAD}, and is the value the report + * branch publishes as the source of a run's scenarios. + */ + static final String PINNED_REVISION = "ff68adb4c7617ad2d980988241e92603bc247926"; + + /** SHA-256 of the three asset trees at {@link #PINNED_REVISION}, as {@link #digest} computes it. */ + static final String PINNED_DIGEST = "7a753b5f5f60248336d93f565bc12ad844af9f04de40867916b98f283363354b"; + + /** The generated resource directories, in the order they are digested. */ + private static final List ASSET_DIRECTORIES = Arrays.asList("flags", "gherkin", "openapi"); + + @Test + @DisplayName("the packaged canonical assets are the pinned revision's, byte for byte") + void thePackagedAssetsAreThePinnedRevisions() { + String actual = digest(); + assertThat(actual) + .as( + "The canonical assets packaged in this build are not the ones open-feature/spec %s " + + "defines.%n%n" + + " expected %s%n" + + " actual %s%n%n" + + "The usual cause is a spec submodule whose working tree has not caught up with " + + "its gitlink: a rebase or a checkout moves the gitlink, and only `git submodule " + + "update` moves the checkout. Run, from the repository root:%n%n" + + " git submodule update --init tools/tck/spec%n" + + " git -C tools/tck/spec rev-parse HEAD # must print %s%n%n" + + "then rebuild with `clean`, because the copies under src/main/resources are " + + "generated and gitignored.%n%n" + + "If you meant to move the pin, update PINNED_REVISION and PINNED_DIGEST in this " + + "class in the same commit as the gitlink, taking the digest from the `actual` " + + "line above.", + PINNED_REVISION, PINNED_DIGEST, actual, PINNED_REVISION) + .isEqualTo(PINNED_DIGEST); + } + + /** + * Digests the packaged assets. + * + *

Every regular file under {@code flags/}, {@code gherkin/} and {@code openapi/}, ordered by + * its path relative to the code source root with {@code /} separators. Each contributes its path + * and then its content, both terminated by a zero byte so that no rename can be absorbed into a + * neighbouring file's bytes. Carriage returns are dropped from the content; see the class + * comment. + */ + private static String digest() { + MessageDigest sha256; + try { + sha256 = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("This JVM does not provide SHA-256", e); + } + Path root = codeSourceRoot(); + for (Path file : assetFiles(root)) { + sha256.update(relativePath(root, file).getBytes(StandardCharsets.UTF_8)); + sha256.update((byte) 0); + sha256.update(withoutCarriageReturns(read(file))); + sha256.update((byte) 0); + } + StringBuilder hex = new StringBuilder(); + for (byte b : sha256.digest()) { + hex.append(String.format("%02x", b)); + } + return hex.toString(); + } + + private static List assetFiles(Path root) { + List files = new ArrayList<>(); + for (String directory : ASSET_DIRECTORIES) { + Path assets = root.resolve(directory); + if (!Files.isDirectory(assets)) { + throw new IllegalStateException("No " + directory + "/ directory in the tck code source " + root + + ". The canonical assets were not copied from the spec submodule; run the build " + + "through Maven rather than compiling the sources alone."); + } + try (Stream walk = Files.walk(assets)) { + files.addAll(walk.filter(Files::isRegularFile).collect(Collectors.toList())); + } catch (IOException e) { + throw new UncheckedIOException("Could not list the canonical assets under " + assets, e); + } + } + files.sort(Comparator.comparing(file -> relativePath(root, file))); + return files; + } + + private static String relativePath(Path root, Path file) { + return root.relativize(file).toString().replace('\\', '/'); + } + + private static byte[] read(Path file) { + try { + return Files.readAllBytes(file); + } catch (IOException e) { + throw new UncheckedIOException("Could not read the canonical asset " + file, e); + } + } + + private static byte[] withoutCarriageReturns(byte[] content) { + byte[] stripped = new byte[content.length]; + int length = 0; + for (byte b : content) { + if (b != '\r') { + stripped[length++] = b; + } + } + byte[] exact = new byte[length]; + System.arraycopy(stripped, 0, exact, 0, length); + return exact; + } + + /** + * The directory this artifact's classes and resources were loaded from. + * + *

Read through the code source rather than the classloader, as {@link CanonicalTagCoverageTest} + * reads it, and for the same reason: an asset placed at the same classpath path on another root + * shadows the packaged one, and a check that digested the shadowing copy would be comparing the + * replacement against itself. + */ + private static Path codeSourceRoot() { + CodeSource codeSource = ProviderTck.class.getProtectionDomain().getCodeSource(); + if (codeSource == null || codeSource.getLocation() == null) { + throw new IllegalStateException( + "The tck code source is not visible to this JVM, so the packaged canonical assets " + + "cannot be read."); + } + try { + return Paths.get(codeSource.getLocation().toURI()); + } catch (URISyntaxException | IllegalArgumentException e) { + throw new IllegalStateException("The tck code source is not a file: " + codeSource.getLocation(), e); + } + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/CanonicalFlagsTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/CanonicalFlagsTest.java new file mode 100644 index 0000000000..c9ee37f19b --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/CanonicalFlagsTest.java @@ -0,0 +1,233 @@ +package dev.openfeature.contrib.tools.tck; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.openfeature.sdk.ImmutableContext; +import dev.openfeature.sdk.Value; +import dev.openfeature.sdk.providers.memory.InMemoryProvider; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * Pins the in-process flag set to the definition this artifact packages. + * + *

This is the test that makes {@link CanonicalFlags} worth having. The flag set the in-memory + * self-tests run against used to be written out as Java literals, and the failure mode of that was + * quiet: a rename or a retyped value in {@code flags/canonical-flags.json} left the suite verifying + * itself against a second, private baseline, so it reported green having tested the wrong flags. The + * only way to catch that is to read the packaged file independently of the decoder and hold + * the decoded set against it, which is what happens below. + * + *

The comparison is deliberately not a value table written out here. A table is another + * transcription, drifts the same way, and a test that compares two copies of the same mistake passes. + * Every expectation comes out of the file instead: the keys it defines, and for each the state it is + * in and the value of the variant it says the flag resolves to. The state matters as much as the + * value now that the set contains four {@code disabled-*} flags, which resolve to nothing at all — + * a decoder that ignored {@code state} would serve their configured values and look correct against + * a variant table. + * + *

The values are read back through an {@link InMemoryProvider} rather than off the decoded map, + * because the provider matches a variant against the type of the accessor it was asked through. That + * is what makes the types load-bearing and what this asserts: a JSON float is fetched through + * {@code getDoubleEvaluation} and a JSON integer through {@code getIntegerEvaluation}, so a decoder + * that turned {@code 10.0} into the integer {@code 10} — the mistake the file's own comment warns + * about, because it lets the lossless-coercion scenario pass without coercing anything — fails here + * with a type mismatch rather than sailing through a value comparison. + */ +class CanonicalFlagsTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + /** The {@code flags} object of the packaged definition, read without going through the decoder. */ + private static Map packaged; + + @BeforeAll + static void readThePackagedDefinition() throws IOException { + try (InputStream in = CanonicalFlagsTest.class.getClassLoader().getResourceAsStream(CanonicalFlags.RESOURCE)) { + assertThat(in) + .as( + "%s must be packaged on the classpath by the copy-provider-tck-flags execution", + CanonicalFlags.RESOURCE) + .isNotNull(); + + JsonNode flags = MAPPER.readTree(in).path("flags"); + assertThat(flags.isObject()).isTrue(); + + packaged = new LinkedHashMap<>(); + for (Iterator> it = flags.fields(); it.hasNext(); ) { + Map.Entry entry = it.next(); + if (!"$comment".equals(entry.getKey())) { + packaged.put(entry.getKey(), entry.getValue()); + } + } + } + assertThat(packaged) + .as("the packaged definition has to define flags for any of this to mean anything") + .isNotEmpty(); + } + + @Test + @DisplayName("the decoded flag set defines exactly the keys the packaged definition defines") + void theDecodedSetHasExactlyThePackagedKeys() { + assertThat(CanonicalFlags.flagSet().keySet()) + .as("a key in one and not the other is the drift this replaced a transcription to prevent") + .containsExactlyInAnyOrderElementsOf(packaged.keySet()) + .as("$comment is documentation, not a flag") + .doesNotContain("$comment"); + } + + @Test + @DisplayName("every flag resolves as its packaged state and default variant say it should") + void everyFlagResolvesToItsPackagedDefaultVariant() throws Exception { + InMemoryProvider provider = new InProcessBackendControl().createProvider(); + provider.initialize(new ImmutableContext()); + ImmutableContext context = new ImmutableContext(); + + List checked = new ArrayList<>(); + List disabled = new ArrayList<>(); + for (Map.Entry entry : packaged.entrySet()) { + String key = entry.getKey(); + String defaultVariant = entry.getValue().path("defaultVariant").asText(null); + assertThat(defaultVariant) + .as("%s names no defaultVariant, so the definition itself is broken", key) + .isNotNull(); + + JsonNode expected = entry.getValue().path("variants").path(defaultVariant); + assertThat(expected.isMissingNode()) + .as("%s resolves to variant '%s', which the definition does not define", key, defaultVariant) + .isFalse(); + + String state = entry.getValue().path("state").asText(null); + assertThat(state) + .as("%s names no state, so the definition itself is broken", key) + .isIn("ENABLED", "DISABLED"); + + boolean enabled = "ENABLED".equals(state); + assertResolves(provider, context, key, defaultVariant, expected, enabled); + checked.add(key); + if (!enabled) { + disabled.add(key); + } + } + + assertThat(checked) + .as("the loop must actually have run over the packaged flags") + .hasSameSizeAs(packaged.keySet()); + + assertThat(disabled) + .as("the disabled half of the assertion has to have been exercised, or a decoder that " + + "dropped the state would pass here unnoticed") + .isNotEmpty(); + } + + /** + * Resolves one flag through the accessor its packaged type calls for, and checks the value. + * + *

The accessor is chosen from the JSON type rather than from the decoded one, so the decoding + * is being held against the file rather than asked to agree with itself. The default handed to + * the accessor is deliberately never the expected value: {@code boolean-zero-flag} resolves to + * {@code false} and {@code string-zero-flag} to {@code ""}, and a comparison whose fallback + * happened to equal the answer would pass on a flag that had gone missing. + * + *

That property is also what lets one dispatch serve both states. A flag the definition marks + * {@code DISABLED} resolves to nothing at all — the caller's default stands in — so the expected + * answer is precisely the fallback this already had to pick to be distinct, and {@code enabled} + * only chooses which of the two the resolution must equal. The four {@code disabled-*} flags are + * the whole reason the parameter exists: a decoder that dropped {@code state} on the floor would + * make them serve their configured values, and that is what fails here rather than + * later, in a scenario, against a provider that did nothing wrong. + */ + private static void assertResolves( + InMemoryProvider provider, + ImmutableContext context, + String key, + String variant, + JsonNode expected, + boolean enabled) { + + String where = key + " variant '" + variant + "'" + (enabled ? "" : ", disabled so the default stands in"); + switch (expected.getNodeType()) { + case BOOLEAN: + boolean bool = expected.booleanValue(); + assertThat(provider.getBooleanEvaluation(key, !bool, context).getValue()) + .as(where) + .isEqualTo(enabled ? bool : !bool); + break; + case STRING: + String string = expected.textValue(); + String stringFallback = string + "-fallback"; + assertThat(provider.getStringEvaluation(key, stringFallback, context) + .getValue()) + .as(where) + .isEqualTo(enabled ? string : stringFallback); + break; + case NUMBER: + assertNumberResolves(provider, context, key, where, expected, enabled); + break; + case OBJECT: + case ARRAY: + Value structure = Value.objectToValue(MAPPER.convertValue(expected, Object.class)); + Value objectFallback = new Value("fallback"); + assertThat(provider.getObjectEvaluation(key, objectFallback, context) + .getValue()) + .as(where) + .isEqualTo(enabled ? structure : objectFallback); + break; + default: + fail("%s is a %s, which is not a flag value", where, expected.getNodeType()); + break; + } + } + + /** + * Resolves a numeric flag through the accessor its literal calls for. + * + *

Which accessor that is is the assertion. An integral literal goes through the + * integer accessor and a fractional one through the float accessor, and the SDK's provider + * refuses a variant of the other type, so this is where a decoder that widened or narrowed a + * number fails. 2^53 − 1 has no room in an {@link Integer} and goes through the long accessor, + * which is also why {@code @large-integers} is {@linkplain Capability#inexpressible() + * inexpressible} in Java — a flag definition can hold the value, and the client API cannot ask + * for it. + * + *

The accessor is still chosen by the literal for a disabled flag, so a {@code disabled-*} + * flag whose type was mangled by the decoder is caught the same way: the answer is then neither + * the configured value nor the fallback. + */ + private static void assertNumberResolves( + InMemoryProvider provider, + ImmutableContext context, + String key, + String where, + JsonNode expected, + boolean enabled) { + + if (!expected.isIntegralNumber()) { + double value = expected.doubleValue(); + assertThat(provider.getDoubleEvaluation(key, value + 1, context).getValue()) + .as(where) + .isEqualTo(enabled ? value : value + 1); + } else if (expected.canConvertToInt()) { + int value = expected.intValue(); + assertThat(provider.getIntegerEvaluation(key, value + 1, context).getValue()) + .as(where) + .isEqualTo(enabled ? value : value + 1); + } else { + long value = expected.longValue(); + assertThat(provider.getLongEvaluation(key, value + 1, context).getValue()) + .as(where) + .isEqualTo(enabled ? value : value + 1); + } + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/CanonicalTagCoverageTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/CanonicalTagCoverageTest.java new file mode 100644 index 0000000000..8913d02079 --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/CanonicalTagCoverageTest.java @@ -0,0 +1,242 @@ +package dev.openfeature.contrib.tools.tck; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.cucumber.gherkin.GherkinParser; +import io.cucumber.messages.types.Envelope; +import io.cucumber.messages.types.Examples; +import io.cucumber.messages.types.Feature; +import io.cucumber.messages.types.FeatureChild; +import io.cucumber.messages.types.GherkinDocument; +import io.cucumber.messages.types.Rule; +import io.cucumber.messages.types.RuleChild; +import io.cucumber.messages.types.Scenario; +import io.cucumber.messages.types.Tag; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.CodeSource; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * Every capability this suite does not call reserved is carried by at least one canonical scenario, + * every reserved one is carried by none, and every tag they carry is a capability this suite knows. + * + *

{@link CapabilityGate#requireNoExpiredReservation} already fails a run where a scenario + * carries a reserved tag — a capability no adopter may declare, gating something, so the scenario is + * skipped forever and nothing notices. This is the other half of the same rule: a capability this + * suite says has scenarios that gates nothing. Declarable, that is a claim no result + * can contradict, which tells a report's reader that a capability was examined when nothing examined + * it. That is the same vacuous claim, arrived at from the opposite direction. + * + *

The first test is over every capability that is not reserved, rather than over + * {@link Capability#declarable()}, and the difference matters. An + * {@linkplain Capability#inexpressible() inexpressible} capability is not declarable here, but its + * scenarios are precisely what distinguish it from a reservation: they exist, and other languages + * run them. Checking only the declarable set would stop looking at the one capability whose whole + * justification is that the scenarios are there. + * + *

It has one realistic cause, and it is a build accident rather than a design mistake: the + * canonical assets are copied out of the {@code spec} submodule at {@code generate-resources}, and + * the submodule's working tree and the gitlink are moved by different commands. A rebase or a branch + * switch updates the gitlink; only {@code git submodule update} moves the checkout. Between the two, + * the copy step happily overwrites the new assets with the old ones, and the result is internally + * consistent — the old feature files agree with each other — so counting scenarios does not + * catch it. A capability added in the same commit as the pin that gives it scenarios is then + * declarable, and gates nothing. + * + *

This is a test rather than a runtime check on purpose. The reserved direction has to fail an + * adopter's run, because a reservation expires when the specification writes scenarios for it and + * this package may not have followed. This direction can only be introduced by a build of + * this artifact, so it belongs in this artifact's own tests — and making every adopter parse + * six feature files at suite start to detect a mistake only this repository can make would be a cost + * paid in the wrong place. + * + *

The tags are parsed, not scanned. {@code gherkin/events.feature} names + * {@code @caching} inside a Gherkin {@code #} comment, explaining which stale-provider behaviour is + * deliberately uncovered, so a text scan reports a reserved tag that no scenario carries. The third + * test below asserts exactly that, so the distinction is pinned rather than described. + */ +class CanonicalTagCoverageTest { + + /** Tags carried by the canonical feature files this artifact ships, as Gherkin parses them. */ + private static final Set CARRIED = readCarriedTags(); + + @Test + @DisplayName("every capability that is not reserved is carried by at least one canonical scenario") + void everyUnreservedCapabilityGatesSomething() { + for (Capability capability : Capability.values()) { + if (capability.reserved()) { + continue; + } + assertThat(CARRIED) + .as( + "%s (%s) is not reserved, so this suite says scenarios for it exist — but no " + + "canonical scenario carries its tag. Either the packaged gherkin/ is stale " + + "(check that the spec submodule working tree matches the gitlink: git -C " + + "tools/tck/spec rev-parse HEAD) or the capability was added ahead of its " + + "scenarios, in which case mark it reserved until they arrive.", + capability.name(), capability.tag()) + .contains(capability.tag()); + } + } + + @Test + @DisplayName("no reserved capability's tag is carried by a canonical scenario") + void noReservedCapabilityGatesAnything() { + for (Capability capability : Capability.values()) { + if (!capability.reserved()) { + continue; + } + assertThat(CARRIED) + .as( + "%s (%s) is still marked reserved, but a canonical scenario now carries its tag. " + + "The reservation has expired: drop the reserved flag so an adopter can " + + "declare it and be held to it. CapabilityGate fails such a scenario at " + + "run time; this says it at build time.", + capability.name(), capability.tag()) + .doesNotContain(capability.tag()); + } + } + + @Test + @DisplayName("every tag a canonical scenario carries resolves to a capability") + void everyCarriedTagIsInTheVocabulary() { + // The reverse of the first test, and the direction that is easy to leave out: that one + // catches a capability with no scenarios, this one a scenario tag with no capability. An + // unknown tag gates nothing, so its scenarios stay mandatory for every adopter — a suite + // that has not learned a new capability keeps demanding the old behaviour, and the only + // symptom is a provider that legitimately withholds it failing while the rest stay green. + // + // This fires on the re-pin that adds a tag, which is the moment it is needed: the pin and + // the Capability constant move in the same commit, and nothing else notices if only the + // pin moves. CapabilityGate.requireKnownVocabulary is the same rule at run time, for the + // canonical assets an adopter actually executes rather than the ones packaged here. + for (String tag : CARRIED) { + assertThat(Capability.fromTag(tag)) + .as( + "A canonical scenario carries %s, which Capability.fromTag does not resolve. The " + + "pinned specification revision has a capability this package does not: add " + + "it to the Capability enum and say in its javadoc what declaring it claims. " + + "Until then the tag gates nothing and its scenarios are mandatory for every " + + "adopter, including the ones that cannot support it.", + tag) + .isPresent(); + } + } + + @Test + @DisplayName("a tag named only in a Gherkin comment is prose, not a tag") + void aTagInACommentIsNotCarried() { + // The trap that makes a text scan wrong on day one, asserted rather than described. + // events.feature explains what @caching would cover, inside a comment; a grep-shaped + // implementation of the test above would report CACHING as an expired reservation forever. + assertThat(rawCanonicalText()) + .as("events.feature still names @caching in prose, which is what makes this test worth having") + .contains("@caching"); + assertThat(CARRIED).as("but nothing carries it as a tag").doesNotContain(Capability.CACHING.tag()); + } + + private static Set readCarriedTags() { + Set tags = new LinkedHashSet<>(); + forEachCanonicalFeature((name, content) -> { + GherkinParser parser = GherkinParser.builder() + .includeSource(false) + .includePickles(false) + .includeGherkinDocument(true) + .build(); + try (Stream envelopes = parser.parse(name, content)) { + envelopes.forEach(envelope -> envelope.getGherkinDocument() + .flatMap(GherkinDocument::getFeature) + .ifPresent(feature -> collectFeature(feature, tags))); + } + }); + if (tags.isEmpty()) { + throw new IllegalStateException("No tags found in the packaged canonical feature files. The " + + "artifact is not intact, or gherkin/ was not copied from the spec submodule."); + } + return tags; + } + + private static String rawCanonicalText() { + StringBuilder all = new StringBuilder(); + forEachCanonicalFeature((name, content) -> all.append(new String(content, StandardCharsets.UTF_8))); + return all.toString(); + } + + private static void collectFeature(Feature feature, Set tags) { + addAll(feature.getTags(), tags); + for (FeatureChild child : feature.getChildren()) { + child.getScenario().ifPresent(scenario -> collectScenario(scenario, tags)); + child.getRule().ifPresent(rule -> collectRule(rule, tags)); + } + } + + private static void collectRule(Rule rule, Set tags) { + addAll(rule.getTags(), tags); + for (RuleChild child : rule.getChildren()) { + child.getScenario().ifPresent(scenario -> collectScenario(scenario, tags)); + } + } + + private static void collectScenario(Scenario scenario, Set tags) { + addAll(scenario.getTags(), tags); + for (Examples examples : scenario.getExamples()) { + addAll(examples.getTags(), tags); + } + } + + private static void addAll(List from, Set tags) { + for (Tag tag : from) { + tags.add(tag.getName()); + } + } + + /** + * Reads {@code gherkin/} out of this artifact's own code source, as {@link ProviderTck} was + * loaded from, rather than through the classloader — the same rule the canonical set is read by, + * and for the same reason: a feature file placed in {@code gherkin/} on another classpath root + * shadows the canonical one, and a check that read the shadowed copy would be checking the + * replacement against itself. + */ + private static void forEachCanonicalFeature(FeatureConsumer consumer) { + CodeSource codeSource = ProviderTck.class.getProtectionDomain().getCodeSource(); + if (codeSource == null || codeSource.getLocation() == null) { + throw new IllegalStateException("The tck code source is not visible to this JVM, so the packaged " + + "canonical feature files cannot be read."); + } + Path root; + try { + root = Paths.get(codeSource.getLocation().toURI()); + } catch (URISyntaxException | IllegalArgumentException e) { + throw new IllegalStateException("The tck code source is not a file: " + codeSource.getLocation(), e); + } + + Path features = root.resolve(ProviderTck.FEATURES); + if (!Files.isDirectory(features)) { + throw new IllegalStateException("No " + ProviderTck.FEATURES + "/ directory in the tck code source " + root + + ". The canonical assets were not copied from the spec submodule."); + } + try (DirectoryStream entries = Files.newDirectoryStream(features, "*.feature")) { + for (Path entry : entries) { + consumer.accept(ProviderTck.FEATURES + "/" + entry.getFileName(), Files.readAllBytes(entry)); + } + } catch (IOException e) { + throw new IllegalStateException("Could not read the canonical feature files from " + features, e); + } + } + + @FunctionalInterface + private interface FeatureConsumer { + void accept(String name, byte[] content); + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ControllableBackendControl.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ControllableBackendControl.java new file mode 100644 index 0000000000..d4e869a6c8 --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ControllableBackendControl.java @@ -0,0 +1,160 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.exceptions.GeneralError; +import dev.openfeature.sdk.providers.memory.Flag; +import java.util.HashMap; +import java.util.Map; + +/** + * In-process control for {@link ControllableProvider}, with a backend that can refuse to answer. + * + *

Everything {@link InProcessBackendControl} does, plus the one thing it cannot: hand out a + * provider whose initialisation fails. That is what unlocks the {@code @lifecycle} and + * {@code @unavailable} scenarios without Docker — an unreachable in-JVM store, rather than a closed + * socket, is enough for a provider to settle into {@code ERROR} observably and for a shutdown + * against a dead backend to be timed. + * + *

Used only by {@link ControllableProviderTckTest}. {@link InProcessBackendControl} remains the + * one an adopter with no backend writes against, and this does not widen it. + */ +final class ControllableBackendControl implements BackendControl { + + /** + * The flag {@link #changeFlag()} mutates, named by {@code POST /change} in the control API and + * by the canonical flag definition, which have to agree. + */ + private static final String CHANGING_FLAG = "changing-flag"; + + /** The canonical flag set, never mutated after construction. */ + private final Map> baseline = CanonicalFlags.flagSet(); + + /** {@code changing-flag} as the definition ships it, the source of its variants. */ + private final Flag changingBaseline = requireChangingFlag(); + + /** The provider serving the current scenario, or {@code null} between scenarios. */ + private ControllableProvider current; + + /** Which variant {@code changing-flag} currently resolves to. */ + private String changingVariant = changingBaseline.getDefaultVariant(); + + /** + * Creates the provider for the scenario about to run, over a reachable store. + * + *

The store is read at {@code initialize()} time rather than now, which is the whole reason + * this provider can cover the lifecycle feature: the flag set is something initialisation + * acquires. + * + * @return a configured, uninitialised provider + */ + ControllableProvider createProvider() { + changingVariant = changingBaseline.getDefaultVariant(); + current = new ControllableProvider(this::snapshot); + return current; + } + + /** + * Creates a provider whose backend will not answer, so {@code initialize()} throws. + * + *

Not registered as {@link #current}: the {@code @unavailable} scenarios never change a flag, + * and leaving the field alone means a later {@link #changeFlag()} fails loudly rather than + * mutating a provider that never initialised. + * + * @return a provider that cannot initialise + */ + ControllableProvider createUnavailableProvider() { + return new ControllableProvider(ControllableBackendControl::unreachable); + } + + @Override + public String description() { + return "in-process control of " + ControllableProvider.class.getSimpleName(); + } + + /** + * {@inheritDoc} + * + *

{@link ControlApi#IN_PROCESS}: there is no backend beyond a map in this JVM, which is the + * whole point of this double. Stated rather than defaulted, like every other control. + */ + @Override + public ControlApi controlApi() { + return ControlApi.IN_PROCESS; + } + + /** + * {@inheritDoc} + * + *

Drops the reference to the previous scenario's provider. The baseline map is never mutated, + * so the {@link #createProvider()} call that follows starts from an untouched copy of it. + */ + @Override + public void prepareScenario() { + current = null; + } + + /** + * {@inheritDoc} + * + *

Flips {@code changing-flag} between its two variants, and lets the provider emit the event + * from itself — see {@link ControllableProvider#changeFlag}. + */ + @Override + public void changeFlag() { + changingVariant = otherVariant(changingVariant); + requireProvider().changeFlag(CHANGING_FLAG, changingFlag(changingVariant)); + } + + /** A fresh copy of the baseline, which is what initialisation acquires. */ + private Map> snapshot() { + return new HashMap<>(baseline); + } + + /** The unreachable backend: every attempt to read it fails the way a dead socket would. */ + private static Map> unreachable() { + throw new GeneralError("the TCK's in-process backend is unreachable for this provider"); + } + + /** + * Returns a variant of {@code changing-flag} other than the given one. + * + *

Read out of the definition rather than named here, so that renaming either variant in the + * spec cannot leave this switching to a name the file no longer defines. + */ + private String otherVariant(String resolved) { + for (String variant : changingBaseline.getVariants().keySet()) { + if (!variant.equals(resolved)) { + return variant; + } + } + throw new IllegalStateException("The canonical definition of '" + CHANGING_FLAG + "' has only the variant '" + + resolved + "'. changeFlag() has to switch to a different one, so the flag needs at least two."); + } + + /** Rebuilds {@code changing-flag} with a different variant as the one it resolves to. */ + private Flag changingFlag(String defaultVariant) { + return Flag.builder() + .variants(changingBaseline.getVariants()) + .defaultVariant(defaultVariant) + .disabled(changingBaseline.isDisabled()) + .build(); + } + + /** The canonical definition of {@code changing-flag}, which the suite cannot do without. */ + private Flag requireChangingFlag() { + Flag flag = baseline.get(CHANGING_FLAG); + if (flag == null) { + throw new IllegalStateException("The canonical flag definition does not define '" + CHANGING_FLAG + + "', which is the flag POST /change mutates and the @configuration-change scenarios evaluate."); + } + return flag; + } + + private ControllableProvider requireProvider() { + if (current == null) { + throw new IllegalStateException("No controllable provider exists for this scenario. In-process backend " + + "control manipulates the provider itself, so the scenario must create one — with " + + "'Given a stable provider' — before any step that changes flag state."); + } + return current; + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ControllableProvider.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ControllableProvider.java new file mode 100644 index 0000000000..54f2a0d967 --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ControllableProvider.java @@ -0,0 +1,166 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.EvaluationContext; +import dev.openfeature.sdk.EventProvider; +import dev.openfeature.sdk.Metadata; +import dev.openfeature.sdk.ProviderEvaluation; +import dev.openfeature.sdk.ProviderEventDetails; +import dev.openfeature.sdk.ProviderState; +import dev.openfeature.sdk.Value; +import dev.openfeature.sdk.exceptions.GeneralError; +import dev.openfeature.sdk.providers.memory.Flag; +import dev.openfeature.sdk.providers.memory.InMemoryProvider; +import java.util.Collections; +import java.util.Map; +import java.util.function.Supplier; + +/** + * An in-JVM provider with a real initialisation, for the TCK's own Docker-free + * self-test. + * + *

It exists because {@link InMemoryProvider} cannot cover the lifecycle feature and never will: + * it is handed its whole flag set by its constructor, so {@code initialize()} does nothing but + * record a state, and {@code shutdown()} releases nothing there is any way to observe. Running the + * lifecycle scenarios against it would establish nothing, which is why + * {@link InMemoryProviderTckTest} leaves {@link Capability#LIFECYCLE} undeclared. The consequence + * was that the {@code @lifecycle} and {@code @reinitialization} steps — shutdown, double shutdown, + * shutdown against a dead backend, initialise again — had no coverage without Docker, and a + * regression in them surfaced first inside a containerised provider suite, where it looks like a + * provider defect. + * + *

This provider therefore acquires something. It starts owning nothing, {@code initialize()} + * reaches a {@linkplain Supplier store} that may refuse it, and {@code shutdown()} drops what was + * acquired. The store is in this JVM rather than over a socket, but the shape is the one + * the lifecycle scenarios assert: initialisation can fail, its outcome is observable, shutdown + * releases and can be repeated, and initialising again brings the provider back. + * + *

Composition rather than {@code extends InMemoryProvider}, deliberately. Seeding a subclass's + * flags at {@code initialize()} time means calling {@code updateFlags}, which emits + * {@code PROVIDER_CONFIGURATION_CHANGED} — so initialisation would emit a configuration change + * every time, and a test double that emits events the thing it stands in for would not emit is + * worse than no double. Holding the delegate in a field keeps every emission deliberate. + * + *

Not part of the published API. An adopter with no backend uses + * {@link InProcessBackendControl} and the SDK's own {@link InMemoryProvider}; this is the TCK + * testing itself. + */ +final class ControllableProvider extends EventProvider { + + /** What this provider calls itself. Asserted only as "not empty", by the metadata feature. */ + private static final String NAME = "TckControllableProvider"; + + /** + * The backend initialisation reaches. Returns the flag set to serve, or throws when the backend + * is unreachable — which is how the {@code @unavailable} scenarios get a provider that fails to + * initialise without needing a closed socket. + */ + private final Supplier>> backend; + + /** The flag store serving the current session, or {@code null} before init and after shutdown. */ + private volatile InMemoryProvider delegate; + + ControllableProvider(Supplier>> backend) { + this.backend = backend; + } + + @Override + public Metadata getMetadata() { + return () -> NAME; + } + + /** + * Acquires the flag set from the backend, and fails if the backend will not give it up. + * + *

Called by the SDK on registration, and directly by the {@code the provider is initialized + * again} step. Both paths are the same code, which is the point of the reinitialisation + * scenario: a provider that returns early because an {@code initialized} flag was never cleared + * would pass the first and fail the second. + * + * @param context the scenario's evaluation context + * @throws Exception if the backend is unreachable + */ + @Override + public void initialize(EvaluationContext context) throws Exception { + InMemoryProvider acquired = new InMemoryProvider(backend.get()); + acquired.initialize(context); + delegate = acquired; + } + + /** + * Releases the flag set. + * + *

Idempotent, which is what "shutting down a provider twice has no further effect" asks for: + * the second call finds {@code null} and returns. {@code super.shutdown()} is deliberately not + * called — it terminates {@link EventProvider}'s emitter executor, which would make this + * provider unusable after a shutdown the specification permits it to recover from. The SDK + * terminates that executor when the provider is replaced, which the TCK does after every + * scenario. + */ + @Override + public void shutdown() { + delegate = null; + } + + @Override + public ProviderState getState() { + InMemoryProvider current = delegate; + return current == null ? ProviderState.NOT_READY : current.getState(); + } + + /** + * Changes a flag and says so, from this provider rather than from the delegate. + * + *

The delegate is not registered with the SDK, so an event emitted from it reaches nobody. + * Mutating the delegate's store and emitting from here is what makes the event the suite awaits + * arrive on the client the scenario is holding. + * + * @param key the flag that changed + * @param flag its new definition + */ + void changeFlag(String key, Flag flag) { + requireDelegate().updateFlag(key, flag); + emitProviderConfigurationChanged(ProviderEventDetails.builder() + .flagsChanged(Collections.singletonList(key)) + .message("flag changed by the TCK's in-process control") + .build()); + } + + @Override + public ProviderEvaluation getBooleanEvaluation(String key, Boolean fallback, EvaluationContext ctx) { + return requireDelegate().getBooleanEvaluation(key, fallback, ctx); + } + + @Override + public ProviderEvaluation getStringEvaluation(String key, String fallback, EvaluationContext ctx) { + return requireDelegate().getStringEvaluation(key, fallback, ctx); + } + + @Override + public ProviderEvaluation getIntegerEvaluation(String key, Integer fallback, EvaluationContext ctx) { + return requireDelegate().getIntegerEvaluation(key, fallback, ctx); + } + + @Override + public ProviderEvaluation getDoubleEvaluation(String key, Double fallback, EvaluationContext ctx) { + return requireDelegate().getDoubleEvaluation(key, fallback, ctx); + } + + @Override + public ProviderEvaluation getObjectEvaluation(String key, Value fallback, EvaluationContext ctx) { + return requireDelegate().getObjectEvaluation(key, fallback, ctx); + } + + /** + * The store, or a failure that names the cause. + * + *

An evaluation reaching a shut-down provider is a real error rather than a reason to serve + * stale values: the whole claim of the shutdown scenarios is that shutdown released something. + */ + private InMemoryProvider requireDelegate() { + InMemoryProvider current = delegate; + if (current == null) { + throw new GeneralError(NAME + " has no flag store: initialize() has not run, or shutdown() released it."); + } + return current; + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ControllableProviderTckTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ControllableProviderTckTest.java new file mode 100644 index 0000000000..750a95b83a --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ControllableProviderTckTest.java @@ -0,0 +1,128 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.FeatureProvider; +import java.util.EnumSet; +import java.util.Set; + +/** + * Runs the suite against a provider that has a real initialisation, with no Docker. + * + *

This is the suite that covers the lifecycle feature without a container, and + * it exists because {@link InMemoryProviderTckTest} cannot. The SDK's {@code InMemoryProvider} is + * handed its whole flag set by its constructor, so its {@code initialize()} records a state and its + * {@code shutdown()} releases nothing observable; running the lifecycle scenarios against it would + * establish nothing, which is why that suite leaves {@link Capability#LIFECYCLE} undeclared and + * why those scenarios were skipped there. Everything they assert — shutdown releases what + * initialisation acquired, shutdown can be repeated, shutdown against a dead backend returns + * promptly, and a provider that offers reuse really is reusable — therefore had no coverage at all + * outside a containerised provider suite, where a break in those step definitions looks like a + * provider defect rather than a TCK one. + * + *

{@link ControllableProvider} closes that gap by acquiring its flag store at + * {@code initialize()} time from a store that may refuse it. The store is in this JVM rather than + * over a socket, so this is not a licence for a provider that does have a backend to test itself + * this way — see {@link BackendControl}. What it is, is the TCK exercising its own lifecycle steps + * in seconds, on any machine, with no daemon. + * + *

It is a strict superset of {@link InMemoryProviderTckTest}'s coverage, not a replacement for + * it: that one stays the reference adoption for a provider with no backend, written against + * the published {@link InProcessBackendControl} and the SDK's own provider, and it is the thing an + * adopter copies. + * + *

If you are porting this shape to another language, the one non-obvious constraint is that + * {@link ControllableProvider} composes the SDK's in-memory provider instead of + * extending it: seeding a subclass's flags during {@code initialize()} means calling + * {@code updateFlags}, which emits {@code PROVIDER_CONFIGURATION_CHANGED}, so every initialisation + * would fire a spurious configuration-change event at the very scenarios that assert which events + * occur. That class's javadoc has the full reasoning and the emission it avoids. + */ +public class ControllableProviderTckTest extends ProviderTckTest { + + private final ControllableBackendControl control = new ControllableBackendControl(); + + @Override + public BackendControl backendControl() { + return control; + } + + @Override + public FeatureProvider createProvider() { + return control.createProvider(); + } + + /** + * {@inheritDoc} + * + *

A provider over an in-JVM store that refuses to answer. No socket and no port: the + * unreachability is the store's, which is all the {@code @unavailable} scenarios need — that + * initialisation fails observably and promptly, that a code default still comes back with reason + * {@code ERROR}, and that shutting such a provider down does not hang. + */ + @Override + public FeatureProvider createUnavailableProvider() { + return control.createUnavailableProvider(); + } + + /** + * {@inheritDoc} + * + *

{@link InMemoryProviderTckTest}'s six, plus the three this suite exists for. Each addition + * is a fact about {@link ControllableProvider} rather than a convenience: + * + *

    + *
  • {@link Capability#LIFECYCLE} — initialisation reaches a store it does not already hold, + * and can be refused by it, so {@code READY} is the observable outcome of that call rather + * than something the SDK manufactured for a provider with no initialisation step. That is + * the distinction the tag exists to draw, and it is why {@link InMemoryProviderTckTest} + * withholds it. + *
  • {@link Capability#REINITIALIZATION} — true here, and only declared because it is true: + * {@code shutdown()} drops the store and nothing else, {@code initialize()} acquires a + * fresh copy, and neither keeps a flag that would make the second call return early. + * Requirement 2.5.2 only permits reuse, so a provider that released something it + * could not recreate would leave this undeclared rather than record a + * {@link KnownDeviation}. + *
  • {@link Capability#UNAVAILABLE_INIT} — {@link #createUnavailableProvider()} really does + * fail to initialise, so the three {@code @unavailable} scenarios run instead of skipping. + *
+ * + *

And the omissions, which are the same as {@link InMemoryProviderTckTest}'s because every + * resolution decision here is still the SDK provider's — this class adds a lifecycle and + * delegates all evaluation. Each is a property of the delegate rather than a defect, so none of + * them rests on the self-test carve-out: + * + *

    + *
  • {@link Capability#NUMERIC_COERCION} — the delegate type-checks rather than coerces, so + * the two lossless scenarios would fail. Strict typing is a choice the SDK's reference + * provider is entitled to; the rule is borrowed from flagd's ADR rather than normative. + *
  • {@link Capability#STALE} — omitted. This provider's backend can refuse an + * initialisation, which is what {@code @unavailable} needs, but it cannot take a + * connection away from a running provider and put it back, which is what {@code @stale} + * needs. {@link BackendControl#disconnect()} therefore stays at its throwing default and + * the scenario is skipped before any step can reach it. Worth adding later; it is the one + * capability still without Docker-free coverage. + *
  • {@link Capability#TARGETING} — the delegate evaluates no rules, so + * {@code targeting-key-flag} resolves its {@code miss} variant whatever the context. + *
  • {@link Capability#CACHING} — reserved, so not declarable, and nothing is skipped by + * leaving it out. + *
+ * + *

{@link Capability#LARGE_INTEGERS} is absent from both lists because it is not a decision + * this suite takes: it is {@linkplain Capability#inexpressible() inexpressible} in Java and + * refused centrally. + */ + @Override + public Set capabilities() { + return EnumSet.of( + Capability.EVENTS, + Capability.LIFECYCLE, + Capability.REINITIALIZATION, + Capability.UNAVAILABLE_INIT, + Capability.CONFIGURATION_CHANGE, + Capability.OBJECT, + Capability.VARIANTS, + Capability.DISABLED_FLAGS, + Capability.STRING_TYPING, + Capability.FULLY_TYPED_VALUES, + Capability.STANDARD_REASONS); + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/DeclarationApiTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/DeclarationApiTest.java new file mode 100644 index 0000000000..b498ec7f41 --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/DeclarationApiTest.java @@ -0,0 +1,309 @@ +package dev.openfeature.contrib.tools.tck; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.opentest4j.TestAbortedException; + +/** + * The vocabulary an adopter declares conformance in, exercised where it is defined. + * + *

Everything here is something a provider author writes or a reader of a run + * needs: which capabilities are declarable, which gaps are defects rather than choices, + * what the configuration under test is called, and which of the two control contracts the run was + * conducted under. It is all usable with nothing downstream of it — no report, no emitter — which is + * the point of it living here. + */ +class DeclarationApiTest { + + @Test + @DisplayName("a reserved capability is not declarable and declaring one fails the run") + void reservedCapabilitiesAreNotDeclarable() { + assertThat(Capability.declarable()) + .as("declarable() is every capability some scenario gates") + .doesNotContain(Capability.CACHING) + .contains(Capability.EVENTS, Capability.OBJECT); + + assertThat(Capability.declarableExcept(Capability.STALE)) + .doesNotContain(Capability.STALE, Capability.CACHING) + .contains(Capability.EVENTS); + + assertThatThrownBy(() -> Capability.requireDeclarable(EnumSet.allOf(Capability.class))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("CACHING") + .hasMessageContaining("declares reserved"); + } + + @Test + @DisplayName("targeting is a declarable capability, not a reserved one") + void targetingIsDeclarable() { + // It was reserved while no scenario carried the tag. targeting-key-flag's three scenarios + // carry it now, so it gates something and the claim can be contradicted by a result -- + // which is the whole test for whether a capability may be declared. + assertThat(Capability.fromTag("@targeting")).contains(Capability.TARGETING); + assertThat(Capability.TARGETING.reserved()).isFalse(); + assertThat(Capability.declarable()).contains(Capability.TARGETING); + Capability.requireDeclarable(EnumSet.of(Capability.TARGETING)); + + // And the "declare everything" shortcut still excludes @caching, which is now the only + // reserved tag. That is the accident the shortcut exists to prevent, not a general one. + assertThat(Capability.CACHING.reserved()).isTrue(); + assertThat(Capability.declarable()).doesNotContain(Capability.CACHING); + assertThat(Capability.declarableExcept(Capability.TARGETING)) + .doesNotContain(Capability.TARGETING, Capability.CACHING); + } + + @Test + @DisplayName("variants is declarable, because 2.2.4 is a SHOULD and the field is optional") + void variantsIsADeclarableChoice() { + // Requirement 2.2.4 says a provider SHOULD populate the variant, and types.md types the + // field optional, so a backend with no variant concept withholds the tag rather than + // recording a deviation against a MUST that does not exist. + assertThat(Capability.fromTag("@variants")).contains(Capability.VARIANTS); + assertThat(Capability.VARIANTS.reserved()).isFalse(); + assertThat(Capability.declarable()).contains(Capability.VARIANTS); + + TestAbortedException aborted = catchThrowableOfType( + () -> CapabilityGate.requireDeclared( + Arrays.asList("@variants"), Capability.declarableExcept(Capability.VARIANTS)), + TestAbortedException.class); + assertThat(aborted).isNotNull(); + assertThat(aborted) + .hasMessageContaining("VARIANTS") + .hasMessageContaining("@variants") + .hasMessageContaining("does not declare"); + } + + @Test + @DisplayName("a capability the Java SDK cannot express is refused here, not left to every adopter") + void aCapabilityTheSdkCannotExpressIsRefusedCentrally() { + // @large-integers asks for 2^53 - 1 and the Java SDK's accessor is a 32-bit Integer, so no + // Java provider can be asked it -- ever, until the SDK changes. Three suites in this + // repository used to withhold it by hand, each with its own comment restating this + // paragraph. The implementation refuses it instead, so an adopter neither has to know it + // nor can get it wrong. + assertThat(Capability.LARGE_INTEGERS.inexpressible()).isTrue(); + assertThat(Capability.declarable()) + .as("no Java provider may claim it, so \"everything\" does not include it") + .doesNotContain(Capability.LARGE_INTEGERS); + assertThat(Capability.declarableExcept(Capability.EVENTS)) + .as("nor does \"everything except\", which is what the adoptions call") + .doesNotContain(Capability.LARGE_INTEGERS); + + assertThatThrownBy(() -> Capability.requireDeclarable(EnumSet.of(Capability.EVENTS, Capability.LARGE_INTEGERS))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("LARGE_INTEGERS") + .hasMessageContaining("@large-integers") + .as("the message says why, naming the SDK property rather than citing a rule") + .hasMessageContaining("Client.getIntegerDetails") + .hasMessageContaining("cannot express"); + } + + @Test + @DisplayName("the two refusals are different facts, and neither message could be mistaken for the other") + void reservedAndInexpressibleAreToldApart() { + // A reader who sees a capability missing from a report has to be able to tell "no scenario + // anywhere carries this tag yet" from "the scenarios exist and this SDK cannot ask them". + // Only the second is permanent, and neither says anything about the provider under test -- + // which is the third thing they must not be mistaken for. + assertThat(Capability.CACHING.reserved()).isTrue(); + assertThat(Capability.CACHING.inexpressible()) + .as("a reservation is global and expires; it is not a language's limit") + .isFalse(); + assertThat(Capability.LARGE_INTEGERS.reserved()) + .as("scenarios do carry @large-integers, which is what makes it not a reservation") + .isFalse(); + + String reserved = catchThrowableOfType( + () -> Capability.requireDeclarable(EnumSet.of(Capability.CACHING)), + IllegalArgumentException.class) + .getMessage(); + String inexpressible = catchThrowableOfType( + () -> Capability.requireDeclarable(EnumSet.of(Capability.LARGE_INTEGERS)), + IllegalArgumentException.class) + .getMessage(); + + assertThat(reserved) + .as("the reserved refusal says there is nothing to gate yet") + .contains("no scenario in the suite carries") + .as("and never blames the SDK, because a reservation is every language's") + .doesNotContain("SDK"); + assertThat(inexpressible) + .as("the inexpressible refusal says the opposite: the scenarios exist elsewhere") + .contains("the scenarios exist and are asked in languages whose API is wide enough") + .contains("This is not a reserved capability") + .doesNotContain("no scenario in the suite carries"); + + // And the same distinction survives into the skip, which is where a report's reader meets + // it. An undeclared capability names the provider; an inexpressible one must not, because + // the provider had no say. + TestAbortedException undeclared = catchThrowableOfType( + () -> CapabilityGate.requireDeclared( + Arrays.asList("@variants"), Capability.declarableExcept(Capability.VARIANTS)), + TestAbortedException.class); + TestAbortedException unaskable = catchThrowableOfType( + () -> CapabilityGate.requireDeclared(Arrays.asList("@large-integers"), Capability.declarable()), + TestAbortedException.class); + + assertThat(undeclared).hasMessageContaining("provider does not declare capability"); + assertThat(unaskable) + .hasMessageContaining("the Java SDK cannot express capability LARGE_INTEGERS") + .hasMessageContaining("Client.getIntegerDetails") + .as("not the provider's decision, and the reason has to say so") + .hasMessageContaining("not the provider under test declining"); + assertThat(unaskable.getMessage()).doesNotContain("provider does not declare"); + + // The inexpressible reason is reached whatever the declaration says, because a declaration + // cannot contain it. Checking it after the declaration would make the right reason appear + // only by luck. + TestAbortedException evenIfSomehowDeclared = catchThrowableOfType( + () -> CapabilityGate.requireDeclared( + Arrays.asList("@large-integers"), EnumSet.of(Capability.LARGE_INTEGERS)), + TestAbortedException.class); + assertThat(evenIfSomehowDeclared).hasMessageContaining("the Java SDK cannot express capability"); + } + + @Test + @DisplayName("a tag maps back to the capability it gates") + void tagsMapBackToCapabilities() { + assertThat(Capability.fromTag("@numeric-coercion")).contains(Capability.NUMERIC_COERCION); + assertThat(Capability.fromTag("@not-a-capability")).isEmpty(); + } + + @Test + @DisplayName("reinitialisation is a declarable choice, and withholding it skips only its own scenario") + void reinitialisationIsADeclarableChoice() { + // Requirement 2.5.2 permits reuse after shutdown rather than requiring it, so a provider + // that refuses it withholds the tag instead of recording a deviation. That makes it an + // ordinary declarable capability rather than a reserved one. + assertThat(Capability.fromTag("@reinitialization")).contains(Capability.REINITIALIZATION); + assertThat(Capability.REINITIALIZATION.reserved()).isFalse(); + assertThat(Capability.declarable()).contains(Capability.REINITIALIZATION); + + // The scenario carries @lifecycle too. A provider that initialises against a backend it + // does not reopen declares the first and withholds the second, and only the one scenario + // is skipped -- the rest of the lifecycle set still runs. + Set reachesBackendButNoReuse = EnumSet.of(Capability.LIFECYCLE); + CapabilityGate.requireDeclared(Arrays.asList("@lifecycle"), reachesBackendButNoReuse); + + TestAbortedException aborted = catchThrowableOfType( + () -> CapabilityGate.requireDeclared( + Arrays.asList("@lifecycle", "@reinitialization"), reachesBackendButNoReuse), + TestAbortedException.class); + assertThat(aborted).isNotNull(); + assertThat(aborted) + .hasMessageContaining("REINITIALIZATION") + .hasMessageContaining("@reinitialization") + .hasMessageContaining("does not declare"); + } + + @Test + @DisplayName("the gate skips an undeclared capability and lets an untagged scenario run") + void theGateSkipsUndeclaredCapabilities() { + Set declared = EnumSet.of(Capability.EVENTS); + + TestAbortedException aborted = catchThrowableOfType( + () -> CapabilityGate.requireDeclared(Arrays.asList("@object"), declared), TestAbortedException.class); + assertThat(aborted) + .as("an undeclared capability aborts, which is what the JUnit Platform reports as skipped") + .isNotNull(); + assertThat(aborted).hasMessageContaining("OBJECT").hasMessageContaining("@object"); + + CapabilityGate.requireDeclared(Arrays.asList("@events", "@not-a-capability"), declared); + CapabilityGate.requireDeclared(Collections.emptyList(), declared); + } + + @Test + @DisplayName("a deviation records the capability the gap is about, tracked or not") + void deviationsRecordTheCapabilityTheGapIsAbout() { + KnownDeviation tracked = KnownDeviation.tracked( + Capability.NUMERIC_COERCION, "https://example.invalid/1234", "0.5 as an integer returns 0"); + assertThat(tracked.capability).isEqualTo("@numeric-coercion"); + assertThat(tracked.issue).isEqualTo("https://example.invalid/1234"); + assertThat(tracked.summary).isEqualTo("0.5 as an integer returns 0"); + + KnownDeviation untracked = KnownDeviation.untracked(null, "a gap against a mandatory scenario"); + assertThat(untracked.capability).isNull(); + assertThat(untracked.issue).isNull(); + } + + @Test + @DisplayName("a deviation cannot name a capability whose scenarios were never put to the provider") + void deviationsCannotNameAnUnaskedCapability() { + // A deviation asserts that the provider fails something it is required to do, so it has to + // be about a question that was actually asked. Two never are, and they are refused apart + // for the same reason the declaration refuses them apart -- this is the same claim reaching + // the report by a second route, and the more damaging one, because a deviation reads as an + // admission of fault. + assertThatThrownBy(() -> KnownDeviation.untracked(Capability.CACHING, "no scenario carries it")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("reserved CACHING") + .hasMessageContaining("no scenario in the suite carries") + .hasMessageNotContaining("SDK"); + + assertThatThrownBy(() -> KnownDeviation.tracked( + Capability.LARGE_INTEGERS, "https://example.invalid/1", "cannot be asked")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("LARGE_INTEGERS") + .hasMessageContaining("the Java SDK cannot express") + .hasMessageContaining("Client.getIntegerDetails") + .as("it is not a reservation, and the message must not read as one") + .hasMessageNotContaining("no scenario in the suite carries"); + + // Null still means "a gap against a mandatory, ungated scenario", and an ordinary capability + // is still allowed in both shapes. + assertThat(KnownDeviation.untracked(null, "a mandatory gap").capability).isNull(); + assertThat(KnownDeviation.untracked(Capability.DISABLED_FLAGS, "withheld and skipped").capability) + .isEqualTo("@disabled-flags"); + } + + @Test + @DisplayName("a configuration name is derived from the suite class, and is overridable") + void configurationNamesAreDerivedFromTheSuiteClass() { + assertThat(new TckSuiteFixture().configuration()).isEqualTo("tck-suite-fixture"); + assertThat(new NamedConfiguration().configuration()).isEqualTo("a-name-of-my-own"); + } + + @Test + @DisplayName("an adopter declares nothing by default, which is silence rather than a claim") + void theDefaultsAreSilence() { + assertThat(new TckSuiteFixture().knownDeviations()).isEmpty(); + assertThat(new TckSuiteFixture().capabilities()).isEqualTo(Capability.declarable()); + } + + @Test + @DisplayName("a backend says which of the two control contracts drove it, and cannot stay silent") + void backendsSayHowTheyWereDriven() throws NoSuchMethodException { + assertThat(new InProcessBackendControl().controlApi()) + .as("a provider with no backend is controlled in-process, the narrow allowance") + .isEqualTo(ControlApi.IN_PROCESS); + + assertThat(ControlApi.HTTP.wireValue()).isEqualTo("http"); + assertThat(ControlApi.IN_PROCESS.wireValue()).isEqualTo("in-process"); + + // Closed on purpose: the report schema's enum has exactly these two members, and there is + // no third case an unanswered value would legitimately cover. + assertThat(ControlApi.values()).containsExactly(ControlApi.HTTP, ControlApi.IN_PROCESS); + + // Required on purpose: a default here would answer a question only the author of a custom + // control can answer, so the compiler has to ask for it. + assertThat(BackendControl.class.getMethod("controlApi").isDefault()) + .as("controlApi() must be abstract, so that a custom control states it") + .isFalse(); + } + + /** A suite that names its configuration rather than taking the derived name. */ + private static final class NamedConfiguration extends TckSuiteFixture { + @Override + public String configuration() { + return "a-name-of-my-own"; + } + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ExtensionPointTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ExtensionPointTest.java new file mode 100644 index 0000000000..8bfb0c7549 --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ExtensionPointTest.java @@ -0,0 +1,183 @@ +package dev.openfeature.contrib.tools.tck; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.cucumber.junit.platform.engine.Constants; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.platform.engine.TestSource; +import org.junit.platform.engine.discovery.DiscoverySelectors; +import org.junit.platform.engine.support.descriptor.ClasspathResourceSource; +import org.junit.platform.launcher.TestIdentifier; +import org.junit.platform.launcher.TestPlan; +import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; +import org.junit.platform.launcher.core.LauncherFactory; +import org.junit.platform.launcher.listeners.SummaryGeneratingListener; +import org.junit.platform.suite.api.ConfigurationParameter; + +/** + * An adopter extends the suite by convention, writing no annotations. + * + *

What that promise decomposes into, and what each test here checks: + * + *

    + *
  • a feature file under {@code extensions/} on the adopter's test classpath is discovered by + * the suite, into the same Cucumber engine as the canonical set — which is what "the same + * backend lifecycle phase" means, since {@code @BeforeAll} is scoped to exactly that; + *
  • a step class in {@code openfeature.tck.extensions} is resolved from the glue path; + *
  • the extension directory shipped in this artifact keeps the selector resolvable for an adopter + * who extends nothing. + *
+ * + *

The fixture feature and steps are test-scoped, so they are not in the released JAR. They live + * where an adopter's would live, which is the only way to check that the convention holds without + * asserting it about a path that no build actually uses. + */ +class ExtensionPointTest { + + private static final String EXTENSION_FEATURE = ProviderTck.EXTENSIONS + "/extension-selftest.feature"; + + @Test + @DisplayName("an extension feature is discovered into the same suite and engine as the canonical set") + void extensionsJoinTheCanonicalSuite() { + TestPlan plan = discover(LauncherDiscoveryRequestBuilder.request() + .selectors(DiscoverySelectors.selectClass(TckSuiteFixture.class))); + + TestIdentifier suite = only( + plan, identifier -> TckSuiteListener.harnessClassOf(identifier).isPresent()); + TestIdentifier cucumber = only( + plan, identifier -> "cucumber".equals(engineIdOf(identifier)) && isDescendant(plan, identifier, suite)); + + List resources = new ArrayList<>(); + for (TestIdentifier test : testsUnder(plan, cucumber)) { + resourceOf(test).ifPresent(resources::add); + } + + assertThat(resources) + .as("the extension scenario and the canonical scenarios are children of one Cucumber engine, " + + "under one suite — so they share its @BeforeAll and its backend") + .contains(EXTENSION_FEATURE) + .contains(ProviderTck.FEATURES + "/errors.feature"); + } + + @Test + @DisplayName("the extension glue package is resolved, so an adopter's steps need no registration") + void theExtensionGluePackageIsResolved() { + // Executed rather than discovered, because an unresolved glue package is not a discovery + // failure — it produces undefined steps at execution time, which is what this rules out. + // Only the extension glue is on the path here: the canonical steps would start a Compose + // stack in their @BeforeAll, and Docker is not this module's test dependency. + SummaryGeneratingListener summary = new SummaryGeneratingListener(); + LauncherFactory.create() + .execute( + LauncherDiscoveryRequestBuilder.request() + .selectors(DiscoverySelectors.selectClasspathResource(EXTENSION_FEATURE)) + .configurationParameter(Constants.GLUE_PROPERTY_NAME, ProviderTck.EXTENSION_GLUE) + .configurationParameter( + Constants.PARALLEL_EXECUTION_ENABLED_PROPERTY_NAME, + ProviderTck.PARALLEL_EXECUTION_ENABLED) + .configurationParameter( + Constants.OBJECT_FACTORY_PROPERTY_NAME, ProviderTck.OBJECT_FACTORY) + .build(), + summary); + + assertThat(summary.getSummary().getTestsSucceededCount()) + .as("the fixture scenario ran with its steps resolved from %s", ProviderTck.EXTENSION_GLUE) + .isEqualTo(1); + assertThat(summary.getSummary().getTotalFailureCount()).isZero(); + } + + @Test + @DisplayName("the extension directory ships in this artifact, so the selector resolves with no adopter files") + void theExtensionDirectoryIsShipped() { + // A @SelectClasspathResource naming a resource on no classpath root is a hard discovery + // error, so an adopter who extends nothing depends on this file existing in the JAR. + assertThat(getClass().getClassLoader().getResource(ProviderTck.EXTENSIONS + "/README.md")) + .as( + "%s/README.md keeps the extension selector resolvable for an adopter who adds nothing", + ProviderTck.EXTENSIONS) + .isNotNull(); + } + + @Test + @DisplayName("the glue constant composes in an annotation value") + void theGlueConstantComposesInAnAnnotationValue() { + // The assertion is a formality; the compilation of VendorGlue is the actual evidence, since + // an annotation value has to be a compile-time constant and a method call would not compile. + ConfigurationParameter parameter = VendorGlue.class.getAnnotation(ConfigurationParameter.class); + + assertThat(parameter.value()) + .isEqualTo("dev.openfeature.contrib.tools.tck.steps," + "openfeature.tck.extensions,com.vendor.steps"); + } + + /** An adopter who wants a third glue package writes this, and does not restate our package. */ + @ConfigurationParameter(key = Constants.GLUE_PROPERTY_NAME, value = ProviderTck.ALL_GLUE + ",com.vendor.steps") + private static final class VendorGlue {} + + private static TestPlan discover(LauncherDiscoveryRequestBuilder request) { + return LauncherFactory.create().discover(request.build()); + } + + private static Optional resourceOf(TestIdentifier identifier) { + Optional source = identifier.getSource(); + if (!source.isPresent() || !(source.get() instanceof ClasspathResourceSource)) { + return Optional.empty(); + } + return Optional.of(((ClasspathResourceSource) source.get()).getClasspathResourceName()); + } + + private static String engineIdOf(TestIdentifier identifier) { + return identifier.getUniqueIdObject().getLastSegment().getType().equals("engine") + ? identifier.getUniqueIdObject().getLastSegment().getValue() + : null; + } + + private static boolean isDescendant(TestPlan plan, TestIdentifier identifier, TestIdentifier ancestor) { + Optional parent = plan.getParent(identifier); + while (parent.isPresent()) { + if (parent.get().equals(ancestor)) { + return true; + } + parent = plan.getParent(parent.get()); + } + return false; + } + + private static List testsUnder(TestPlan plan, TestIdentifier root) { + List tests = new ArrayList<>(); + collectTests(plan, root, tests); + return tests; + } + + private static void collectTests(TestPlan plan, TestIdentifier identifier, List into) { + if (identifier.isTest()) { + into.add(identifier); + } + for (TestIdentifier child : plan.getChildren(identifier)) { + collectTests(plan, child, into); + } + } + + private static TestIdentifier only(TestPlan plan, Predicate predicate) { + List matching = new ArrayList<>(); + for (TestIdentifier root : plan.getRoots()) { + collectMatching(plan, root, predicate, matching); + } + assertThat(matching).hasSize(1); + return matching.get(0); + } + + private static void collectMatching( + TestPlan plan, TestIdentifier identifier, Predicate predicate, List into) { + if (predicate.test(identifier)) { + into.add(identifier); + } + for (TestIdentifier child : plan.getChildren(identifier)) { + collectMatching(plan, child, predicate, into); + } + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/HttpBackendControlTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/HttpBackendControlTest.java new file mode 100644 index 0000000000..73a39e17a8 --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/HttpBackendControlTest.java @@ -0,0 +1,392 @@ +package dev.openfeature.contrib.tools.tck; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.IOException; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * Guards the request sequence {@link HttpBackendControl} issues, which no scenario can assert. + * + *

Every scenario's isolation rests on {@link HttpBackendControl#prepareScenario()} choosing the + * right endpoint against a backend that implements only part of the control API, and on a + * disconnect being remembered until something ends it. Both are invisible from inside the Gherkin: + * a control that reset nothing would leave each scenario running against whatever state the + * previous one left behind, and the suite would report those results as conformance. A control that + * reset a backend it had just stopped would register the next scenario's provider against a backend + * that is still down, and report the failure as a provider defect. + * + *

So the control API is stubbed with the JDK's own {@link HttpServer} — no Docker, no + * Testcontainers, nothing off loopback — and the requests actually sent are asserted, in order. The + * three rules being pinned are normative in {@code openapi/control-api.yaml}: {@code /reset} is + * preferred and {@code /start} is the documented fallback; the fallback is detected once per suite + * and cached; and {@code /reset} is not specified to start a stopped backend, so the scenario after + * a disconnect must use {@code /start}. + */ +class HttpBackendControlTest { + + private static final Duration PROBE_BUDGET = Duration.ofSeconds(5); + + @Test + @DisplayName("prepareScenario uses /reset for every scenario when the backend implements it") + void prefersReset() throws Exception { + try (StubControlApi stub = StubControlApi.start()) { + HttpBackendControl control = new HttpBackendControl(stub.baseUrl(), "default"); + + control.prepareScenario(); + control.prepareScenario(); + + // The preferred primitive, because it causes no availability blip: a /start between + // scenarios restarts the backend, which the provider under test may legitimately + // report as a lifecycle event in the scenario that follows. + assertThat(stub.paths()).containsExactly("/reset", "/reset"); + } + } + + @Test + @DisplayName("an unimplemented /reset is probed once, then every scenario falls back to /start") + void fallsBackToStartAndCachesTheAnswer() throws Exception { + try (StubControlApi stub = StubControlApi.start().answering("/reset", 404)) { + HttpBackendControl control = new HttpBackendControl(stub.baseUrl(), "default"); + + control.prepareScenario(); + control.prepareScenario(); + control.prepareScenario(); + + // Once per suite, not once per scenario: a wasted 404 before every scenario is a slow + // suite, and never probing at all would mean a backend that grows /reset is never used + // properly. This is the path flagd-testbed actually takes — its launchpad has no /reset. + assertThat(stub.paths()).containsExactly("/reset", "/start", "/start", "/start"); + } + } + + @ParameterizedTest + @ValueSource(ints = {404, 501}) + @DisplayName("both documented not-implemented statuses trigger the fallback") + void bothNotImplementedStatusesFallBack(int status) throws Exception { + try (StubControlApi stub = StubControlApi.start().answering("/reset", status)) { + // control-api.yaml permits either, so neither may be treated as a failed control call. + new HttpBackendControl(stub.baseUrl(), "default").prepareScenario(); + + assertThat(stub.paths()).containsExactly("/reset", "/start"); + } + } + + @Test + @DisplayName("the scenario after a disconnect starts the backend rather than resetting it") + void aDisconnectForcesStart() throws Exception { + try (StubControlApi stub = StubControlApi.start()) { + HttpBackendControl control = new HttpBackendControl(stub.baseUrl(), "default"); + control.prepareScenario(); // settles on /reset, which this stub implements + stub.forget(); + + control.disconnect(); + control.prepareScenario(); + + // /reset restores flag state and is explicitly not specified to start a stopped + // backend. Without this the next scenario would prepare a backend that is still down. + assertThat(stub.paths()).containsExactly("/stop", "/start"); + } + } + + @Test + @DisplayName("reconnecting clears the disconnect, so the next scenario resets again") + void reconnectClearsTheDisconnect() throws Exception { + try (StubControlApi stub = StubControlApi.start()) { + HttpBackendControl control = new HttpBackendControl(stub.baseUrl(), "default"); + control.prepareScenario(); + control.disconnect(); + control.reconnect(); + stub.forget(); + + control.prepareScenario(); + + // A scenario that ended its own outage leaves the backend up, so the blip-free + // primitive is available again and the next scenario should not pay for a restart. + assertThat(stub.paths()).containsExactly("/reset"); + } + } + + @Test + @DisplayName("the fallback and the reconnect both name the configuration under test") + void startNamesTheBackendConfiguration() throws Exception { + try (StubControlApi stub = StubControlApi.start().answering("/reset", 404)) { + new HttpBackendControl(stub.baseUrl(), "ssl").prepareScenario(); + + assertThat(stub.requests()).containsExactly("POST /reset", "POST /start?config=ssl"); + } + } + + @Test + @DisplayName("no operation ever reaches /restart") + void nothingCallsRestart() throws Exception { + try (StubControlApi stub = StubControlApi.start()) { + HttpBackendControl control = new HttpBackendControl(stub.baseUrl(), "default"); + + control.prepareScenario(); + control.changeFlag(); + control.disconnect(); + control.reconnect(); + control.prepareScenario(); + + // /restart is optional in control-api.yaml and no shipped scenario reaches it: the + // disconnect/reconnect scenario is an unbounded outage asserted in two steps, because + // a self-healing restart races the stale assertion. Asserted over the wire rather than + // by reflection, so reinstating a caller cannot slip past by using a different name. + assertThat(stub.paths()).doesNotContain("/restart"); + } + } + + @Test + @DisplayName("changeFlag posts /change") + void changeFlagPostsChange() throws Exception { + try (StubControlApi stub = StubControlApi.start()) { + new HttpBackendControl(stub.baseUrl(), "default").changeFlag(); + + assertThat(stub.paths()).containsExactly("/change"); + } + } + + @Test + @DisplayName("an unexpected status fails loudly instead of passing silently") + void anUnexpectedStatusThrows() throws Exception { + try (StubControlApi stub = StubControlApi.start().answering("/change", 500)) { + HttpBackendControl control = new HttpBackendControl(stub.baseUrl(), "default"); + + // A control call that did nothing would leave the scenario in an unknown state and its + // assertions would then be measuring the previous scenario's backend. + assertThatThrownBy(control::changeFlag) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("HTTP 500") + .hasMessageContaining("control-api.yaml"); + } + } + + @Test + @DisplayName("an unreachable control API says which call failed and where") + void anUnreachableControlApiThrows() throws Exception { + String baseUrl = StubControlApi.addressOfAClosedServer(); + HttpBackendControl control = new HttpBackendControl(baseUrl, "default"); + + // The control API must stay reachable even while the backend is deliberately down, so this + // is a broken stack rather than an outage, and it has to read that way. + assertThatThrownBy(control::changeFlag) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("POST " + baseUrl + "/change"); + } + + @Test + @DisplayName("baseUrl is reportable without rebuilding it") + void baseUrlIsReportable() throws Exception { + try (StubControlApi stub = StubControlApi.start()) { + HttpBackendControl control = new HttpBackendControl(stub.baseUrl(), "default"); + + assertThat(control.baseUrl()).isEqualTo(stub.baseUrl()); + assertThat(control.description()).contains(stub.baseUrl()); + assertThat(control.controlApi()).isEqualTo(ControlApi.HTTP); + } + } + + // -- awaitReady -------------------------------------------------------------------------- + + @Test + @DisplayName("awaitReady returns as soon as /healthz answers, without sleeping first") + void awaitReadyReturnsOnTheFirstAnswer() throws Exception { + try (StubControlApi stub = StubControlApi.start()) { + new HttpBackendControl(stub.baseUrl(), "default").awaitReady(PROBE_BUDGET); + + // The readiness check is what replaced the old post-command settle: it probes the thing + // whose readiness is in question, so a slow control API is waited for and a dead one is + // reported, and neither costs a fixed pause. + assertThat(stub.requests()).containsExactly("GET /healthz"); + } + } + + @Test + @DisplayName("awaitReady treats an unimplemented /healthz as ready") + void awaitReadyAcceptsNotImplemented() throws Exception { + try (StubControlApi stub = StubControlApi.start().answering("/healthz", 404)) { + // 404 is "not implemented", which control-api.yaml defines as ready: readiness then + // rests on the control port accepting a connection, already established by the Compose + // wait strategy. flagd-testbed's launchpad serves no /healthz, so this is the normal + // path rather than an edge case. + new HttpBackendControl(stub.baseUrl(), "default").awaitReady(PROBE_BUDGET); + + assertThat(stub.paths()).containsExactly("/healthz"); + } + } + + @Test + @DisplayName("awaitReady keeps probing while the control API says not yet") + void awaitReadyRetriesNotReady() throws Exception { + try (StubControlApi stub = StubControlApi.start().scripting("/healthz", 503, 503, 200)) { + new HttpBackendControl(stub.baseUrl(), "default").awaitReady(Duration.ofSeconds(10)); + + // 503 is the control API saying "not ready", so it is retried rather than accepted. + assertThat(stub.paths()).containsExactly("/healthz", "/healthz", "/healthz"); + } + } + + @Test + @DisplayName("awaitReady gives up reporting what the last probe actually saw") + void awaitReadyReportsTheLastProbe() throws Exception { + try (StubControlApi stub = StubControlApi.start().answering("/healthz", 503)) { + HttpBackendControl control = new HttpBackendControl(stub.baseUrl(), "default"); + + // "did not become ready" alone sends an adopter to the wrong place: a refused + // connection is a stack that never came up, a 503 is one that is up and not finished. + assertThatThrownBy(() -> control.awaitReady(Duration.ofMillis(300))) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("did not become ready") + .hasRootCauseMessage("control API not ready, HTTP 503"); + } + } + + @Test + @DisplayName("awaitReady reports a control API that is not there at all") + void awaitReadyReportsAnAbsentControlApi() throws Exception { + String baseUrl = StubControlApi.addressOfAClosedServer(); + HttpBackendControl control = new HttpBackendControl(baseUrl, "default"); + + assertThatThrownBy(() -> control.awaitReady(Duration.ofMillis(300))) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("did not become ready") + .hasRootCauseInstanceOf(IOException.class); + } + + /** + * A control API that records every request it received and answers a scripted status. + * + *

Bound to the loopback interface on an ephemeral port, so a test never contends for a + * fixed port and never leaves the machine. + */ + private static final class StubControlApi implements AutoCloseable { + + private final HttpServer server; + private final List requests = Collections.synchronizedList(new ArrayList<>()); + private final Map statuses = new HashMap<>(); + private final Map> scripted = new HashMap<>(); + + private StubControlApi(HttpServer server) { + this.server = server; + } + + static StubControlApi start() throws IOException { + HttpServer server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + StubControlApi stub = new StubControlApi(server); + server.createContext("/", stub::answer); + server.start(); + return stub; + } + + /** + * Binds and immediately closes a server, so the returned address is almost certainly not + * listening. Used for the "the stack is not up" cases. + */ + static String addressOfAClosedServer() throws IOException { + try (StubControlApi stub = start()) { + return stub.baseUrl(); + } + } + + /** Answers {@code status} for every request to {@code path}. */ + StubControlApi answering(String path, int status) { + statuses.put(path, status); + return this; + } + + /** + * Answers the given statuses one per request to {@code path}, which is how "not ready, then + * ready" is expressed. The last one repeats once the script is exhausted. + */ + StubControlApi scripting(String path, int... sequence) { + Deque queue = new ArrayDeque<>(); + for (int status : sequence) { + queue.add(status); + } + scripted.put(path, queue); + return this; + } + + String baseUrl() { + return "http://" + server.getAddress().getHostString() + ":" + + server.getAddress().getPort(); + } + + /** Every request as {@code METHOD path[?query]}, in the order received. */ + List requests() { + synchronized (requests) { + return new ArrayList<>(requests); + } + } + + /** Every request's path, dropping the method and the query string. */ + List paths() { + return requests().stream() + .map(request -> request.substring(request.indexOf(' ') + 1)) + .map(target -> target.contains("?") ? target.substring(0, target.indexOf('?')) : target) + .collect(Collectors.toList()); + } + + /** Drops what has been recorded, so a test can assert only the part it set up. */ + void forget() { + requests.clear(); + } + + private void answer(HttpExchange exchange) { + try { + String path = exchange.getRequestURI().getPath(); + String query = exchange.getRequestURI().getQuery(); + requests.add(exchange.getRequestMethod() + " " + path + (query == null ? "" : "?" + query)); + + int status = nextStatus(path); + + byte[] body = "{\"status\":\"stub\"}".getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, body.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(body); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } finally { + exchange.close(); + } + } + + /** The scripted status for this path, or the fixed one, defaulting to 200. */ + private synchronized int nextStatus(String path) { + Deque script = scripted.get(path); + if (script == null) { + return statuses.getOrDefault(path, 200); + } + // The last entry repeats, so a script cannot run dry and turn into a 200 by accident. + return script.size() > 1 ? script.poll() : script.peek(); + } + + @Override + public void close() { + server.stop(0); + } + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/InMemoryProviderTckTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/InMemoryProviderTckTest.java new file mode 100644 index 0000000000..b69a25d279 --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/InMemoryProviderTckTest.java @@ -0,0 +1,128 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.FeatureProvider; +import dev.openfeature.sdk.providers.memory.InMemoryProvider; +import java.util.EnumSet; +import java.util.Set; + +/** + * Runs the OpenFeature Provider TCK against the SDK's own {@link InMemoryProvider}. + * + *

This is the TCK's self-test, and it earns its keep twice over. + * + *

It is the reference adoption for a provider with no backend. Everything a + * file-based or environment-variable provider needs to write is here, and it is three methods: hand + * over a {@link BackendControl}, hand over a provider, and say which capabilities hold. + * + *

It is also the Docker-free canary. Because it needs no container, no Compose + * stack and no network, it runs in seconds on any machine and in any CI job, which makes it the + * fast check that catches a broken step definition, a mis-wired capability gate or a regression in + * the shared harness long before the containerised suites get a chance to. When a change breaks + * both this and the flagd suite, this one tells you within seconds and points at the TCK rather + * than at a provider. + * + *

Note what it does not do: it is not a licence for providers that have a backend to + * test themselves this way. See {@link BackendControl} for why. + */ +public class InMemoryProviderTckTest extends ProviderTckTest { + + /** + * Both the flag store and the factory for the provider that serves it. + * + *

In-process the two are the same thing — {@code changeFlag()} has to reach the live provider + * instance to emit an event from it — so one instance backs both harness methods below. + */ + private final InProcessBackendControl control = new InProcessBackendControl(); + + @Override + public BackendControl backendControl() { + return control; + } + + @Override + public FeatureProvider createProvider() { + return control.createProvider(); + } + + /** + * {@inheritDoc} + * + *

Six capabilities, three of which are not obvious. {@link Capability#VARIANTS} holds because + * {@link InMemoryProvider} does name the variant it served, so the gated variant outline runs + * and passes here. {@link Capability#DISABLED_FLAGS} holds because it honours a flag's state: the + * four {@code disabled-*} flags resolve to nothing and the caller's default stands in, with no + * error code, so all four rows of that outline pass. That is not a given for an in-memory + * provider — the capability is gated precisely because whether the substitution can happen at all + * depends on where it happens — and it was measured rather than assumed. + * + *

{@link Capability#STANDARD_REASONS} is the third, and it was measured the same way rather + * than inferred from the provider's source. {@link InMemoryProvider} reports {@code STATIC} for a + * rule-less flag, {@code ERROR} beside {@code FLAG_NOT_FOUND} and {@code TYPE_MISMATCH}, and + * {@code DISABLED} for a disabled flag, so seven of {@code reason.feature}'s nine scenarios run + * and pass. The other two carry {@code @targeting} as well and are skipped for that omission — + * the tag composition doing its job, since a provider that evaluates no rules has no + * {@code TARGETING_MATCH} to report and failing it for the absence would say nothing. Each + * omission below is a fact about {@link InMemoryProvider} rather than a convenience, and each is + * a property rather than a defect — so none of them leans on the self-test carve-out + * Appendix F grants these suites, and {@link MultiProviderTckTest} is the only one in this + * module that does: + * + *

    + *
  • {@link Capability#NUMERIC_COERCION} — omitted. {@link InMemoryProvider} keeps the two + * numeric types strictly apart in both directions: a variant satisfies a request only if + * it is an instance of the requested type. That passes the lossy half of the rule — 0.5 + * requested as an integer is {@code TYPE_MISMATCH} — and fails the lossless half, because + * {@code integral-float-flag} (10.0) requested as an integer and {@code integer-flag} (10) + * requested as a float are refused just the same, and the tag requires all three. The + * rule is borrowed from flagd's ADR rather than from the specification, so strict typing + * is a choice the SDK's reference provider is entitled to, not a defect to declare; the + * capability is withheld and the three scenarios are skipped with that reason. Declare it + * again if the SDK ever adopts the coercion rule. Appendix F's scenario-level declaring + * rule does not reach this omission: it decides whether a question is askable, + * not whether the provider owes an answer, and no requirement says this one is owed. + *
  • {@link Capability#LIFECYCLE} — omitted. {@link InMemoryProvider} is handed its whole + * flag set by its constructor, so initialisation acquires nothing and cannot be refused, + * and the readiness scenario would pass without demonstrating anything — which is exactly + * what that capability exists to distinguish. The six scenarios it gates are covered + * without Docker by {@link ControllableProviderTckTest}, whose provider does acquire its + * store at {@code initialize()} time. + *
  • {@link Capability#REINITIALIZATION} — omitted, and nothing turns on it here: the + * scenario it gates carries {@code @lifecycle} as well, so it is already skipped for the + * omission above. Named anyway, because {@link Capability#declarable()} would have claimed + * it and this suite never examined it. {@link ControllableProviderTckTest} does. + *
  • {@link Capability#STALE} — omitted. There is no connection to lose, so the provider can + * never go {@code STALE}. {@link InProcessBackendControl} leaves + * {@link BackendControl#disconnect()} unimplemented for the same reason, and this omission + * is what keeps the two consistent: the scenario is skipped before any step can reach the + * unsupported operation. + *
  • {@link Capability#UNAVAILABLE_INIT} — omitted. Initialisation cannot fail when there is + * nothing to connect to, so + * {@link ProviderTckHarness#createUnavailableProvider()} is left at its throwing default. + *
  • {@link Capability#TARGETING} — omitted. {@link InMemoryProvider} evaluates no rules: it + * reads a flag's {@code variants} and {@code defaultVariant} and returns the default one, + * so the {@code targeting} member of {@code targeting-key-flag} is inert here and a + * matching context resolves {@code miss} like any other. The three scenarios are skipped + * with that reason rather than failed, which is what the tag is for. + *
  • {@link Capability#CACHING} — reserved, so not declarable and nothing is skipped by + * leaving it out. + *
+ * + *

{@link Capability#LARGE_INTEGERS} is not in that list and is not this suite's to omit: + * it is {@linkplain Capability#inexpressible() inexpressible} in Java, so + * {@link Capability#requireDeclarable} refuses it and its scenario is skipped for a reason that + * names the SDK. That used to be a bullet here, and an identical one in every other suite in + * this repository. + */ + @Override + public Set capabilities() { + return EnumSet.of( + Capability.EVENTS, + Capability.CONFIGURATION_CHANGE, + Capability.OBJECT, + Capability.VARIANTS, + Capability.DISABLED_FLAGS, + Capability.STRING_TYPING, + Capability.FULLY_TYPED_VALUES, + Capability.STANDARD_REASONS); + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/InProcessBackendControlTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/InProcessBackendControlTest.java new file mode 100644 index 0000000000..caeb3a4d7c --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/InProcessBackendControlTest.java @@ -0,0 +1,133 @@ +package dev.openfeature.contrib.tools.tck; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.openfeature.sdk.ImmutableContext; +import dev.openfeature.sdk.ProviderEvaluation; +import dev.openfeature.sdk.exceptions.TypeMismatchError; +import dev.openfeature.sdk.providers.memory.InMemoryProvider; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * Guards the two properties of {@link InProcessBackendControl} that the Gherkin cannot assert + * about itself. + * + *

The first is that unsupported operations fail loudly. The whole + * skipped-by-capability design collapses into false confidence if a connection operation quietly + * does nothing, and a scenario that never runs cannot prove that it would have failed. These tests + * call the operations directly. + * + *

The second is that scenario isolation actually isolates. {@link InMemoryProviderTckTest} would + * still pass if {@code changeFlag()} leaked into the next scenario, because no scenario evaluates + * {@code changing-flag} before modifying it. + */ +class InProcessBackendControlTest { + + @Test + @DisplayName("connection operations throw rather than silently doing nothing") + void connectionOperationsThrow() { + InProcessBackendControl control = new InProcessBackendControl(); + + // The message has to name the fix, because whoever hits this is looking at a red scenario + // that reads like a provider defect and is in fact a capability declared in error. + assertThatThrownBy(control::disconnect) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("does not support 'disconnect'") + .hasMessageContaining("test-configuration bug") + .hasMessageContaining("Capability.STALE"); + + assertThatThrownBy(control::reconnect) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("does not support 'reconnect'"); + } + + @Test + @DisplayName("changing a flag without a provider fails instead of being lost") + void changeFlagWithoutProviderThrows() { + InProcessBackendControl control = new InProcessBackendControl(); + control.prepareScenario(); + + assertThatThrownBy(control::changeFlag) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Given a stable provider"); + } + + @Test + @DisplayName("changeFlag changes the resolved value and the next scenario starts from baseline") + void changeFlagIsVisibleAndDoesNotLeak() throws Exception { + InProcessBackendControl control = new InProcessBackendControl(); + + control.prepareScenario(); + InMemoryProvider first = control.createProvider(); + first.initialize(new ImmutableContext()); + assertThat(resolveChangingFlag(first)).isEqualTo("foo"); + + control.changeFlag(); + assertThat(resolveChangingFlag(first)) + .as("changeFlag must actually change what the provider resolves, not merely emit an event") + .isEqualTo("bar"); + + // The next scenario must not inherit that change. The baseline map is shared between every + // provider this control hands out, so a mutation that reached it would leak forwards. + control.prepareScenario(); + InMemoryProvider second = control.createProvider(); + second.initialize(new ImmutableContext()); + assertThat(resolveChangingFlag(second)) + .as("each scenario starts from the canonical baseline") + .isEqualTo("foo"); + } + + @Test + @DisplayName("the canonical flag set omits missing-flag") + void missingFlagIsAbsent() throws Exception { + InProcessBackendControl control = new InProcessBackendControl(); + InMemoryProvider provider = control.createProvider(); + provider.initialize(new ImmutableContext()); + + // Absence is what the FLAG_NOT_FOUND scenario tests, so seeding it by accident would turn + // that scenario green for the wrong reason. + assertThatThrownBy(() -> provider.getStringEvaluation("missing-flag", "fallback", new ImmutableContext())) + .hasMessageContaining("missing-flag"); + } + + @Test + @DisplayName("the canonical flag set keeps the values and types the scenarios depend on") + void canonicalFlagsKeepTheirValuesAndTypes() throws Exception { + InMemoryProvider provider = new InProcessBackendControl().createProvider(); + provider.initialize(new ImmutableContext()); + ImmutableContext context = new ImmutableContext(); + + // Seeded as the integer 10, the lossless-coercion scenario would pass without coercing. + assertThat(provider.getDoubleEvaluation("integral-float-flag", 0.1, context) + .getValue()) + .as("integral-float-flag is a Double") + .isEqualTo(10.0); + // Which is also why the self-tests withhold NUMERIC_COERCION: the SDK's provider keeps the + // two numeric types strictly apart and refuses the lossless direction along with the lossy one. + assertThatThrownBy(() -> provider.getIntegerEvaluation("integral-float-flag", 1, context)) + .isInstanceOf(TypeMismatchError.class); + + assertThat(provider.getIntegerEvaluation("large-integer-flag", 1, context) + .getValue()) + .isEqualTo(2147483647); + + // Values, not absences: each default differs from what the flag resolves to. + assertThat(provider.getBooleanEvaluation("boolean-zero-flag", true, context) + .getValue()) + .isFalse(); + assertThat(provider.getIntegerEvaluation("integer-zero-flag", 1, context) + .getValue()) + .isZero(); + assertThat(provider.getStringEvaluation("string-zero-flag", "fallback", context) + .getValue()) + .isEmpty(); + } + + private static String resolveChangingFlag(InMemoryProvider provider) { + ProviderEvaluation evaluation = + provider.getStringEvaluation("changing-flag", "unset", new ImmutableContext()); + return evaluation.getValue(); + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/MultiProviderTckTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/MultiProviderTckTest.java new file mode 100644 index 0000000000..2f5076ac26 --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/MultiProviderTckTest.java @@ -0,0 +1,115 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.FeatureProvider; +import dev.openfeature.sdk.multiprovider.MultiProvider; +import dev.openfeature.sdk.providers.memory.InMemoryProvider; +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; + +/** + * Runs the OpenFeature Provider TCK against the SDK's {@link MultiProvider}, wrapping a single + * {@link InMemoryProvider}. + * + *

A provider that delegates is still a provider, and delegation is where the contract is easiest + * to drop on the floor: a variant that does not survive the hop, a reason rewritten to + * {@code DEFAULT}, an error code flattened to {@code GENERAL}, an event that never reaches the + * client. Wrapping exactly one child makes every one of those observable, because the correct + * answer is precisely what {@link InMemoryProviderTckTest} already asserts. Any difference between + * these two suites is attributable to {@link MultiProvider} and nothing else. + * + *

That framing is the point of running it here rather than in the SDK: this is not a test of + * aggregation across several backends, it is a test that delegation is transparent. + * + *

It costs one class, needs no Docker, and it has already earned its place — see the capability + * note below. + */ +public class MultiProviderTckTest extends ProviderTckTest { + + private final InProcessBackendControl control = new InProcessBackendControl(); + + @Override + public BackendControl backendControl() { + return control; + } + + /** + * {@inheritDoc} + * + *

Exactly one child. See the class javadoc for why that is the interesting configuration + * rather than a degenerate one. + */ + @Override + public FeatureProvider createProvider() { + return new MultiProvider(Collections.singletonList(control.createProvider())); + } + + /** + * {@inheritDoc} + * + *

{@link Capability#CONFIGURATION_CHANGE} is not declared, and that is a + * finding rather than a configuration choice. + * + *

{@link MultiProvider} extends {@code EventProvider} but never subscribes to its children, + * so a child's {@code PROVIDER_CONFIGURATION_CHANGED} — along with its {@code PROVIDER_ERROR} + * and {@code PROVIDER_STALE} — is swallowed and never reaches the client. Wrapping an + * in-memory provider in a multi-provider therefore silently costs you configuration-change + * events, with nothing in the API to suggest it. + * + *

This is a known gap, tracked as + * open-feature/java-sdk#1882 + * (gap 1, "child provider event aggregation and status tracking", High). The suite reproduced + * it from the outside, which is a reasonable advertisement for what the TCK is for: the gap was + * originally found by hand-comparing implementations against the js-sdk reference. + * + *

Delete this omission once #1882 is fixed. Until then the + * {@code @configuration-change} scenario is reported as skipped-with-reason rather than passing + * on a provider that cannot satisfy it. + * + *

This is the one omission in this module that rests on Appendix F's self-test + * carve-out, and it is worth naming as such. The rule for an adoption is that a + * provider which attempts a behaviour and gets it wrong declares the capability and lets the + * scenario fail; + * Appendix + * F exempts a TCK's own self-tests, because they run the scenarios against an SDK provider + * as a fixture, report on nobody, and run in the ordinary build where a permanently failing + * scenario is a broken build rather than a finding — the fix is an SDK release away. The + * exemption has one condition: the defect is pinned by a test of its own, so that the + * skip is not the only record. That condition is met only partly here. The issue is named above + * and the deletion criterion with it, but nothing in this module asserts the swallowed event + * directly, so a reader has the javadoc and the skip reason and no executing assertion. The + * other two self-test suites do not need the carve-out at all: every capability they omit is a + * property of the provider rather than a defect. + * + *

Everything else holds. Values, variants, reasons, the full type-mismatch matrix, + * {@code FLAG_NOT_FOUND}, falsy values, 32-bit integer precision and structured values all + * survive the delegation hop unchanged — {@link Capability#VARIANTS} is declared for exactly + * that reason, and a variant lost in delegation is one of the likelier ways a facade breaks the + * contract. {@link Capability#DISABLED_FLAGS} is declared on the same evidence and is the more + * interesting of the two: a disabled flag resolves to nothing, so the child hands back the + * caller's default and a facade that substituted a default of its own, or that read the absence + * as an error, would be caught on the value. All four rows pass, so the substitution survives the + * hop exactly as the child performs it. {@link Capability#STANDARD_REASONS} is declared for the + * same kind of reason and answers one of the risks named at the top of this class: a reason + * rewritten in delegation. The child reports the standard vocabulary, and + * {@code reason.feature}'s seven applicable scenarios pass through {@code MultiProvider} + * unchanged, so {@code STATIC}, {@code ERROR} and {@code DISABLED} all survive the hop. {@link + * Capability#LIFECYCLE} and + * {@link Capability#NUMERIC_COERCION} are omitted for + * the same reasons as in {@link InMemoryProviderTckTest}: nothing here reaches a backend during + * initialisation, and the child refuses the lossless coercions the tag now requires — a facade + * cannot declare what its only child does not have. {@link Capability#TARGETING} is omitted for + * the same reason again: the child evaluates no rules, so there is no targeting to delegate. + */ + @Override + public Set capabilities() { + return EnumSet.of( + Capability.EVENTS, + Capability.OBJECT, + Capability.VARIANTS, + Capability.DISABLED_FLAGS, + Capability.STRING_TYPING, + Capability.FULLY_TYPED_VALUES, + Capability.STANDARD_REASONS); + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ReservedTagExpiryTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ReservedTagExpiryTest.java new file mode 100644 index 0000000000..74bfff1d55 --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ReservedTagExpiryTest.java @@ -0,0 +1,106 @@ +package dev.openfeature.contrib.tools.tck; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.platform.engine.discovery.DiscoverySelectors; +import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; +import org.junit.platform.launcher.core.LauncherFactory; +import org.junit.platform.launcher.listeners.SummaryGeneratingListener; +import org.junit.platform.launcher.listeners.TestExecutionSummary; +import org.opentest4j.TestAbortedException; + +/** + * A scenario carrying a {@linkplain Capability#reserved() reserved} tag fails the run. + * + *

This is the expiry check on the reserved list, and it is here because the failure it catches is + * silent in both directions. A reserved capability cannot be declared, so the day the specification + * adds the first scenario for one, every adopter's run reports that scenario as skipped for a + * capability they are not permitted to claim. The report is well-formed and the suite is green, so + * the new scenario is executed by nobody — the unclaimable-capability failure of Appendix F. Nothing + * else in this module notices it: the scenario was collected and gated rather than dropped, and a + * capability-gated skip is explicitly not a gap. + * + *

The first test runs {@link ReservedTagSuiteFixture}, a real suite over two real scenarios, and + * looks at the outcome each one got. Running it rather than calling {@link CapabilityGate} directly + * is what makes it a test of the rule as an adopter meets it: the tags are Cucumber's own parse, the + * hook is the one the suite installs, and the failure has to survive into the JUnit results the same + * way the skip does. + * + *

It is also written so that removing the check does not merely change an error message. + * With the check gone, the tagged scenario is skipped for an undeclared capability instead — one + * fewer failure, one more abort — which is exactly the silent outcome the check exists to prevent, + * and both counts are asserted. + */ +class ReservedTagExpiryTest { + + @Test + @DisplayName("a reserved tag fails its scenario, and a comment naming one does not") + void aReservedTagFailsTheRun() { + SummaryGeneratingListener listener = new SummaryGeneratingListener(); + LauncherFactory.create() + .execute( + LauncherDiscoveryRequestBuilder.request() + .selectors(DiscoverySelectors.selectClass(ReservedTagSuiteFixture.class)) + .build(), + listener); + TestExecutionSummary summary = listener.getSummary(); + + assertThat(summary.getTestsFailedCount()) + .as("the scenario tagged %s fails the run rather than being quietly skipped", Capability.CACHING.tag()) + .isEqualTo(1); + assertThat(summary.getTestsAbortedCount()) + .as("and it is a failure, not an abort — an abort is the skip this check exists to " + + "prevent, and is what remains if the check is removed") + .isZero(); + assertThat(summary.getTestsSkippedCount()).isZero(); + + TestExecutionSummary.Failure failure = summary.getFailures().get(0); + assertThat(failure.getException()) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining(Capability.CACHING.name()) + .hasMessageContaining(Capability.CACHING.tag()); + assertThat(failure.getTestIdentifier().getDisplayName()) + .as("the tagged scenario is the one that failed") + .contains("still calls reserved"); + + assertThat(summary.getTestsSucceededCount()) + .as( + "the untagged scenario passes, though a Gherkin comment in it names %s — " + + "gherkin/events.feature carries exactly such a comment, so a check that " + + "scanned the feature files as text would fail every adoption", + Capability.CACHING.tag()) + .isEqualTo(1); + } + + @Test + @DisplayName("the reserved tag is reported even when another tag on the scenario is undeclared") + void theExpiryIsReportedBeforeTheSkip() { + assertThatThrownBy(() -> CapabilityGate.requireDeclared( + Arrays.asList(Capability.EVENTS.tag(), Capability.CACHING.tag()), + EnumSet.noneOf(Capability.class))) + .as("both tags are undeclared, and the expired reservation is the one worth saying — " + + "gating tag by tag would abort on @events and never reach @caching") + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining(Capability.CACHING.tag()); + } + + @Test + @DisplayName("a tag that gates nothing, and a declared capability, still pass the gate") + void ordinaryTagsAreUnaffected() { + assertThatCode(() -> CapabilityGate.requireDeclared( + Arrays.asList("@some-adopter-tag", Capability.EVENTS.tag()), EnumSet.of(Capability.EVENTS))) + .doesNotThrowAnyException(); + + assertThatThrownBy(() -> CapabilityGate.requireDeclared( + Collections.singletonList(Capability.EVENTS.tag()), EnumSet.noneOf(Capability.class))) + .as("an undeclared capability is still a skip, not a failure") + .isInstanceOf(TestAbortedException.class); + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ReservedTagSuiteFixture.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ReservedTagSuiteFixture.java new file mode 100644 index 0000000000..2188d16e30 --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ReservedTagSuiteFixture.java @@ -0,0 +1,73 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.FeatureProvider; +import io.cucumber.junit.platform.engine.Constants; +import java.util.EnumSet; +import java.util.Set; +import org.junit.platform.suite.api.ConfigurationParameter; +import org.junit.platform.suite.api.IncludeEngines; +import org.junit.platform.suite.api.SelectClasspathResource; +import org.junit.platform.suite.api.Suite; + +/** + * A real TCK suite over {@code reserved-selftest/}, used by {@link ReservedTagExpiryTest}. + * + *

Executed rather than discovered: the rule under test lives in a {@code @Before} hook, so the + * only way to prove it is to run scenarios through it. The suite is otherwise ordinary — the + * canonical glue, the canonical object factory, an {@link InProcessBackendControl} over the SDK's + * in-memory provider — so what runs is the path an adopter's run takes, not a hand-built + * {@code Scenario}. + * + *

It does not extend {@link ProviderTckTest}, and that is the point of writing + * the annotations out. The suite engine collects {@code @SelectClasspathResource} from the whole + * class hierarchy, so a subclass of {@link ProviderTckTest} would select {@code gherkin/} and + * {@code extensions/} as well and run the entire canonical set to observe two scenarios. Here the + * selection is exactly one directory, which is also why that directory is not under + * {@code extensions/}: every other suite in this module selects that one, and this fixture's second + * scenario is meant to fail. + * + *

Deliberately not named {@code *Test}, so Surefire does not find it and run it as a suite of its + * own — which would fail the build, correctly, and for the reason this fixture exists. + */ +@Suite +@IncludeEngines("cucumber") +@SelectClasspathResource(ReservedTagSuiteFixture.FEATURES) +@ConfigurationParameter(key = Constants.PLUGIN_PROPERTY_NAME, value = ProviderTck.PLUGINS) +@ConfigurationParameter( + key = Constants.PARALLEL_EXECUTION_ENABLED_PROPERTY_NAME, + value = ProviderTck.PARALLEL_EXECUTION_ENABLED) +@ConfigurationParameter( + key = Constants.EXECUTION_MODE_FEATURE_PROPERTY_NAME, + value = ProviderTck.FEATURE_EXECUTION_MODE) +@ConfigurationParameter(key = Constants.GLUE_PROPERTY_NAME, value = ProviderTck.ALL_GLUE) +@ConfigurationParameter(key = Constants.OBJECT_FACTORY_PROPERTY_NAME, value = ProviderTck.OBJECT_FACTORY) +public class ReservedTagSuiteFixture implements ProviderTckHarness { + + /** The classpath directory holding this fixture's feature file. */ + public static final String FEATURES = "reserved-selftest"; + + private final InProcessBackendControl control = new InProcessBackendControl(); + + @Override + public BackendControl backendControl() { + return control; + } + + @Override + public FeatureProvider createProvider() { + return control.createProvider(); + } + + /** + * {@inheritDoc} + * + *

Empty, and it has to be: {@code @caching} is reserved, so there is no declaration that + * would let the tagged scenario through. Every other scenario in the fixture is untagged and + * therefore mandatory, so nothing here rests on the declaration at all — which is what makes the + * failure this suite produces attributable to the reserved tag and to nothing else. + */ + @Override + public Set capabilities() { + return EnumSet.noneOf(Capability.class); + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/TckSuiteFixture.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/TckSuiteFixture.java new file mode 100644 index 0000000000..452345a25b --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/TckSuiteFixture.java @@ -0,0 +1,43 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.FeatureProvider; +import dev.openfeature.sdk.NoOpProvider; +import java.io.File; +import java.util.Collections; +import java.util.List; + +/** + * A concrete TCK suite, used by these tests for what the JUnit Platform makes of it. + * + *

Only ever discovered, never executed: discovery needs the annotations on + * {@link ProviderTckTest} and nothing else, so these tests exercise the real suite configuration — + * the real selectors, the real glue, the real engines — without Docker. + * + *

Extends {@link ContainerizedProviderTckTest} rather than {@link ProviderTckTest} directly, so + * that the suite under discovery is shaped like the one an adopter with a real backend writes. + * + *

Deliberately not named {@code *Test}, so Surefire does not find it and try to run it. Running it + * would start a Compose stack that does not exist. + */ +public class TckSuiteFixture extends ContainerizedProviderTckTest { + + @Override + public File composeFile() { + return new File("src/test/resources/there-is-no-stack.yaml"); + } + + @Override + public List backendPorts() { + return Collections.singletonList(8013); + } + + @Override + public FeatureProvider createProvider(BackendEndpoint endpoint) { + return new NoOpProvider(); + } + + @Override + public FeatureProvider createUnavailableProvider() { + return new NoOpProvider(); + } +} diff --git a/tools/tck/src/test/java/openfeature/tck/extensions/ExtensionSelfTestSteps.java b/tools/tck/src/test/java/openfeature/tck/extensions/ExtensionSelfTestSteps.java new file mode 100644 index 0000000000..2c14d36631 --- /dev/null +++ b/tools/tck/src/test/java/openfeature/tck/extensions/ExtensionSelfTestSteps.java @@ -0,0 +1,40 @@ +package openfeature.tck.extensions; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; + +/** + * Step definitions for the TCK's own extension fixture, in the package an adopter would use. + * + *

Test-scoped, so it is not in the released JAR. It exists to demonstrate the claim the extension + * point makes: a step class in {@code openfeature.tck.extensions} is on the suite's glue path + * without an annotation, a runner or a line of configuration written for it. + * + *

The steps deliberately need nothing from {@link + * dev.openfeature.contrib.tools.tck.TckRuntime}. An extension scenario in a real suite runs + * after the canonical {@code @BeforeAll} and has the started backend and its control — but asserting + * that here would make the TCK's own unit tests need Docker, which is a worse trade than proving the + * lifecycle structurally: the extension features are discovered into the same suite and the same + * Cucumber engine descriptor as the canonical ones, which is what a shared {@code @BeforeAll} + * is. + */ +public class ExtensionSelfTestSteps { + + private boolean ran; + + /** A step Cucumber can only find if the extension glue package is on the glue path. */ + @Given("a step defined in the extension glue package") + public void aStepDefinedInTheExtensionGluePackage() { + ran = true; + } + + /** Asserts the step above ran, so that a missing glue package fails rather than passes. */ + @Then("the extension step ran") + public void theExtensionStepRan() { + assertThat(ran) + .as("the extension glue package was resolved and its steps executed") + .isTrue(); + } +} diff --git a/tools/tck/src/test/resources/extensions/extension-selftest.feature b/tools/tck/src/test/resources/extensions/extension-selftest.feature new file mode 100644 index 0000000000..627b76b67d --- /dev/null +++ b/tools/tck/src/test/resources/extensions/extension-selftest.feature @@ -0,0 +1,14 @@ +@extension-selftest +Feature: An adopter's own scenarios run inside the TCK suite + + This file is the TCK's own proof of its extension point. It sits exactly where an adopter's + extension features sit — on the test classpath under extensions/ — and is picked up with no + annotation, no selector and no runner configuration written anywhere for it. + + It is test-scoped, so it is not packaged in the released JAR and cannot reach an adopter's run. + It carries no canonical scenario and cannot stand in for one: the canonical set is what gherkin/ + contains, and this file is not in it. + + Scenario: A step class in the adopter's own glue package is on the suite's glue path + Given a step defined in the extension glue package + Then the extension step ran diff --git a/tools/tck/src/test/resources/reserved-selftest/reserved-tag.feature b/tools/tck/src/test/resources/reserved-selftest/reserved-tag.feature new file mode 100644 index 0000000000..256b1668ad --- /dev/null +++ b/tools/tck/src/test/resources/reserved-selftest/reserved-tag.feature @@ -0,0 +1,20 @@ +Feature: A reserved capability tag on a scenario fails the run + + This file is the TCK's own proof of the expiry check on Capability.reserved(). It is driven by + ReservedTagExpiryTest through ReservedTagSuiteFixture, which selects this directory and nothing + else, so the two scenarios below run inside a real suite — real parse, real @Before hook, real + CapabilityGate — without affecting any other suite in this module. + + It lives outside extensions/ on purpose. A feature file under extensions/ is selected by every + suite this module runs, and the second scenario here is meant to fail. + + Scenario: A Gherkin comment naming a reserved tag is prose, not a tag + # This scenario is untagged. The line you are reading mentions @caching, exactly as + # gherkin/events.feature does where it explains which stale-provider behaviour is deliberately + # not covered yet. A check that scanned feature files as text rather than reading the parsed + # tags would fail this scenario, and would therefore fail every adoption on the day it shipped. + Given a stable provider + + @caching + Scenario: A tag this implementation still calls reserved fails the run + Given a stable provider diff --git a/tools/tck/version.txt b/tools/tck/version.txt new file mode 100644 index 0000000000..6e8bf73aa5 --- /dev/null +++ b/tools/tck/version.txt @@ -0,0 +1 @@ +0.1.0