diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8c09e514..f79c4405 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,6 +16,13 @@ on: - reopened branches: - main + # Temporary, for the duration of the provider conformance suite's review. + # Without it a stacked pull request gets no CI at all: this filter matches + # the pull request's BASE branch, so only the suite PR itself -- the one + # targeting main -- was ever checked, and the report and adoption PRs + # stacked on it were merged-in-theory and tested never. Remove once the + # chain has landed. See open-feature/spec#417. + - 'feat/provider-tck*' permissions: contents: read @@ -66,11 +73,20 @@ jobs: tools/openfeature-flagd-api-testkit: - 'tools/openfeature-flagd-api-testkit/**' - 'uv.lock' + tools/openfeature-tck: + - 'tools/openfeature-tck/**' + - 'uv.lock' build: needs: changes runs-on: ubuntu-latest strategy: + # A matrix of every supported Python against every changed package exists + # to say *which* combinations fail. The default `fail-fast: true` discards + # exactly that: the first failing cell cancels every other one, so a run + # that could have reported thirty results reports one, and an unrelated + # package's failure is indistinguishable from your own. + fail-fast: false matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] package: ${{ fromJSON(needs.changes.outputs.packages) }} diff --git a/.gitmodules b/.gitmodules index 7e8bf9ed..637a63b1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "providers/openfeature-provider-flagd/openfeature/test-harness"] path = providers/openfeature-provider-flagd/openfeature/test-harness url = https://github.com/open-feature/flagd-testbed.git +[submodule "tools/openfeature-tck/spec"] + path = tools/openfeature-tck/spec + url = https://github.com/open-feature/spec diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 94d58394..2c0bdfd1 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -8,5 +8,6 @@ "providers/openfeature-provider-unleash": "0.1.2", "tools/openfeature-flagd-api": "1.0.0", "tools/openfeature-flagd-core": "1.0.0", - "tools/openfeature-flagd-api-testkit": "0.1.0" + "tools/openfeature-flagd-api-testkit": "0.1.0", + "tools/openfeature-tck": "0.1.0" } diff --git a/pyproject.toml b/pyproject.toml index 647571bc..244df631 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "openfeature-flagd-api", "openfeature-flagd-core", "openfeature-flagd-api-testkit", + "openfeature-tck", ] [dependency-groups] @@ -43,6 +44,7 @@ openfeature-provider-unleash = { workspace = true } openfeature-flagd-api = { workspace = true } openfeature-flagd-core = { workspace = true } openfeature-flagd-api-testkit = { workspace = true } +openfeature-tck = { workspace = true } [tool.uv.workspace] members = [ @@ -58,7 +60,10 @@ exclude = [ ".venv", "__pycache__", "venv", - "providers/openfeature-provider-flagd/src/openfeature/schemas/**" + "providers/openfeature-provider-flagd/src/openfeature/schemas/**", + # Submodules of other repositories: not ours to lint or format. + "providers/openfeature-provider-flagd/openfeature/spec/**", + "tools/openfeature-tck/spec/**", ] [tool.ruff.lint] diff --git a/release-please-config.json b/release-please-config.json index a6335415..f890331f 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -99,6 +99,15 @@ "extra-files": [ "README.md" ] + }, + "tools/openfeature-tck": { + "package-name": "openfeature-tck", + "bump-minor-pre-major": true, + "bump-patch-for-minor-pre-major": true, + "versioning": "default", + "extra-files": [ + "README.md" + ] } }, "changelog-sections": [ diff --git a/tools/openfeature-tck/.gitignore b/tools/openfeature-tck/.gitignore new file mode 100644 index 00000000..41520f2e --- /dev/null +++ b/tools/openfeature-tck/.gitignore @@ -0,0 +1,7 @@ +# Copied from the open-feature/spec submodule by hatch_build_sync.py. +# DO NOT EDIT the copies, and do not commit them: the canonical definitions live +# in spec/specification/assets/provider-tck/, and the revision this package is +# built against is recorded by the submodule pin. +src/openfeature/contrib/tools/tck/gherkin/ +src/openfeature/contrib/tools/tck/flag_data/ +src/openfeature/contrib/tools/tck/control-api.yaml diff --git a/tools/openfeature-tck/LICENSE b/tools/openfeature-tck/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/tools/openfeature-tck/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/openfeature-tck/README.md b/tools/openfeature-tck/README.md new file mode 100644 index 00000000..589370d3 --- /dev/null +++ b/tools/openfeature-tck/README.md @@ -0,0 +1,336 @@ +# OpenFeature TCK (Python) + +A conformance suite any OpenFeature Python provider can adopt to check that it implements the +provider contract of the specification. + +It is the Python implementation of [Appendix F][appendix-f], which defines the Gherkin scenarios, the +canonical flag set and the control API every language's TCK runs, and carries the reasoning behind +all of it. **This README documents the Python binding and nothing else**; where a question is not +Python's, it links there. + +Tracking issue: [open-feature/spec#417][tracking]. **Status: proof of concept**, so expect breaking +changes. The one known gap that is this package's rather than the suite's: no multi-provider suite, +because the Python SDK has no multi-provider. + +## Quick start + +Two fixtures and one call. + +```bash +pip install 'openfeature-tck[compose]' +``` + +```python +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.tck import ( + Capability, + ComposeBackend, + RunningBackend, + TckConfig, + feature_paths, +) + + +@pytest.fixture(scope="session") +def compose_backend(): + return ComposeBackend( + compose_file="tests/tck/docker-compose.yaml", + backend_ports=[8013], + ) + + +@pytest.fixture(scope="session") +def tck_config(tck_backend: RunningBackend): + return TckConfig( + name="my-provider", + control=tck_backend.control, + new_provider=lambda: MyProvider( + host=tck_backend.endpoint.host, + port=tck_backend.endpoint.port(8013), + ), + capabilities={Capability.EVENTS, Capability.OBJECT}, + ) + + +scenarios(*feature_paths()) +``` + +That is the whole adoption. **There is no `conftest.py` to write and nothing to import for the +steps** — they arrive through this package's pytest plugin, registered via a `pytest11` entry point — +and the assets ship in the distribution, so **adopting needs no git submodule**. The runner is +**pytest-bdd**, which the flagd provider and testkit already use, so an adopting package gains no new +test framework, and one test is generated per scenario and per Scenario Outline row. + +The suite owns the container stack and the provider lifecycle: starting Compose, discovering the +mapped host ports, building the HTTP control, registering the provider, awaiting events, resetting +the backend between scenarios, tearing down. **If you find yourself writing test infrastructure, +that is a defect here rather than something for you to work around.** + +### A provider with no backend + +An in-memory, environment-variable or file-based provider supplies a `BackendControl` of its own, +where flag operations are direct manipulations of the provider's state. No Compose file and no +container tooling: `testcontainers` is the optional `compose` extra, imported lazily. + +```python +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="my-provider", + control=control, + new_provider=control.new_provider, + capabilities={Capability.EVENTS, Capability.OBJECT}, + ) +``` + +It is a narrow allowance — **a provider with an external backend must drive it over the HTTP control +API**, for which `HttpControl` is the client, built on `urllib.request` alone so the TCK gains no +HTTP dependency; `control_api_spec()` returns the document a backend under test implements. Every +control states which path it took: `control_api` is a **required** member of `BackendControl`, typed +`Literal["http", "in-process"]`, with no default and no inference from the concrete type. A +backend-less one does not implement `ConnectionControl`, so `STALE` and `UNAVAILABLE_INIT` go +undeclared and their scenarios skip. + +## The options + +### `TckConfig` + +| field | required | default | meaning | +| --- | --- | --- | --- | +| `name` | yes | — | the provider's name, as it appears in a report | +| `control` | yes | — | the `BackendControl` the scenarios drive the backend through | +| `new_provider` | yes | — | builds the provider under test, configured but uninitialised. Called **once per scenario**, because the mapped ports do not exist until the stack is up | +| `new_unavailable_provider` | no | `None` | builds a provider pointed at a closed port, for the initialisation-failure scenarios. Needed only if `capabilities` includes `UNAVAILABLE_INIT`; give it a short connection deadline, since the scenario allows a bounded time for the error | +| `capabilities` | no | `DECLARABLE_CAPABILITIES` | which optional parts of the contract this provider supports | +| `known_deviations` | no | `()` | gaps the provider is known to have | +| `event_timeout` | no | `12.0` | seconds to wait for a provider event | +| `ready_timeout` | no | `30.0` | seconds to wait for `READY`, and the longest a direct `shutdown`/`initialize` is given before the wait is recorded as a failure | + +`event_timeout` is the knob that matters: set it comfortably above your provider's worst-case +detection latency — a poller may need most of a poll interval — or the suite reports timeouts that +are really just impatience. + +### `ComposeBackend` + +| field | required | default | meaning | +| --- | --- | --- | --- | +| `compose_file` | yes | — | path to the Compose file, resolved relative to the package directory | +| `backend_ports` | yes | — | container-internal ports the **provider** connects to. The control port is exposed automatically and must not be listed here | +| `backend_service` | no | `"backend"` | the Compose service hosting both the control API and the backend | +| `control_port` | no | `8080` | container-internal port of the control API | +| `additional_ports` | no | `{}` | extra service to ports, for a stack with more than one service. Resolved through the endpoint by service name | +| `backend_configuration` | no | `"default"` | the configuration name passed to `POST /start` | +| `startup_timeout` | no | `60.0` | seconds to wait for the stack and its control API to become reachable | + +These names and defaults are fixed across all four languages' TCKs, so a provider shipped in two of +them writes one Compose file and two declarations against it. + +`tck_backend` is a session-scoped fixture the plugin builds from your `compose_backend` fixture, +yielding `.control` — the `HttpControl`, already awaited ready, and one per stack, since it remembers +whether a disconnect left the backend down — and `.endpoint`, with `host`, `port(internal)` and +`port(internal, service=...)`. Your Compose file must **not pin host ports**; declaring +`backend_ports` is what lets the harness say "the Compose file does not publish 8013" at startup +rather than three scenarios later. For a different fixture name or scope: + +```python +@pytest.fixture(scope="session") +def tck_backend(): + yield from run_compose_backend(ComposeBackend(...)) +``` + +## Declaring capabilities + +Each scenario exercising an optional part of the contract carries a Gherkin tag, pytest-bdd turns it +into a marker, and a provider declares what it supports. **An undeclared capability's scenarios are +reported as skipped with the reason — never as passed:** + +``` +SKIPPED provider does not declare capability @stale. + Declared: @events @large-integers @object @variants +``` + +| Capability | Tag | Meaning | +| --- | --- | --- | +| `Capability.LIFECYCLE` | `@lifecycle` | reaches its backend during initialisation, observably | +| `Capability.EVENTS` | `@events` | emits lifecycle events at all | +| `Capability.STALE` | `@stale` | enters `STALE` and emits `PROVIDER_STALE` on backend loss | +| `Capability.CONFIGURATION_CHANGE` | `@configuration-change` | emits `PROVIDER_CONFIGURATION_CHANGED` on a configuration change | +| `Capability.OBJECT` | `@object` | supports structured flag values | +| `Capability.VARIANTS` | `@variants` | names the variant it resolved | +| `Capability.DISABLED_FLAGS` | `@disabled-flags` | resolves a disabled flag to the code default | +| `Capability.UNAVAILABLE_INIT` | `@unavailable` | errors rather than hangs against a dead backend | +| `Capability.NUMERIC_COERCION` | `@numeric-coercion` | coerces int/float only when lossless, else `TYPE_MISMATCH` | +| `Capability.LARGE_INTEGERS` | `@large-integers` | resolves integers up to 2^53 − 1 exactly | +| `Capability.REINITIALIZATION` | `@reinitialization` | can be initialised again after `shutdown` | +| `Capability.TARGETING` | `@targeting` | resolves differently for a matching evaluation context | +| `Capability.STANDARD_REASONS` | `@standard-reasons` | uses the standard resolution reasons with their standard meanings | +| `Capability.CACHING` | `@caching` | reserved; **not declarable** — no scenarios yet | + +Untagged scenarios are mandatory and always run. `capabilities` defaults to +`DECLARABLE_CAPABILITIES`; narrow it rather than widen it. Tags compose, so declaring +`@reinitialization` without the `@lifecycle` its feature carries leaves that scenario skipped. +**What counts as "cannot do" is [Appendix F][appendix-f]'s rules for declaring**; the adoptions here +cite them rather than restating them, and so should yours. + +`TckConfig` refuses two declarations at construction rather than letting them reach a report: a +**reserved** capability, which no scenario carries, and one this language's SDK **cannot express** — +the two errors and the two skip reasons deliberately differ, and a scenario arriving with a reserved +tag fails the run. `INEXPRESSIBLE_CAPABILITIES` is **empty in Python**, measured rather than assumed: +`int` is arbitrary-precision and the integer and float accessors reach separate provider methods, so +both tags are ordinary declarable capabilities here. + +### Known deviations + +`known_deviations` says what a capability set cannot: that the provider fails something it is +**required** to do. It is a declaration, not a switch — the scenario still runs and still fails. + +```python +TckConfig( + # ... + known_deviations=( + KnownDeviation.tracked( + summary="what is wrong, for someone comparing providers", + issue="https://github.com/open-feature/flagd/issues/1996", + capability=Capability.NUMERIC_COERCION, + ), + ), +) +``` + +`summary` is required; `issue` is optional, with `KnownDeviation.untracked(summary=...)` for a gap +tracked nowhere yet; `capability` is optional, left out for a mandatory ungated scenario, and a +reserved one is refused since no scenario carries the tag. [Appendix F][appendix-f] settles which of +the two shapes — declare and let it fail, or withhold and let it skip — to reach for. + +## Running it + +**Keep the adoption suite out of the default build, give it a task of its own, and write down that +you did** — the reasoning is Appendix F's ["Running the suite in CI"][appendix-f]. The Python part is +these tasks, which CI reaches through `poe cov`: + +```toml +[tool.poe.tasks] +test = { sequence = ["test-default", "test-tck-collect"], ignore_fail = "return_non_zero" } +test-cov = { sequence = ["test-cov-default", "test-tck-collect"], ignore_fail = "return_non_zero" } +test-default = "pytest tests --ignore=tests/tck" +test-cov-default = "coverage run -m pytest tests --ignore=tests/tck" +test-tck = "pytest tests/tck" +test-tck-collect = "pytest tests/tck --collect-only -q" +``` + +Put a comment above them saying why, so the exclusion cannot read as an oversight, and record the +current tally where a reviewer will see it — **read the tally rather than the green check**, since a +conformance suite carries failures by design. Both adoptions here do exactly that. + +Two Python-specific notes on top of the appendix: + +- **Docker is not what decides it here.** `tests/e2e` needs Docker too, has needed it for years, and + still runs in the default build on `ubuntu-latest`. The exclusion rests entirely on the second + reason, that a conformance suite's honest output is red. +- **`--ignore` does not import the suite, so nothing checks that it still would**, and `mypy` here is + configured over `src` alone. Hence `test-tck-collect` in both default tasks: it imports every test + module and resolves the feature files while starting no container, which is the appendix's "keep it + compiling" in the form Python has available. It cannot pass vacuously — pytest exits 5 on a + directory that collects nothing and 4 on a path that does not exist, so a suite that moved out from + under the task fails it rather than skipping it. `ignore_fail = "return_non_zero"` is what keeps the + check reachable: poe otherwise aborts the sequence at the first failing subtask, and a compile check + that only runs while the rest of the build is green is not a check. + +## Extending it + +Provider behaviour the specification does not describe — flagd's `fractional` targeting, a vendor's +own rollout rule — belongs in the same run rather than a second harness. Create an `extensions` +directory beside the module that calls `scenarios()`, with step definitions in a `conftest.py`: + +``` +tests/tck/ +├── conftest.py # your step definitions +├── test_my_provider.py # the fixtures and the one call, unchanged +└── extensions/ + └── fractional.feature +``` + +The directory is `tests/tck` because that is what the tasks above exclude and what `poe test-tck` +runs; the module inside it needs no `conformance` or `tck` in its name, since the directory is what +selects the suite. Both adoptions in this repository are laid out that way. + +```python +# conftest.py +from pytest_bdd import then + +from openfeature.contrib.tools.tck import TckState + + +@then("the fractional rule splits the population") +def fractional_splits(tck_state: TckState) -> None: ... +``` + +**No registration, no option and no new argument.** pytest collects `conftest.py` and pytest-bdd +resolves steps through the fixture system, so the canonical vocabulary is in scope beside your own, +and `tck_state` is the per-scenario state those steps use — your scenario runs against the provider +the suite registered, in the same lifecycle and reset. `feature_paths()` already includes an +`extensions` directory when there is one, so the `scenarios()` call never changes. + +**Your scenarios cannot stand in for ours.** Canonical features are identified by their path under +`gherkin/` and yours under `extensions/`, and `extensions.py` refuses a file of yours under the +reserved prefix, or two that would share one uri. Both are reachable here, because pytest-bdd names a +feature file by its parent directory joined to its own name — so `extensions/gherkin/errors.feature` +would land on the canonical `errors.feature`'s uri, and a run could go green having asked the +adopter's questions instead of the specification's. + +## Python-specific notes + +**A boolean satisfies an Integer request**, because the client type-checks with +`isinstance(value, int)` and `bool` subclasses `int`. `boolean-flag` through `get_integer_details` +returns `True` with no error code where the specification requires the code default and +`TYPE_MISMATCH`, and the identical scenario passes in every other language's suite. Expect it in your +own run until a release carries [python-sdk#619](https://github.com/open-feature/python-sdk/issues/619); +the same cause reaches flagd-core's type check +([python-sdk-contrib#417](https://github.com/open-feature/python-sdk-contrib/issues/417)). + +**Three steps reach the provider directly rather than through a client**, because going through the +SDK would test the registry's bookkeeping as much as the provider: the shutdown and re-initialise +steps call the registered instance's own `shutdown()` and `initialize()`, and the metadata step asks +it for `get_metadata()`. The registry is not told, so the SDK shuts the provider down once more when +the scenario ends — requirement 2.5.3 makes that harmless — and a direct call outlasting +`ready_timeout` fails its scenario rather than hanging the session. + +**The SDK's `InMemoryProvider` is why this package ships `ControllableInMemoryProvider`** and why the +self-tests declare less than they otherwise would: it cannot update its flag set, which +[Appendix A][appendix-a] requires +([python-sdk#620](https://github.com/open-feature/python-sdk/issues/620)), never reads +`InMemoryFlag.state` ([python-sdk#627](https://github.com/open-feature/python-sdk/issues/627)), and +hands each variant back untouched, so it does not attempt numeric coercion at all — a permitted +choice rather than a defect, and the reason `@numeric-coercion` is simply not declared there. + +## Contributing + +The Gherkin, the canonical flag set and the control-API document are **not owned by this +repository** — they are the artifacts in [open-feature/spec][spec], copied into the wheel and the +sdist at build time. So adopting needs no submodule and contributing does: + +```bash +git submodule update --init tools/openfeature-tck/spec +poe test # syncs the assets first; 221 passed, 42 skipped, 2 xfailed, no Docker +``` + +The copies under `src/` are gitignored, generated and carry a `DO-NOT-EDIT.txt`: a change goes to +[open-feature/spec][spec] first, then the submodule pin moves here, and that pin is the only record +of the revision this package targets. **The checkout is part of the sync, not something you +remember** — a rebase moves the gitlink and not the submodule's working tree, which is how an +adoption suite in another language ran a whole pass against the previous pin's feature files and +reported numbers identical to the run before it. So the sync checks the submodule out at the pinned +revision before copying and `poe test` depends on it: **the suite cannot run without a fresh sync, +and a sync cannot succeed against any revision but the pinned one.** +`OPENFEATURE_TCK_SPEC_UNPINNED=1` opts out while drafting a change to the assets; where the pin +cannot be read at all the sync warns and continues with the guarantee off, which covers an unpacked +sdist and a worktree whose `.git` points outside the running process's filesystem namespace, such as +a Windows worktree driven from WSL. + +[appendix-a]: https://github.com/open-feature/spec/blob/main/specification/appendix-a-included-utilities.md +[appendix-f]: https://github.com/open-feature/spec/blob/main/specification/appendix-f-provider-conformance.md +[spec]: https://github.com/open-feature/spec +[tracking]: https://github.com/open-feature/spec/issues/417 diff --git a/tools/openfeature-tck/hatch_build.py b/tools/openfeature-tck/hatch_build.py new file mode 100644 index 00000000..5bff5a40 --- /dev/null +++ b/tools/openfeature-tck/hatch_build.py @@ -0,0 +1,51 @@ +"""Hatch build hook to copy the canonical conformance assets into the package. + +The feature files, the canonical flag set and the control-API document are owned +by open-feature/spec and reach this package through a git submodule, so nothing +in this repository can fork the definition of conformance. They are copied into +the source tree at build time and force-included into the distribution, which is +what lets an *adopter* install the wheel and run the suite with no submodule of +their own. +""" + +import sys +from pathlib import Path + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface + +# Hatchling loads this file by path rather than importing it as part of a +# package, so its directory is not on sys.path and the sibling sync module -- +# the single definition of what gets copied where -- would not be importable. +sys.path.insert(0, str(Path(__file__).parent)) + +from hatch_build_sync import FILES, PACKAGE_REL, SPEC_ASSETS, TREES, sync + + +class SpecAssetsCopyHook(BuildHookInterface): + PLUGIN_NAME = "spec-assets-copy" + + def initialize(self, version: str, build_data: dict) -> None: + root = Path(self.root) + copies = [root / PACKAGE_REL / dest for _, dest in TREES + FILES] + + # Building from a checkout: refresh from the submodule, so what ships is + # always the revision the pin names. Building from an sdist: there is no + # submodule, but the copies are already in the tree. + if SPEC_ASSETS.exists(): + sync() + elif not all(path.exists() for path in copies): + missing = ", ".join(str(p) for p in copies if not p.exists()) + msg = ( + f"Conformance assets missing ({missing}) and the open-feature/spec " + f"submodule is not checked out at {SPEC_ASSETS}. Run " + "`git submodule update --init tools/openfeature-tck/spec`." + ) + raise FileNotFoundError(msg) + + # Force-include the gitignored copies into both sdist and wheel. + force = build_data.setdefault("force_include", {}) + for path in copies: + for member in [path] if path.is_file() else path.rglob("*"): + if member.is_file(): + rel = str(member.relative_to(root)) + force[rel] = rel diff --git a/tools/openfeature-tck/hatch_build_sync.py b/tools/openfeature-tck/hatch_build_sync.py new file mode 100644 index 00000000..5c7c9343 --- /dev/null +++ b/tools/openfeature-tck/hatch_build_sync.py @@ -0,0 +1,210 @@ +"""Copy the canonical conformance assets from the spec submodule into the package. + +Used by `poe sync-spec-assets` for local development and CI testing. The hatch +build hook (hatch_build.py) handles inclusion in the wheel and sdist. + +The assets are owned by open-feature/spec, not by this repository. Copying them +in at build time -- rather than committing copies -- means the spec revision this +package was built against is recorded by the submodule pin and nowhere else, so +the two cannot drift apart unnoticed. An *adopter* installing the wheel still +needs no submodule: the copies are inside the distribution. + +Copying is the second half of the job. The first is making sure the submodule +working tree is at the revision the pin names, because **a rebase moves the +gitlink and not the working tree** -- see `checkout_pinned_spec`. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import warnings +from collections.abc import Callable, Sequence +from pathlib import Path + +ROOT = Path(__file__).parent +SPEC_DIRNAME = "spec" +SPEC_ROOT = (ROOT / SPEC_DIRNAME).resolve() +SPEC_ASSETS = (SPEC_ROOT / "specification/assets/provider-tck").resolve() +PACKAGE_REL = Path("src/openfeature/contrib/tools/tck") +DEST_BASE = ROOT / PACKAGE_REL + +UNPINNED_ENV = "OPENFEATURE_TCK_SPEC_UNPINNED" +"""Set to run against whatever is checked out in the submodule, pin or no pin. + +For the one workflow that legitimately wants it: drafting a change to the +canonical assets in a local spec checkout before there is a revision to pin. It +warns on every sync, and it names the revision actually in use, because a suite +running against assets nobody can identify must not be quiet about it. +""" + +DO_NOT_EDIT = ( + "Generated by hatch_build_sync.py from the open-feature/spec submodule.\n" + "DO NOT EDIT. Changes belong in open-feature/spec under\n" + "specification/assets/provider-tck/, then bump the submodule pin.\n" +) + +# (source directory or file, destination) relative to SPEC_ASSETS / DEST_BASE. +# +# "gherkin" copies to a directory of the same name on purpose, and the pair is not +# redundant: a canonical feature is identified by its path relative to the asset +# directory, so the destination name *is* the reported uri prefix. Renaming it +# locally -- it used to land in "features" -- silently renamed the uri, which is +# how this suite reported features/errors.feature for the file Go reports as +# gherkin/errors.feature. Keep the two equal. +TREES = [("gherkin", "gherkin"), ("flags", "flag_data")] +FILES = [("openapi/control-api.yaml", "control-api.yaml")] + +GitRunner = Callable[[Sequence[str], Path], str | None] +"""Runs a git command in a directory and returns its output, or ``None`` if it failed. + +A seam, so the decision `checkout_pinned_spec` makes can be tested without a +fixture repository on disk. Failure is ``None`` rather than an exception because +every caller here treats "git could not answer" as a state to report rather than +a crash: building from an unpacked sdist has no repository at all. +""" + + +def _run_git(args: Sequence[str], cwd: Path) -> str | None: + try: + completed = subprocess.run( # noqa: S603 + ["git", *args], # noqa: S607 + cwd=cwd, + capture_output=True, + check=True, + text=True, + ) + except (OSError, subprocess.CalledProcessError): + return None + return completed.stdout.strip() + + +def pinned_revision(git: GitRunner = _run_git) -> str | None: + """The spec revision the superproject's index records for the submodule. + + The index rather than ``HEAD``, because the index is what the next commit + will record and so what a run is about to claim it tested against. ``None`` + when there is no repository to ask, which is ordinary: an sdist has none. + """ + entry = git(["ls-files", "-s", "--", SPEC_DIRNAME], ROOT) + if not entry: + return None + fields = entry.split() + # "160000 0spec". Anything else is not a gitlink -- a plain + # directory checked in under that name, most likely -- and there is no pin. + if len(fields) < 2 or fields[0] != "160000": + return None + return fields[1] + + +def checkout_pinned_spec(git: GitRunner = _run_git) -> str | None: + """Bring the submodule working tree to the revision the pin names. + + **This is the reason this function exists: a rebase moves the gitlink and + not the submodule's working tree.** So a checkout can have a pin saying one + revision and assets on disk from another, and nothing about the build says + so. That is not hypothetical -- it happened here, the copied Gherkin was one + pin behind the capability the suite was declaring, and a self-test of the + enum against the assets is what caught it. That guard fires for one symptom. + A pin that changes only the *content* of a scenario would pass every guard + in this package and still run the wrong suite, which is what happened in + another language: a whole adoption suite ran against stale assets and + reported byte-identical numbers to the previous pass, and nothing failed and + nothing warned. + + So the checkout is part of the copy rather than something the operator is + expected to remember, and the copy is a dependency of the test task. The + suite cannot run against assets it did not just fetch. + + Returns the pinned revision, or ``None`` when there is no pin to read. + + **When the pin cannot be read this warns rather than failing.** Building + from an unpacked sdist is the ordinary case -- no repository, no pin, and + the assets are already in the tree. The other case is a git checkout whose + superproject this process cannot reach, which happens for a linked worktree + whose ``.git`` file names a path in another filesystem namespace: git inside + the submodule answers, git in the superproject does not. Warning rather than + failing keeps that environment usable; the warning is what says the + guarantee is not in force, and the remedy is to run the sync from a shell + that can see the superproject. + """ + if os.environ.get(UNPINNED_ENV): + head = git(["rev-parse", "HEAD"], SPEC_ROOT) + warnings.warn( + f"{UNPINNED_ENV} is set: syncing the conformance assets from whatever " + f"is checked out in {SPEC_ROOT} ({head or 'unknown revision'}) rather " + f"than from the revision the submodule pin names. Nothing records " + f"which questions this run asked", + stacklevel=2, + ) + return None + + pinned = pinned_revision(git) + if pinned is None: + if SPEC_ROOT.exists(): + warnings.warn( + f"could not read the submodule pin for {SPEC_DIRNAME} from a git " + f"index in {ROOT}, so the conformance assets are being copied from " + f"the submodule working tree unverified. If this is a checkout " + f"rather than an unpacked sdist, a rebase may have moved the pin " + f"without moving that working tree: run `git submodule update " + f"--init tools/openfeature-tck/spec` from a shell that can see the " + f"superproject, then sync again", + stacklevel=2, + ) + return None + + head = git(["rev-parse", "HEAD"], SPEC_ROOT) + if head == pinned: + return pinned + + print( # noqa: T201 + f"spec submodule is at {head or '(not checked out)'}, the pin names " + f"{pinned}: checking out the pinned revision" + ) + git(["submodule", "update", "--init", "--", SPEC_DIRNAME], ROOT) + + head = git(["rev-parse", "HEAD"], SPEC_ROOT) + if head != pinned: + msg = ( + f"the {SPEC_DIRNAME} submodule is at {head or '(not checked out)'} and " + f"the pin names {pinned}, and `git submodule update --init` did not " + f"move it. The assets would be from the wrong revision, so nothing is " + f"copied. Most likely the pinned commit is not in the local object " + f"store and could not be fetched; check network access to " + f"open-feature/spec, or fetch it by hand. To copy from the working " + f"tree anyway -- for drafting a spec change that has no revision yet " + f"-- put {UNPINNED_ENV}=1 in the environment." + ) + raise RuntimeError(msg) + return pinned + + +def sync() -> None: + checkout_pinned_spec() + + if not SPEC_ASSETS.exists(): + msg = ( + f"Conformance assets not found at {SPEC_ASSETS}. " + "Make sure submodules are initialized: " + "`git submodule update --init tools/openfeature-tck/spec`." + ) + raise FileNotFoundError(msg) + + for src_name, dest_name in TREES: + dest = DEST_BASE / dest_name + if dest.exists(): + shutil.rmtree(dest) + shutil.copytree(SPEC_ASSETS / src_name, dest) + (dest / "DO-NOT-EDIT.txt").write_text(DO_NOT_EDIT, encoding="utf-8") + + for src_name, dest_name in FILES: + dest = DEST_BASE / dest_name + if dest.exists(): + dest.unlink() + shutil.copy2(SPEC_ASSETS / src_name, dest) + + +if __name__ == "__main__": + sync() diff --git a/tools/openfeature-tck/pyproject.toml b/tools/openfeature-tck/pyproject.toml new file mode 100644 index 00000000..2cb6f12f --- /dev/null +++ b/tools/openfeature-tck/pyproject.toml @@ -0,0 +1,116 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "openfeature-tck" +version = "0.1.0" +description = "OpenFeature provider conformance suite (TCK)" +readme = "README.md" +authors = [{ name = "OpenFeature", email = "openfeature-core@groups.io" }] +license = { file = "LICENSE" } +classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Framework :: Pytest", +] +keywords = ["openfeature", "conformance", "tck", "feature-flags"] +dependencies = [ + "openfeature-sdk>=0.10.0", + "pytest>=8.4.0", + # Same runner the flagd provider and the flagd testkit already use, so an + # adopting module gains no new test framework. + "pytest-bdd>=8.1.0,<9.0.0", +] +requires-python = ">=3.10" + +# Kept out of the required dependencies on purpose: a provider with no backend +# runs the whole suite without a container, and should not have to install +# container tooling to do it. Everything in `compose.py` imports it lazily. +[project.optional-dependencies] +compose = ["testcontainers>=4.12.0,<5.0.0"] + +[project.urls] +Homepage = "https://github.com/open-feature/python-sdk-contrib" + +# Shipping the step definitions as a pytest plugin is what keeps adoption to a +# single fixture: pytest-bdd resolves steps through the fixture system, and +# fixtures from an installed plugin are visible to every test, so an adopter +# never has to `from ... import *` to pull the vocabulary in. +[project.entry-points.pytest11] +openfeature_tck = "openfeature.contrib.tools.tck.plugin" + +[dependency-groups] +dev = [ + "coverage[toml]>=7.10.0,<8.0.0", + "mypy>=1.18.0,<2.0.0", + "poethepoet>=0.37.0", + # The `compose` extra, so the lazy import in compose.py is type-checked + # against the real DockerCompose rather than waved through. Nothing in this + # package's own tests starts a container -- the flagd adoption is what proves + # the harness end to end. + "testcontainers>=4.12.0,<5.0.0", +] + +[tool.hatch.build.targets.sdist] +# The conformance assets are gitignored copies of the spec submodule; the build +# hook force-includes them so an sdist builds into a wheel without a submodule. +force-include = {} +# Which is why the submodule itself has no business in the sdist: it is the whole +# spec repository, and only the copies of the four assets are needed downstream. +exclude = ["/spec"] + +[tool.hatch.build.targets.wheel] +packages = ["src/openfeature"] +# Ship the conformance assets even though they are gitignored: an adopter +# installing this package must need no submodule of their own. +artifacts = [ + "src/openfeature/contrib/tools/tck/gherkin/", + "src/openfeature/contrib/tools/tck/flag_data/", + "src/openfeature/contrib/tools/tck/control-api.yaml", +] + +[tool.hatch.build.hooks.custom] + +# `hatch_build_sync.py` sits beside this file rather than under src/, because it +# is build machinery rather than package content. It is on the path anyway, so +# the tests that hold its submodule-pin check to its promise can import it the +# way the build hook does. +[tool.pytest.ini_options] +pythonpath = ["."] + +[tool.mypy] +mypy_path = ["src", "."] +files = ["src", "tests", "hatch_build_sync.py"] +python_version = "3.10" +namespace_packages = true +explicit_package_bases = true +local_partial_types = true +allow_redefinition_new = true +fixed_format_cache = true +pretty = true +strict = true +disallow_any_generics = false + +# testcontainers 4.14 ships no py.typed for `testcontainers.compose`, so the one +# lazy import in compose.py cannot be checked against it. Scoped to that module +# rather than made a global `ignore_missing_imports`, and it costs nothing here: +# the harness talks to a stack through the `ComposeStack` protocol, which is what +# the port resolution is actually checked against. +[[tool.mypy.overrides]] +module = [ + "testcontainers.*", +] +ignore_missing_imports = true + +[tool.coverage.run] +omit = ["tests/**"] + +[tool.poe.tasks] +sync-spec-assets = "python hatch_build_sync.py" +test = ["sync-spec-assets", {cmd = "pytest tests"}] +test-cov = ["sync-spec-assets", {cmd = "coverage run -m pytest tests"}] +cov-report = "coverage xml" +cov = ["test-cov", "cov-report"] +mypy = "mypy" diff --git a/tools/openfeature-tck/spec b/tools/openfeature-tck/spec new file mode 160000 index 00000000..aa2ad24f --- /dev/null +++ b/tools/openfeature-tck/spec @@ -0,0 +1 @@ +Subproject commit aa2ad24f5a14ae2b5756df0b6d23f493f39507e6 diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/__init__.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/__init__.py new file mode 100644 index 00000000..d0b7f91f --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/__init__.py @@ -0,0 +1,183 @@ +"""The OpenFeature Provider Conformance Suite (TCK) for Python. + +The suite answers one question: does this provider map its backend onto the +OpenFeature provider contract correctly? It is the Python implementation of +`Appendix F`_ of the specification, and it runs the same Gherkin scenarios, +against the same canonical flag set, that every other language's TCK runs. That +shared basis is the whole point -- "conformant" only means something if the +question is identical everywhere. + +**What a provider author writes.** Two fixtures and one call:: + + import pytest + from pytest_bdd import scenarios + + from openfeature.contrib.tools.tck import ( + Capability, + ComposeBackend, + RunningBackend, + TckConfig, + feature_paths, + ) + + @pytest.fixture(scope="session") + def compose_backend(): + return ComposeBackend( + compose_file="tests/tck/docker-compose.yaml", + backend_ports=[8013], + ) + + @pytest.fixture(scope="session") + def tck_config(tck_backend: RunningBackend): + return TckConfig( + name="my-provider", + control=tck_backend.control, + new_provider=lambda: MyProvider( + host=tck_backend.endpoint.host, + port=tck_backend.endpoint.port(8013), + ), + capabilities={Capability.EVENTS, Capability.OBJECT}, + ) + + scenarios(*feature_paths()) + +The suite owns the container stack: it starts the Compose file once, discovers +the dynamically mapped host ports, builds the HTTP control against the control +API, waits until it accepts commands, and tears down after the last scenario. +See :mod:`~.compose`. + +**A provider with no backend supplies its own control instead** -- in-memory, +environment-variable, file-based -- and needs no Compose file and no container +tooling:: + + @pytest.fixture(scope="session") + def tck_config(): + control = InProcessControl() + return TckConfig( + name="my-provider", + control=control, + new_provider=control.new_provider, + capabilities={Capability.EVENTS, Capability.OBJECT}, + ) + +``scenarios()`` is pytest-bdd's own, called directly rather than wrapped: it +injects the generated tests into the *calling module* by walking the stack, so a +convenience wrapper around it would deposit them inside this package instead. +:func:`~.extensions.feature_paths` is the canonical assets plus a +``extensions`` directory beside the calling module, if there is one -- see +:mod:`~.extensions`. + +The step definitions arrive through this package's pytest plugin, so there is +nothing to import for them and no ``conftest.py`` to write. Everything else -- +registering the provider, awaiting events, resetting the backend between +scenarios, tearing down -- belongs to the TCK. If you find yourself writing test +infrastructure, that is a defect here rather than something for you to work +around. + +.. _Appendix F: https://github.com/open-feature/spec/blob/main/specification/appendix-f-provider-conformance.md +""" + +from __future__ import annotations + +import importlib.resources + +from .capability import ( + DECLARABLE_CAPABILITIES, + INEXPRESSIBLE_CAPABILITIES, + RESERVED_CAPABILITIES, + Capability, +) +from .compose import ( + DEFAULT_BACKEND_SERVICE, + DEFAULT_CONTROL_PORT, + BackendEndpoint, + ComposeBackend, + RunningBackend, + run_compose_backend, +) +from .config import KnownDeviation, TckConfig +from .control import ( + BackendControl, + ConnectionControl, + ControlApi, + UnsupportedControlError, +) +from .extensions import ( + EXTENSIONS_DIRECTORY, + canonical_root, + feature_paths, +) +from .httpcontrol import ( + DEFAULT_CONFIGURATION, + DEFAULT_STARTUP_TIMEOUT, + ControlApiError, + HttpControl, +) +from .inprocess import InProcessControl +from .provider import ( + CHANGING_FLAG_KEY, + ControllableInMemoryProvider, + canonical_flag_set, + canonical_flags_json, +) +from .state import TckState + +__all__ = [ + "CHANGING_FLAG_KEY", + "DECLARABLE_CAPABILITIES", + "DEFAULT_BACKEND_SERVICE", + "DEFAULT_CONFIGURATION", + "DEFAULT_CONTROL_PORT", + "DEFAULT_STARTUP_TIMEOUT", + "EXTENSIONS_DIRECTORY", + "INEXPRESSIBLE_CAPABILITIES", + "RESERVED_CAPABILITIES", + "BackendControl", + "BackendEndpoint", + "Capability", + "ComposeBackend", + "ConnectionControl", + "ControlApi", + "ControlApiError", + "ControllableInMemoryProvider", + "HttpControl", + "InProcessControl", + "KnownDeviation", + "RunningBackend", + "TckConfig", + "TckState", + "UnsupportedControlError", + "canonical_flag_set", + "canonical_flags_json", + "canonical_root", + "control_api_spec", + "feature_paths", + "run_compose_backend", +] + +# NOTE ON THE SOURCE OF TRUTH +# +# The files under gherkin/ and flag_data/, and control-api.yaml, are NOT owned +# by this repository and are NOT committed to it. They are copies of the +# language-agnostic conformance artifacts defined in open-feature/spec under +# specification/assets/provider-tck/, which reaches this package as a git +# submodule at tools/openfeature-tck/spec and is copied in at build +# time by hatch_build.py. The copies are gitignored, so the only record of which +# spec revision this package targets is the submodule pin, and the two cannot +# drift apart unnoticed. +# +# They are copied into the distribution, so an adopter installing this package +# needs no submodule of their own; only a contributor to this package does. +# +# Changes belong in open-feature/spec first, followed by a bump of the submodule +# pin -- editing the copies locally forks the definition of conformance, which is +# the one thing this suite exists to prevent. +# See https://github.com/open-feature/spec/issues/417. + +_PACKAGE = "openfeature.contrib.tools.tck" + + +def control_api_spec() -> str: + """Return the OpenAPI document a containerised backend under test must implement.""" + ref = importlib.resources.files(_PACKAGE) / "control-api.yaml" + return ref.read_text(encoding="utf-8") diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/capability.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/capability.py new file mode 100644 index 00000000..e98dd5f5 --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/capability.py @@ -0,0 +1,451 @@ +"""Optional parts of the provider contract, and the Gherkin tags that gate them.""" + +from __future__ import annotations + +import typing +from collections.abc import Mapping +from enum import Enum +from types import MappingProxyType + +__all__ = ["Capability"] + + +class Capability(str, Enum): + """An optional part of the OpenFeature provider contract. + + Not every provider implements every part of the specification. A provider + backed by a static file has no meaningful notion of going stale; one with no + streaming transport cannot emit configuration-change events. Rather than + forcing such providers to fail scenarios they were never going to satisfy, + each declares what it supports through :attr:`TckConfig.capabilities`. + + `Appendix F + `_ + owns the vocabulary and the rules for declaring, including what a + declaration means, when to withhold one and when a + :class:`~.config.KnownDeviation` belongs beside it. The docstrings here say + what each tag gates *in this implementation* -- which scenarios run, what a + withholding skips, and what Python's SDK makes of the question -- and link + rather than restate. + + Every capability corresponds to exactly one Gherkin tag. pytest-bdd turns + those tags into pytest markers, and a scenario carrying a marker whose + capability was not declared is skipped with the reason reported -- never + passed. Scenarios with no capability tag are mandatory and always run. + + Two kinds of capability are refused rather than declared, and they are + refused for different reasons and with different messages: + :data:`RESERVED_CAPABILITIES`, which no scenario anywhere carries yet, and + :data:`INEXPRESSIBLE_CAPABILITIES`, whose question this language's SDK cannot + put at all. :data:`DECLARABLE_CAPABILITIES` is what is left. + """ + + LIFECYCLE = "lifecycle" + """Provider reaches its backend during initialisation, observably and promptly. + + Deliberately separate from :attr:`EVENTS`, because the two are independent in + both directions: an SDK dispatches ``PROVIDER_READY`` around ``initialize`` + for *any* provider, so gating the readiness scenario on ``EVENTS`` made it + vacuous for exactly the providers that declared it, while a stateless + provider has a real initialisation to verify and no event stream of its own. + + Declare it if initialisation actually contacts the backend and its outcome, + success or failure, is observable to the application. + """ + + EVENTS = "events" + """Provider emits lifecycle events at all, at minimum ``PROVIDER_READY``.""" + + STALE = "stale" + """Provider enters ``STALE`` and emits ``PROVIDER_STALE`` when it loses its backend.""" + + CONFIGURATION_CHANGE = "configuration-change" + """Provider detects configuration changes and emits ``PROVIDER_CONFIGURATION_CHANGED``. + + **The worked example of one skip meaning two different things**, which is + why it is spelled out on this tag rather than left abstract. Both of this + repository's withholdings of it are real and neither is the other: + + * **A choice.** The OFREP adoption withholds it because every evaluation is + an independent HTTP request: nothing is watching the backend, so there is + nothing to notice. A provider built that way is not defective, and a + :class:`~.config.KnownDeviation` there would assert a defect that does not + exist. + * **A defect.** The in-memory self-test withholds it because the SDK's + ``InMemoryProvider`` copies its flag mapping in the constructor and exposes + no way to change it, where `Appendix A + `_ + **requires** an SDK's in-memory provider to support updating the flag set + and emitting this event. Tracked as `open-feature/python-sdk#620 + `_. + + A report shows the same absence in both cases, which is the whole reason the + declaration is not the last word: the adoption's note says which, and for the + second kind there is a ``ControllableInMemoryProvider`` here supplying what + the SDK lacks, so the scenarios still run somewhere. + """ + + OBJECT = "object" + """Provider supports structured (object) flag values.""" + + VARIANTS = "variants" + """Provider names the variant it resolved. + + Gated because a variant is optional rather than required: `Requirement 2.2.4 + `_ + is a **SHOULD** and ``types.md`` types the field ``variant (string, + optional)``, so a backend with no variant concept for a plain flag never + gives the provider one to report. + + Declaring it runs one Scenario Outline that asserts the variant for each of + the eight flags whose variant name the canonical set fixes. Withholding it + skips those rows with the reason and changes nothing else: the value + assertions live in untagged scenarios, because `Requirement 2.2.3 + `_ + makes the value a **MUST**. + """ + + DISABLED_FLAGS = "disabled-flags" + """Provider resolves a flag disabled in the management system to the code default. + + Gated because it needs two things and only one of them comes for free. The + caller's default value is held by the provider, which always has it. What + the provider also needs is a **signal** that the flag was disabled, told + apart from an ordinary resolution and from a missing flag -- and that is a + property of the backend and its protocol. One that has no disabled state, or + that answers ``FLAG_NOT_FOUND`` for a disabled flag, gives the provider + nothing to act on, and no care in the provider produces a substitution it + was never told to make. + + Nothing in the specification says what a provider owes a disabled flag. + `Requirement 1.4.7 + `_ + is about the SDK propagating whatever reason arrived, and `Requirement 2.2.5 + `_ + only lists ``DISABLED`` among the reason strings a provider **may** use. So + Appendix F states the behaviour, the way it does for + :attr:`NUMERIC_COERCION`, and gates it. + + A remote evaluator is not shut out of this, which is worth knowing before + reading a withholding as an architectural impossibility: flagd's RPC resolver + satisfies the capability by substituting locally on the strength of the + server's ``DISABLED`` reason, and flagd's OFREP endpoint sends the same + signal in another envelope. The two adoptions in this repository record what + each one does with it. + + Declaring it runs one Scenario Outline of four rows, over the four + ``disabled-*`` flags of the canonical set. They mirror ``boolean-flag``, + ``string-flag``, ``integer-flag`` and ``float-flag`` exactly, differing only + in ``state``, and each row's caller default differs from the flag's + configured value -- so a provider that ignores the state returns the + configured value and is caught on the value alone, which rests on 2.2.3, a + **MUST**. + + The rows assert the value and the absence of an error, and deliberately + **not** the reason: pinning ``DISABLED`` here would rest on 2.2.5, a + **SHOULD** that permits "some other string". It is pinned in + ``reason.feature`` instead, which composes this tag with + :attr:`STANDARD_REASONS` so that both must be declared before the reason is + asserted. No variant is asserted either, because a disabled flag has resolved + no variant and there is none to name -- so this capability and + :attr:`VARIANTS` do not compose, which is why the rows are not part of the + variant outline. + + The SDK's own ``InMemoryProvider`` cannot declare this, and the reason is + worth knowing before adopting it as a reference: ``InMemoryFlag`` accepts a + ``state`` of ``DISABLED`` and nothing ever reads it, so a disabled flag is + served like any other. ``_decode_canonical_flags`` passes the state through + faithfully; the provider is where it stops. + """ + + UNAVAILABLE_INIT = "unavailable" + """Provider reports an error state promptly against a backend it cannot reach.""" + + NUMERIC_COERCION = "numeric-coercion" + """Provider coerces between integer and float only when lossless, else ``TYPE_MISMATCH``. + + This is the one entry here that **the specification does not define**: the + rule is borrowed from flagd's `numeric coercion ADR + `_ + while `open-feature/spec#430 `_ + is open, so **a provider that behaves differently is not violating the + specification**. Appendix F's numeric-coercion note carries the rule, and + settles which of "declares and fails" and "withholds" a provider reaches for: + one that *attempts* the coercion and gets a direction wrong declares the + capability and records a :class:`~.config.KnownDeviation` beside the failing + scenario, and withholding is for one that cannot attempt the behaviour at + all. + + Both halves have scenarios, and a provider declaring the tag must satisfy all + three: the lossy half asks for ``float-flag`` (``0.5``) as an integer and + expects ``TYPE_MISMATCH``; the lossless half asks for ``integral-float-flag`` + (``10.0``) as an integer and for ``integer-flag`` (``10``) as a float, and + expects both to succeed. Rejecting every float is an easy way to pass the + first, and the other two are what stop it. + + The SDK's own ``InMemoryProvider`` is the withholding kind: it hands values + back untouched and the client's type check is ``isinstance``-based, so + ``10.0`` requested as an integer is a ``TYPE_MISMATCH`` rather than ``10``. + That is the provider declining to coerce, not the language refusing to ask -- + a provider that does coerce returns an ``int`` and the same check passes it, + which is why :data:`INEXPRESSIBLE_CAPABILITIES` is empty here. The width of + the integer accessor is a separate property, and a separate capability: + :attr:`LARGE_INTEGERS`. + """ + + LARGE_INTEGERS = "large-integers" + """Provider resolves integers up to 2^53 - 1 exactly. + + A property of the language's SDK as much as of the provider, which is why it + is a capability rather than mandatory: a 32-bit integer accessor has no room + for the value. Every language can ask for 2^31 - 1, so that precision + scenario is untagged; only the one asking for 2^53 - 1 carries this tag. + + Python's ``int`` is unbounded, so a Python provider declares it unless + something of its own -- a 32-bit field in its wire format, a float on the way + through -- narrows the value, or unless the backend under test serves no such + flag for it to be asked about. Which makes this one of the two capabilities + Appendix F names as inexpressible somewhere and **not** here: + :data:`INEXPRESSIBLE_CAPABILITIES` is empty in Python, and says on what + measurement. + """ + + REINITIALIZATION = "reinitialization" + """Provider can be initialised again after ``shutdown``, and serves flags afterwards. + + Gated rather than mandatory because `Requirement 2.5.2 + `_ + permits reuse without requiring it: a provider that releases its client on + shutdown and declines to be started again is exercising a choice the + specification offers it, so withholding this capability needs no + :class:`~.config.KnownDeviation` entry. + + **This tag narrows :attr:`LIFECYCLE` rather than standing beside it**, and + that is the trap worth naming. The scenario lives in ``lifecycle.feature``, + which carries ``@lifecycle`` at the feature level, and the gate skips a + scenario if *any* capability gating it is undeclared -- so declaring this one + alone leaves the scenario skipped and the declaration unverified. A provider + that withholds ``LIFECYCLE`` has therefore never run this scenario, and has + no evidence on which to declare this one either way. + """ + + TARGETING = "targeting" + """Provider resolves a flag differently for a matching evaluation context. + + What the three scenarios test is not how a backend evaluates a rule: they + exist to show that the **context reached the backend at all**, which is a + property of the provider and of nothing else. ``targeting-key-flag`` is the + one flag in the canonical set with a rule, and it is what makes passthrough + observable without an echo endpoint on the control API -- a matching context + resolves ``hit`` where anything else resolves ``miss``. + + The three scenarios are the matching context, the non-matching one and no + context at all. The second and third are not padding: a provider that always + returned the targeted value would pass the first, and one that refuses to + evaluate a rule with no targeting key present is caught by the third. + + Two more scenarios carry this tag alongside :attr:`STANDARD_REASONS`, in + ``reason.feature``, asserting ``TARGETING_MATCH`` for the hit and ``DEFAULT`` + for the miss; declaring this capability alone leaves them skipped and changes + nothing about the three above. + + Declare it if the backend under test can express that rule and the provider + forwards the targeting key. A backend with no targeting at all leaves it + undeclared and the three scenarios are skipped with the reason -- which is + also the right answer for an in-memory flag set whose decoder ignores the + ``targeting`` member, as this package's own does. + """ + + STANDARD_REASONS = "standard-reasons" + """Provider reports the standard resolution reasons, with the standard meanings. + + **A claim, not an exemption.** `Requirement 2.2.5 + `_ + is a **SHOULD** that lets a provider populate ``reason`` with *"some other + string indicating the semantic reason for the returned flag value"*, so a + provider whose backend reports vendor-specific reasons is conformant and is + not expected to declare this. Declaring it says "I use the standard + vocabulary with the standard meanings", and ``reason.feature`` is what checks + the claim; Appendix F's ``@standard-reasons`` section is where the meanings + are fixed, including the two rows -- ``STATIC`` for a rule-less flag, and + ``ERROR`` -- that are calls rather than consequences. + + A provider that does not declare it **loses nothing**: its values, variants + and error codes are asserted everywhere else, on **MUST** requirements. + + **Tags compose, and here that is load-bearing.** ``TARGETING_MATCH`` cannot + be observed without targeting and ``DISABLED`` cannot be observed unless the + backend distinguishes a disabled flag, so those scenarios carry + :attr:`TARGETING` and :attr:`DISABLED_FLAGS` as well. A provider declaring + this capability alone runs the four ``STATIC`` rows and the two error + scenarios, and skips the other three with their reason. + + This is also the first capability whose tag is carried at the **feature** + level rather than on each scenario. pytest-bdd marks a scenario from + ``scenario.tags | feature.tags | rule.tags``, so the gate -- which reads + markers -- sees it on every scenario in the file, and ``Scenario.tags`` + alone would not have. + """ + + CACHING = "caching" + """Reserved, and **not declarable**. No scenario carries this tag yet.""" + + @property + def tag(self) -> str: + """Return the Gherkin tag, with its leading at-sign, that gates this capability.""" + return f"@{self.value}" + + @property + def reserved(self) -> bool: + """Whether this capability exists in the vocabulary but gates no scenario.""" + return self in RESERVED_CAPABILITIES + + @property + def inexpressible(self) -> bool: + """Whether this SDK cannot put the question this capability's scenarios ask. + + Distinct from :attr:`reserved` in every respect except that both end in a + refusal. See :data:`INEXPRESSIBLE_CAPABILITIES`. + """ + return self in INEXPRESSIBLE_CAPABILITIES + + @property + def inexpressible_reason(self) -> str | None: + """Which property of this SDK puts the question out of reach, or ``None``. + + The property, not the rule: a message that only says "this cannot be + declared" leaves the adopter to discover why, and the why is the part + they could not have been expected to know. + """ + return INEXPRESSIBLE_CAPABILITIES.get(self) + + def __str__(self) -> str: + return self.tag + + +RESERVED_CAPABILITIES: frozenset[Capability] = frozenset({Capability.CACHING}) +"""Capabilities that exist in the vocabulary and gate no scenario. + +They are documented so the vocabulary has a place for them when scenarios exist, +and until then they **must not be declared** -- nothing carries the tag, so +declaring it cannot be verified and cannot produce a skip. + +Listed once, here, and read everywhere else -- by +:data:`DECLARABLE_CAPABILITIES`, by :attr:`Capability.reserved` and by the +validation in :class:`~.config.TckConfig` -- so that the set and the rule cannot +drift apart. A reservation expires in the specification repository rather than +here, which is what :func:`expired_reservations` exists to notice. +""" + +INEXPRESSIBLE_CAPABILITIES: Mapping[Capability, str] = MappingProxyType({}) +"""Capabilities this language's SDK cannot put the question for, and why. + +**Empty in Python, and that is a measurement rather than an omission.** The two +that exist anywhere are :attr:`Capability.LARGE_INTEGERS`, inexpressible where +the integer accessor is a 32-bit ``Integer``, and +:attr:`Capability.NUMERIC_COERCION`, inexpressible where the language has a +single numeric type and "a float requested as an integer" does not name two +different requests. Python has neither property: ``int`` is arbitrary-precision, +and ``get_integer_details`` and ``get_float_details`` are separate accessors +reaching separate provider methods, type-checked against ``int`` and ``float`` +separately. Both were checked by asking all four questions through the SDK +rather than by reading its source, and every one of them was answered. + +So this mapping carries no entries, and the machinery around it carries no load +here. It exists anyway because the rule is Appendix F's rather than this +package's, because the next capability may hit it, and because the cost of the +two is not symmetric: an unused mechanism is a few lines nobody reads, while a +missing one is discovered by an adopter publishing a claim no scenario could +have examined. + +**Not the same thing as a reservation**, and the value is what carries the +difference: a reader seeing a capability missing from a report has to be able to +tell *"this provider declined"* from *"no provider in this language can be +asked"*. Hence a mapping rather than a set -- the value is the property of the +SDK that puts the question out of reach, which is the half of the message an +adopter could not have worked out for themselves. + +A capability belongs here only when **no** provider in this language could ever +satisfy it. A provider that gets the answer wrong is a different thing entirely: +flagd's two Python resolvers answer the ``@numeric-coercion`` scenarios +differently from each other, and **both** declare the tag, the one that narrows +carrying the :class:`~.config.KnownDeviation`. Listing it here would say the +question cannot be asked -- and two resolvers of one provider giving different +answers to it is the proof that it can. + +Never overlaps :data:`RESERVED_CAPABILITIES`: a tag no scenario carries is +reserved, whatever any SDK could express about it. +""" + +DECLARABLE_CAPABILITIES: frozenset[Capability] = ( + frozenset(Capability) + - RESERVED_CAPABILITIES + - frozenset(INEXPRESSIBLE_CAPABILITIES) +) +"""Every capability an adoption may declare: the vocabulary minus what is refused. + +A reasonable starting point for a new adoption: declare everything, run the +suite, and remove only what the provider genuinely cannot do. Narrowing from this +set surfaces gaps; widening towards it hides them. + +It excludes the reserved capabilities rather than spanning the whole enum, and it +is named for what it is rather than for "all", because a declare-everything +convenience is exactly how a reserved tag reaches a report by accident -- an +adopter writing "every capability except X" picks up every reserved tag on the +way past. + +It excludes :data:`INEXPRESSIBLE_CAPABILITIES` for the same reason and one more: +that set is empty in Python, so a default spanning the whole enum would look +correct here forever and be wrong the day an entry is added, in the one language +where it was added. Derived rather than listed, so it cannot be the thing that is +out of date. +""" + +_BY_MARKER: dict[str, Capability] = {c.value: c for c in Capability} +_BY_TAG: dict[str, Capability] = {c.tag: c for c in Capability} + + +def capability_for_marker(name: str) -> Capability | None: + """Map a pytest marker name onto the capability it gates, if any. + + A marker that does not name a capability gates nothing, which is what lets + the canonical feature files carry organisational tags freely. + """ + return _BY_MARKER.get(name) + + +def capability_for_tag(tag: str) -> Capability | None: + """Map a Gherkin tag, leading at-sign included, onto the capability it gates. + + The tag form rather than the marker form because that is what the + conformance report carries: the report records a scenario's tags as the + feature files spell them, and deciding whether a failure counts against a + capability means reading them back. + """ + return _BY_TAG.get(tag) + + +def expired_reservations(tags: typing.Iterable[str]) -> tuple[Capability, ...]: + """Reserved capabilities that the tags handed in turn out to carry. + + A non-empty answer means some scenario is both unrunnable and unclaimable: + declaring a reserved capability is refused, so the gate skips every scenario + carrying its tag, and the report says a gap exists where the provider may + well have none. + + Two things put a reserved tag on a scenario and this reports only that one + of them happened. Either :data:`RESERVED_CAPABILITIES` is out of date -- + the scenarios the tag was held open for now exist, so an adoption should be + allowed, and required, to say whether it has the capability -- or an adopter + has used a reserved name for a tag of their own. Telling the two apart is the + caller's, because the caller is what knows where the tags came from: the + remedies differ and the consequence does not. + + Compared against tags rather than against a second list, because a + reservation expires in the specification repository while this set lives + here. Deduplicated and ordered by tag: the tags arrive from every scenario + of every feature file, and one carried twice is not two expiries. + """ + carried = set(tags) + expired = (c for c in RESERVED_CAPABILITIES if c.tag in carried) + return tuple(sorted(expired, key=lambda capability: capability.tag)) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/compose.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/compose.py new file mode 100644 index 00000000..09f0b372 --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/compose.py @@ -0,0 +1,494 @@ +"""The container stack, owned by the suite rather than by every adopter. + +A provider that talks to a backend needs that backend running, its dynamically +mapped host ports discovered, and an :class:`~.httpcontrol.HttpControl` built +against its control API. Every adoption needs the same three things, and until +now every adoption wrote them: this package shipped the control client and left +orchestration to the adopter, so the flagd adoption alone carried a 122-line +``conftest.py`` and a 170-line ``suite.py`` of container wiring, and the next +adopter would have paid for it again. + +So the suite owns the stack. **An adopter names a Compose file, says which +service and ports to expose, and supplies a factory that builds a provider from +a discovered endpoint.** Everything else -- start the stack once, discover the +mapped ports, build the control, wait until it accepts commands, tear down after +the last scenario -- happens here. + +Two lines, in the adopter's ``conftest.py``:: + + @pytest.fixture(scope="session") + def compose_backend() -> ComposeBackend: + return ComposeBackend( + compose_file="tests/tck/docker-compose.yaml", + backend_ports=[8013], + ) + + @pytest.fixture(scope="session") + def tck_config(tck_backend: RunningBackend) -> TckConfig: + return TckConfig( + name="my-provider", + control=tck_backend.control, + new_provider=lambda: MyProvider( + host=tck_backend.endpoint.host, + port=tck_backend.endpoint.port(8013), + ), + ) + +``tck_backend`` is a session-scoped fixture this package's plugin supplies. It +is lazy, so a provider with no backend never touches it and never needs Docker +installed -- see :mod:`~.inprocess`. + +**The stack starts once per suite and is never restarted.** Container +orchestrators assign host ports dynamically and cannot reliably preserve them +across a restart, so a restarted backend comes back on a different host port, +silently invalidating every provider already pointed at the old one -- and the +failure looks like a flaky provider rather than a broken test. Backend +unavailability is always simulated *inside* the running stack, through the +control API. That is also why :attr:`TckConfig.new_provider` is a factory rather +than an instance: the ports do not exist until the stack is up. + +**The Compose path does not replace the manual one.** A provider with no backend +at all keeps supplying its own :class:`~.control.BackendControl` exactly as +before. Compose is an additional path, and the one nearly every provider wants. + +**Do not substitute a control of your own that pokes an external backend through +a side channel.** The HTTP control API is the normative contract: another +language's suite drives the same endpoints against the same stack and must get +the same answers. See :class:`~.control.BackendControl`. + +Requires the ``compose`` extra -- ``pip install 'openfeature-tck[compose]'`` -- +which is what pulls in ``testcontainers``. Keeping it optional is deliberate: an +in-memory adopter should not have to install container tooling to run a suite +that never starts a container. +""" + +from __future__ import annotations + +import contextlib +import socket +import time +import typing +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path + +from .httpcontrol import ( + DEFAULT_CONFIGURATION, + DEFAULT_STARTUP_TIMEOUT, + ControlApiError, + HttpControl, +) + +__all__ = [ + "DEFAULT_BACKEND_SERVICE", + "DEFAULT_CONTROL_PORT", + "BackendEndpoint", + "ComposeBackend", + "ComposeStack", + "RunningBackend", + "run_compose_backend", +] + +DEFAULT_BACKEND_SERVICE = "backend" +"""The Compose service name a stack is expected to host the backend under. + +Fixed across every language's TCK, so the same Compose file is the whole of what +an adoption shares between two of them. +""" + +DEFAULT_CONTROL_PORT = 8080 +"""The container-internal port the control API is expected to listen on.""" + +_PORT_PROBE_TIMEOUT = 1.0 +"""Seconds bounding a single TCP connect while waiting for a published port.""" + +_PORT_POLL_INTERVAL = 0.2 + + +class ComposeStack(typing.Protocol): + """The two things this module needs from a Compose stack. + + A protocol rather than ``testcontainers.compose.DockerCompose`` itself, so + that port resolution is checkable and testable without Docker installed and + without the optional dependency present. The real class satisfies it + structurally. + """ + + def get_service_host( + self, service_name: str | None = ..., port: int | None = ... + ) -> str | None: ... + + def get_service_port( + self, service_name: str | None = ..., port: int | None = ... + ) -> int | None: ... + + +@dataclass(frozen=True) +class BackendEndpoint: + """Where the running stack is reachable, handed to the provider factory. + + This type exists because host ports are only known *after* the stack has + started. A Compose file under test must not pin host ports -- Docker assigns + them dynamically -- so a provider cannot be configured until the stack is up, + which is the whole reason :attr:`TckConfig.new_provider` is a factory. + + The mapping is stable for the lifetime of the suite: the stack is started + once and never restarted, so a provider built from this endpoint stays valid + for every scenario. + """ + + stack: ComposeStack + """The running stack the addresses are resolved from.""" + + backend_service: str = DEFAULT_BACKEND_SERVICE + """The service :attr:`host` and the one-argument :meth:`port` resolve against.""" + + service_ports: Mapping[str, Sequence[int]] = field(default_factory=dict) + """Which container-internal ports each service publishes, from the declaration. + + Carried so that a host can be resolved at all. Testcontainers resolves a + service's host *through* one of its published ports, and asked for a service + without naming one it demands that the service publish exactly one -- so on + a stack like flagd's, which publishes three on one service, the no-port form + raises ``NoSuchPortExposed`` and the message blames the port rather than the + call. Any of the service's ports answers the question, so the first declared + one is used. + """ + + @property + def host(self) -> str: + """The host the backend service is reachable on. + + Not necessarily ``localhost``: with a remote Docker daemon, Docker + Desktop on some platforms, or a rootless setup it can be an arbitrary + address. Always use this rather than hard-coding a host. + """ + return self.host_of(self.backend_service) + + def host_of(self, service: str) -> str: + """The host a named service is reachable on. + + :raises ControlApiError: if the declaration names no port for that + service, so there is nothing to resolve a host through -- and + nothing for a provider to connect to either. + """ + probe = next(iter(self.service_ports.get(service, ())), None) + if probe is None: + known = ", ".join(sorted(self.service_ports)) or "(none)" + msg = ( + f"no port is declared for service {service!r}, so its host cannot " + f"be resolved: a host is resolved through one of a service's " + f"published ports. Declared services are {known}; add it to " + f"ComposeBackend.additional_ports" + ) + raise ControlApiError(msg) + host = self.stack.get_service_host(service, probe) + return host or "localhost" + + def port(self, internal_port: int, *, service: str | None = None) -> int: + """The mapped host port for a container-internal port. + + :param internal_port: the container-internal port, as declared in + :attr:`ComposeBackend.backend_ports` or + :attr:`ComposeBackend.additional_ports`. + :param service: the Compose service, defaulting to + :attr:`backend_service`. Use it for a stack with more than one + service -- a proxy, an edge service, a sidecar. + :raises ControlApiError: if that port is not published by the stack, + which means the Compose file does not list it under ``ports:``. + """ + name = service or self.backend_service + mapped = self.stack.get_service_port(name, internal_port) + if mapped is None: + msg = ( + f"the Compose stack publishes no host port for {internal_port} on " + f"service {name!r}. A port is only published if the Compose file " + f"lists it under that service's `ports:` -- unpinned, as a bare " + f"container port, so Docker maps it dynamically" + ) + raise ControlApiError(msg) + return int(mapped) + + +@dataclass(frozen=True) +class ComposeBackend: + """What an adopter declares about the stack under test. + + Everything except :attr:`compose_file` and :attr:`backend_ports` has a + default, and the defaults are the same in every language's TCK. + """ + + compose_file: str | Path + """Path to the Compose file describing the stack. + + Resolved relative to the package directory -- the directory ``pytest`` was + invoked from for a normal ``poe test``, which is the same directory the + package's ``pyproject.toml`` sits in. An absolute path is used as given. + + The stack must not pin host ports. Docker assigns them dynamically and this + module discovers them after startup; a pinned host port makes the suite + unrunnable in parallel and collides with whatever the developer already has + listening. + """ + + backend_ports: Sequence[int] + """Container-internal ports on :attr:`backend_service` that the *provider* + connects to. + + :attr:`control_port` is handled automatically and must not be listed here. + + Declaring them is what lets the harness fail with "the Compose file does not + publish 8013" at startup, rather than with a provider that cannot connect + three scenarios later. + """ + + backend_service: str = DEFAULT_BACKEND_SERVICE + """The Compose service hosting both the control API and the backend.""" + + control_port: int = DEFAULT_CONTROL_PORT + """The container-internal port of the control API.""" + + additional_ports: Mapping[str, Sequence[int]] = field(default_factory=dict) + """Extra service -> ports, for a stack with more than one service. + + Resolved through the endpoint by service name:: + + endpoint.port(9212, service="proxy") + """ + + backend_configuration: str = DEFAULT_CONFIGURATION + """The configuration name passed to ``POST /start``. + + ``default`` is the only name every backend must support, and the one that + serves the canonical flag set the feature files assume. + + Named for the *backend* because ``configuration`` on its own is already + taken, by the conformance report's ``provider.configuration`` -- which is + which materially different mode of the provider was tested, flagd's RPC + versus in-process, and which :attr:`TckConfig.name` feeds. The two are + unrelated and one word for both made a report's ``configuration`` mean + opposite things depending on which language's TCK produced it. + """ + + startup_timeout: float = DEFAULT_STARTUP_TIMEOUT + """Seconds to wait for the stack and its control API to become reachable.""" + + def __post_init__(self) -> None: + # Normalised before anything is validated, so a declaration written as + # any other iterable is checked as the tuple it becomes rather than + # consumed by the checking. + object.__setattr__(self, "backend_ports", tuple(self.backend_ports)) + object.__setattr__( + self, + "additional_ports", + {service: tuple(ports) for service, ports in self.additional_ports.items()}, + ) + + problems: list[str] = [] + + if not str(self.compose_file): + problems.append( + "compose_file is required: it is the path to the Compose file " + "describing the stack under test" + ) + if not self.backend_ports: + problems.append( + "backend_ports is required and must not be empty: it is the " + "container-internal ports the provider connects to, so the harness can " + "check the Compose file publishes them before the first scenario. The " + f"control port ({self.control_port}) is handled automatically and does " + "not belong here" + ) + if self.control_port in self.backend_ports: + problems.append( + f"backend_ports lists the control port {self.control_port}: it is " + "exposed automatically, and a provider that connects to the control " + "API is not exercising the contract this suite tests. Remove it, or " + "set control_port if the control API is somewhere else" + ) + if not self.backend_service: + problems.append( + "backend_service must name a Compose service: it is the service " + "hosting both the control API and the backend" + ) + if self.startup_timeout <= 0: + problems.append( + f"startup_timeout must be positive, not {self.startup_timeout!r}" + ) + + if problems: + joined = "\n - ".join(problems) + msg = f"invalid ComposeBackend:\n - {joined}" + raise ValueError(msg) + + def resolved_compose_file(self, root: Path | None = None) -> Path: + """The Compose file as an absolute path, resolved against ``root``. + + ``root`` defaults to the current working directory, which for a normal + ``poe test`` is the package directory. + """ + candidate = Path(self.compose_file) + if not candidate.is_absolute(): + candidate = (root or Path.cwd()) / candidate + return candidate + + @property + def exposed_ports(self) -> dict[str, tuple[int, ...]]: + """Every service and container-internal port the stack must publish. + + The control port first, because a stack that publishes nothing else + still has to answer control calls, then the backend ports, then whatever + :attr:`additional_ports` adds. + """ + ports: dict[str, tuple[int, ...]] = { + self.backend_service: (self.control_port, *self.backend_ports), + } + for service, extra in self.additional_ports.items(): + merged = (*ports.get(service, ()), *extra) + # dict.fromkeys rather than a set: order is what makes the startup + # failure message read in the order the adopter wrote the ports. + ports[service] = tuple(dict.fromkeys(merged)) + return ports + + +@dataclass(frozen=True) +class RunningBackend: + """The started stack, as the ``tck_backend`` fixture yields it. + + Two things, because two things are all an adoption needs: the control to + hand to :attr:`TckConfig.control`, and the endpoint to build providers from. + """ + + control: HttpControl + """The control API client, already awaited ready. + + One instance per stack, and it must stay that way where two suites drive the + same backend: it remembers whether a disconnect has left the backend down, + so the next scenario is prepared with ``/start`` rather than ``/reset``, and + two instances would each hold half of that knowledge. + """ + + endpoint: BackendEndpoint + """Host and mapped ports of the running stack, for the provider factory.""" + + backend: ComposeBackend + """The declaration this stack was started from.""" + + +def run_compose_backend( + backend: ComposeBackend, *, root: Path | None = None +) -> Iterator[RunningBackend]: + """Start the declared stack, yield it, and tear it down. + + A generator, so the ``tck_backend`` fixture is ``yield from`` over this and + an adopter who wants a fixture of their own naming or scoping can be too:: + + @pytest.fixture(scope="session") + def tck_backend() -> Iterator[RunningBackend]: + yield from run_compose_backend(ComposeBackend(...)) + + Startup is: bring the stack up and wait for its containers, wait for every + declared port to accept a connection, then wait for the control API itself + to accept commands. The last of those is a real readiness check rather than + a pause -- see :meth:`HttpControl.await_ready`. + """ + compose_file = backend.resolved_compose_file(root) + if not compose_file.is_file(): + msg = ( + f"Compose file not found: {compose_file}. " + f"ComposeBackend.compose_file is resolved relative to the package " + f"directory, which is where pytest runs from" + ) + raise FileNotFoundError(msg) + + stack = _docker_compose(compose_file) + stack.start() + try: + endpoint = BackendEndpoint( + stack=typing.cast("ComposeStack", stack), + backend_service=backend.backend_service, + service_ports=backend.exposed_ports, + ) + _await_ports(backend, endpoint) + + control = HttpControl( + f"http://{endpoint.host}:{endpoint.port(backend.control_port)}", + backend_configuration=backend.backend_configuration, + ) + control.await_ready(backend.startup_timeout) + + yield RunningBackend(control=control, endpoint=endpoint, backend=backend) + finally: + stack.stop() + + +def _docker_compose(compose_file: Path) -> typing.Any: + """Build a ``DockerCompose`` for one Compose file. + + Imported here rather than at module scope so that importing this module -- + which the package's ``__init__`` does -- costs nothing and, more to the + point, does not require ``testcontainers`` to be installed. An in-memory + adopter has no use for container tooling and should not have to install it. + """ + try: + # PLC0415: deliberately not at module scope. That is the whole point of + # this function -- see the docstring. + from testcontainers.compose import DockerCompose # noqa: PLC0415 + except ImportError as error: # pragma: no cover - depends on the environment + msg = ( + "the Compose harness needs testcontainers, which is the `compose` " + "extra of this package: `pip install 'openfeature-tck[compose]'`. It is " + "optional because a provider with no backend runs the whole suite " + "without a container -- see the in-process control" + ) + raise ImportError(msg) from error + + return DockerCompose( + context=str(compose_file.parent), + compose_file_name=compose_file.name, + # `docker compose up --wait`, so start() returns once the containers are + # up rather than once the command has been issued. + wait=True, + ) + + +def _await_ports(backend: ComposeBackend, endpoint: BackendEndpoint) -> None: + """Wait until every declared port accepts a TCP connection. + + Java's harness gets this from a Testcontainers listening-port wait strategy + per exposed service port; ``docker compose up --wait`` only promises the + container is up, which for a service with no healthcheck it is well before + anything is listening. Same guarantee, established the same way, so the two + languages fail at the same point rather than one of them failing later and + somewhere less obvious. + """ + deadline = time.monotonic() + backend.startup_timeout + for service, ports in backend.exposed_ports.items(): + host = endpoint.host_of(service) + for internal in ports: + mapped = endpoint.port(internal, service=service) + _await_listening(host, mapped, service, internal, deadline) + + +def _await_listening( + host: str, port: int, service: str, internal: int, deadline: float +) -> None: + last: OSError | None = None + while True: + try: + with contextlib.closing( + socket.create_connection((host, port), timeout=_PORT_PROBE_TIMEOUT) + ): + return + except OSError as error: + last = error + if time.monotonic() >= deadline: + msg = ( + f"nothing is listening on {host}:{port} -- the host port Compose " + f"mapped for container port {internal} of service {service!r} -- " + f"within the startup timeout: {last}. Either the service does not " + f"listen on {internal}, or the stack needs a longer " + f"ComposeBackend.startup_timeout" + ) + raise ControlApiError(msg) + time.sleep(_PORT_POLL_INTERVAL) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/config.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/config.py new file mode 100644 index 00000000..3833d644 --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/config.py @@ -0,0 +1,477 @@ +"""The contract a provider author implements to run the suite.""" + +from __future__ import annotations + +import typing +from collections.abc import Callable, Collection, Iterable, Sequence +from dataclasses import dataclass, field + +from openfeature.provider import FeatureProvider + +from .capability import DECLARABLE_CAPABILITIES, Capability +from .control import BackendControl + +__all__ = ["KnownDeviation", "ProviderFactory", "TckConfig"] + +ProviderFactory = Callable[[], FeatureProvider] +"""Creates the provider under test. + +A factory rather than a single instance because each scenario gets its own +provider, and because a provider often cannot be configured before the suite +starts -- a container stack's host ports do not exist until it is up. +""" + +DEFAULT_EVENT_TIMEOUT = 12.0 +DEFAULT_READY_TIMEOUT = 30.0 + + +@dataclass(frozen=True) +class KnownDeviation: + """A gap the provider is known to have, acknowledged rather than hidden. + + **A ``knownDeviations`` entry says: this provider fails to do something it is + required to do.** The requirement must be a numbered ``MUST``, or a rule the + implementation bound itself to elsewhere. Distinct from an undeclared + capability, which is a *choice* the provider is entitled to make: where the + specification permits the choice, withholding the capability **is** the + honest report, and a deviation entry would assert a defect that does not + exist. + + It is legitimate in two shapes, and a report's results already distinguish + them: + + 1. **The capability is declared, the scenario runs, and it fails.** Prefer + this. The failure stays visible and the deviation says it is known and + why. + 2. **The capability is withheld, and its scenarios skip.** Legitimate only + when the provider cannot attempt the behaviour at all, so running the + scenario would establish nothing. The deviation then explains the + absence, so a reader can tell a defect from a design decision. + + Withdrawing a capability *in order to* turn a failing scenario into a skip is + the failure mode this field exists to prevent. If the provider attempts the + behaviour and gets it wrong, shape 1 is the honest report. + + **A scenario that fails because the backend serves no fixture for it is not a + provider defect and does not belong here.** That is the first consequence + Appendix F draws from its declaring rules, and an entry recording it would + accuse the provider of the backend's gap. Where such a failure sits under the + same tag as a real one -- which is the ordinary case, since the tag is + declared on the scenarios that *can* be asked -- say so in the summary of the + entry that covers the real one. + + It changes nothing about how the suite runs. The scenario still fails, and + the results payload still reports it as failed -- a report that softened a + failure into a footnote would hide exactly what the acknowledgement exists to + keep visible. What this adds is the acknowledgement itself, in the envelope, + so that a consumer can tell a known gap from a surprise. + + Build one with :meth:`tracked` or :meth:`untracked` rather than by calling + the constructor, so that which of the two a deviation is stays a decision + someone made rather than a field someone forgot. The same two forms exist in + the Go, Java and JavaScript suites. + """ + + summary: str + """What is wrong, for a person reading a comparison page. + + Required. A deviation with no summary records that something is wrong without + saying what, which is worth less than the bare skip or failure it + accompanies. + """ + + issue: str | None = None + """Where the gap is tracked, or ``None`` when it is tracked nowhere yet. + + Optional. There is a tracked and an untracked form, and naming an untracked + defect is still what separates it from a capability the provider chose to + withhold -- a declaration that merely omits the tag cannot say which of the + two happened. Prefer :meth:`tracked` as soon as there is an issue to point + at. + """ + + capability: Capability | None = None + """The capability the deviation concerns, when it maps to one. + + Left out when the gap is against a mandatory, ungated scenario, which belongs + to no capability. + + A reserved capability is refused: no scenario carries the tag, so there is + nothing to deviate from. See :data:`~.capability.RESERVED_CAPABILITIES`. So + is one this SDK cannot express, for the opposite reason -- the scenarios + exist and no provider here can attempt them, so the gap is the language's + and not this provider's. See + :data:`~.capability.INEXPRESSIBLE_CAPABILITIES`. + """ + + @classmethod + def tracked( + cls, + summary: str, + issue: str, + capability: Capability | None = None, + ) -> KnownDeviation: + """Record a deviation that is tracked somewhere. + + :param summary: what the gap is. + :param issue: a URI where it is tracked. + :param capability: the capability the gap concerns, or ``None`` when the + gap is against a mandatory scenario and so belongs to no capability. + """ + return cls(summary=summary, issue=issue, capability=capability) + + @classmethod + def untracked( + cls, + summary: str, + capability: Capability | None = None, + ) -> KnownDeviation: + """Record a deviation that is not tracked anywhere yet. + + Worth declaring even so: naming the defect is what separates it from a + capability the provider chose to withhold. Prefer :meth:`tracked` as soon + as there is an issue to point at. + + :param summary: what the gap is. + :param capability: the capability the gap concerns, or ``None`` when the + gap is against a mandatory scenario and so belongs to no capability. + """ + return cls(summary=summary, capability=capability) + + @property + def is_tracked(self) -> bool: + """Whether this deviation points at somewhere the gap is tracked.""" + return bool(self.issue) + + def as_json(self) -> dict[str, typing.Any]: + document: dict[str, typing.Any] = {"summary": self.summary} + # Omitted rather than null: the schema's `issue` is a uri-formatted + # string when present, so an untracked deviation leaves the key out. + if self.issue is not None: + document["issue"] = self.issue + if self.capability is not None: + document["capability"] = self.capability.tag + return document + + +@dataclass(frozen=True) +class TckConfig: + """Everything the TCK needs to test one provider. + + An adopting module supplies this through a session-scoped ``tck_config`` + fixture; the TCK owns everything else -- registering the provider, awaiting + events, resetting the backend between scenarios, tearing down. If you find + yourself writing test infrastructure, that is a defect in this package + rather than something for you to work around. + """ + + name: str + """Identifies the suite in test output, and scopes the OpenFeature domain + the TCK registers providers under so two suites in the same session do not + observe each other's providers. + + Use something that reads well in a failure message: ``"flagd-rpc"``, + ``"in-memory"``. + """ + + control: BackendControl + """The seam through which the TCK manipulates the backend. + + See :class:`~.control.BackendControl` for which implementation is right for + your provider. The short version: a provider with a real backend drives it + over the HTTP control API; a provider with no backend at all may control it + in-process. + """ + + new_provider: ProviderFactory + """Creates the provider under test, against a backend that is already + running and seeded with the canonical flag set. Called once per scenario. + + Return a configured but uninitialised provider; the TCK initialises it. + """ + + new_unavailable_provider: ProviderFactory | None = None + """Creates a provider pointed at a backend that does not exist. + + Used by the initialisation-failure scenarios, which assert that a provider + unable to reach its backend settles into ``ERROR`` rather than hanging or + raising out of registration. + + Point it at a closed port on localhost. Do not point it at the backend under + test -- that must stay up, and simulated outages belong to :attr:`control`. + Configure a short connection deadline: the scenario allows a bounded time + for the error, and a provider with a 30-second connect timeout will not make + it. + + Required only if :attr:`capabilities` includes + :attr:`Capability.UNAVAILABLE_INIT`. Leaving both out is the honest + configuration for a provider with no backend, and those scenarios are then + skipped with the reason reported. + """ + + capabilities: Collection[Capability] = field(default=DECLARABLE_CAPABILITIES) + """Which optional parts of the provider contract this provider supports. + + Typed as a ``Collection`` rather than a ``frozenset`` so that the obvious + thing to write -- a set literal, which is what the README shows -- is also + the correctly typed thing to write. It is normalised to a frozenset on + construction, so a list, a set or a generator all behave identically. + + Scenarios tagged with an undeclared capability are reported as skipped with + the reason, never as passed. Defaults to every *declarable* capability -- + :data:`~.capability.DECLARABLE_CAPABILITIES`, which excludes the reserved + tags no scenario carries -- and narrowing it surfaces gaps where widening + towards it hides them. + + Naming a reserved capability here is rejected at construction rather than + passed into a report. See :data:`~.capability.RESERVED_CAPABILITIES`. + + So is one this language's SDK cannot put the question for at all -- + ``@numeric-coercion`` where the language has a single numeric type, + ``@large-integers`` on a 32-bit accessor. That is a property of the SDK + rather than of the provider, so it is refused here rather than left for + every adopter to know and remember, and the error names the property. The + two refusals are deliberately not the same message, and the scenarios they + skip do not carry the same reason: see + :data:`~.capability.INEXPRESSIBLE_CAPABILITIES`, which is empty in Python. + """ + + known_deviations: Sequence[KnownDeviation] = () + """Gaps this provider is known to have, with each one tracked somewhere. + + An acknowledgement, not an excuse: the scenarios still fail and the results + payload still says so. See :class:`KnownDeviation`. + """ + + event_timeout: float = DEFAULT_EVENT_TIMEOUT + """Seconds to wait for a provider event. + + The single most important knob for a provider author, because providers + observe backend changes on wildly different timescales. A streaming provider + sees a configuration change in milliseconds; one polling every 30 seconds + may need most of a poll interval. Set it to comfortably exceed your + worst-case detection latency, or the suite reports timeouts that are really + just impatience. + + Scenarios can tighten this with the explicit ``within {int}ms`` step, which + always wins over this value. + """ + + ready_timeout: float = DEFAULT_READY_TIMEOUT + """Seconds to wait for a provider to reach ``READY`` during initialisation. + + Also the longest the suite waits on a direct ``shutdown`` or ``initialize`` + call before giving up on it and recording the wait as a failure, so that a + provider whose shutdown hangs on a backend that is gone fails its scenario + with a message rather than hanging the session. + """ + + def __post_init__(self) -> None: + problems: list[str] = [] + + if not self.name: + problems.append( + "name is required: it scopes the OpenFeature domain and identifies " + "the suite in test output" + ) + if self.control is None: + problems.append( + "control is required: see BackendControl for which implementation " + "fits your provider" + ) + if self.new_provider is None: + problems.append( + "new_provider is required: the TCK has nothing to test without it" + ) + + # Normalise whatever iterable the caller passed into a frozenset, so a + # set literal, a list or a generator all behave the same. + object.__setattr__(self, "capabilities", frozenset(self.capabilities)) + + unknown = [c for c in self.capabilities if not isinstance(c, Capability)] + if unknown: + problems.append( + f"unknown capabilities {unknown!r}: capabilities are the members of " + f"the Capability enum" + ) + + object.__setattr__(self, "known_deviations", tuple(self.known_deviations)) + + problems.extend(reserved_problems(self.capabilities)) + problems.extend(inexpressible_problems(self.capabilities)) + problems.extend(deviation_problems(self.known_deviations)) + + if ( + Capability.UNAVAILABLE_INIT in self.capabilities + and self.new_unavailable_provider is None + ): + problems.append( + "capabilities declares Capability.UNAVAILABLE_INIT but " + "new_unavailable_provider is None: the @unavailable scenarios need a " + "provider pointed at a backend that does not exist. Supply one, or " + "remove the capability so those scenarios are skipped with a reason" + ) + + if problems: + joined = "\n - ".join(problems) + msg = f"invalid TckConfig:\n - {joined}" + raise ValueError(msg) + + @property + def domain(self) -> str: + """The OpenFeature domain this suite registers its providers under. + + Suite-scoped rather than scenario-scoped on purpose. Registering a new + provider in the same domain replaces the previous one; a fresh domain + per scenario would leave every provider of the suite registered, which + for a provider holding a network connection means leaking one connection + per scenario. + """ + return f"tck/{self.name}" + + def declares(self, capability: Capability) -> bool: + return capability in self.capabilities + + @property + def sorted_capabilities(self) -> list[str]: + return sorted(c.tag for c in self.capabilities) + + +def reserved_problems(declared: Iterable[Capability]) -> list[str]: + """Refuse a reserved capability named in a configuration. + + A reserved capability gates no scenario, so declaring it cannot be verified + either way: the claim is about something nothing examined, and it would + reach the report's declaration, which the schema forbids. + + Refused rather than dropped quietly. The adopter wrote it down and meant + something by it, so a configuration silently different from the one they + wrote is worse than one that will not build -- and construction is where + their own code is still on the stack to say which line to fix. The + alternative, a warning, is a line of CI output nobody reads while an + untested capability goes on being asserted in a published report, which is + how this got into one in the first place. + """ + reserved = sorted( + capability.tag + for capability in declared + if isinstance(capability, Capability) and capability.reserved + ) + if not reserved: + return [] + declarable = " ".join(sorted(c.tag for c in DECLARABLE_CAPABILITIES)) + return [ + f"reserved capabilities {' '.join(sorted(set(reserved)))} cannot be declared: " + f"no scenario carries them, so the claim cannot be verified, cannot produce a " + f"skip, and would tell a reader of the report only that something was claimed " + f"and nothing examined. The declarable capabilities, which is what " + f"DECLARABLE_CAPABILITIES holds, are {declarable}" + ] + + +def inexpressible_problems(declared: Iterable[Capability]) -> list[str]: + """Refuse a capability this language's SDK cannot put the question for. + + Refused here rather than left to adopters, because leaving it to adopters + means every adopter in the language has to know a fact about their language + and remember to act on it. Three suites in one implementation each left the + same capability undeclared with its own comment restating the same property + of the language: three places to get right, every one of them re-paid by the + next adoption, and a single wrong one puts a claim in a report that no + scenario could have verified. Appendix F makes this the implementation's job + for exactly that reason. + + **The message names the property of the SDK, not the rule.** An adopter who + reaches this has done nothing wrong -- they declared a capability their + provider may well have -- so the error has to tell them something they could + not have known, and "the specification says you may not" is not it. + + Separate from :func:`reserved_problems` on purpose, and it stays separate + even though both end in the same refusal. A reserved capability is global and + temporary: nothing anywhere carries the tag, and the reservation expires when + the specification writes a scenario. An inexpressible one is this language's + and permanent: the scenarios exist and other languages pass them. Collapsing + them into one predicate would make the two indistinguishable at the only + moment anybody is looking. + """ + refused = [ + capability + for capability in declared + if isinstance(capability, Capability) and capability.inexpressible + ] + if not refused: + return [] + return [ + f"{capability.tag} cannot be declared in this language: {reason}. No " + f"provider in this SDK can be asked the question its scenarios put, so a " + f"declaration could not be verified either way, and its absence from a " + f"report says nothing about your provider. Its scenarios are skipped with " + f"that reason. This is not a reservation -- the scenarios exist and other " + f"languages run them -- and there is nothing for you to fix; it changes " + f"when the SDK does" + for capability in sorted(refused, key=lambda c: c.tag) + if (reason := capability.inexpressible_reason) is not None + ] + + +def deviation_problems(deviations: Sequence[KnownDeviation]) -> list[str]: + """Refuse a deviation that says nothing a consumer can use. + + The rules are deliberately narrow. A deviation is prose written by the + provider author for a human comparing providers, and no suite can check + prose; what it can check is that the prose is there and that the capability + it names is one a scenario could have been gated on. + + A reserved capability is refused for the same reason declaring one is: no + scenario carries the tag, so there is no failure and no skip for the + deviation to explain, and nothing it could be about. + """ + problems: list[str] = [] + + for index, deviation in enumerate(deviations): + if not deviation.summary or not deviation.summary.strip(): + problems.append( + f"known_deviations[{index}] has no summary: a deviation exists to " + f"say what the gap is, and one that does not say it leaves a " + f"consumer no better off than the bare skip or failure it " + f"accompanies. It is the one field the report schema requires" + ) + + capability = deviation.capability + if capability is None: + # Legitimate: the gap is against a mandatory, ungated scenario, + # which belongs to no capability. + continue + + if not isinstance(capability, Capability): + problems.append( + f"known_deviations[{index}] names unknown capability " + f"{capability!r}: capabilities are the members of the Capability " + f"enum" + ) + continue + + if capability.reserved: + problems.append( + f"known_deviations[{index}] names the reserved capability " + f"{capability.tag}: no scenario carries that tag, so nothing was " + f"failed or skipped for this deviation to explain and no result " + f"could show the gap. Remove it, or name the capability whose " + f"scenarios the gap actually affects" + ) + elif capability.inexpressible: + problems.append( + f"known_deviations[{index}] names {capability.tag}, which cannot " + f"be expressed in this language: {capability.inexpressible_reason}. " + f"A deviation says this provider fails something it is required to " + f"do, and no provider in this SDK can attempt these scenarios at " + f"all -- so the entry would attribute to your provider a gap that " + f"belongs to the language. The skip already carries that reason" + ) + + return problems + + +def capabilities_of(values: Iterable[Capability]) -> frozenset[Capability]: + """Convenience for building a capability set from any iterable.""" + return frozenset(values) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/control.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/control.py new file mode 100644 index 00000000..56f3a34c --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/control.py @@ -0,0 +1,149 @@ +"""The seam between the scenarios and whatever manipulates the backend.""" + +from __future__ import annotations + +import typing + +__all__ = [ + "BackendControl", + "ConnectionControl", + "ControlApi", + "UnsupportedControlError", + "unsupported_control", +] + +ControlApi = typing.Literal["http", "in-process"] +"""Which of the two control paths a run used, closed to the two the schema allows. + +Named so that a custom control can annotate its own property with it and have +the type checker refuse a third value -- ``"HTTP"``, ``"grpc"``, a typo -- before +it becomes a conformance report that fails validation with nothing to point at +locally. +""" + + +class UnsupportedControlError(RuntimeError): + """Raised when a backend cannot perform a control operation. + + It is always a test-configuration bug rather than a provider defect. The + scenarios needing connection control are gated behind + :attr:`Capability.STALE` and :attr:`Capability.UNAVAILABLE_INIT`, so + reaching an unsupported operation means a capability was declared that the + backend cannot back up. The TCK fails loudly on it rather than skipping, + because a silent no-op would report the scenario as passed. + """ + + +@typing.runtime_checkable +class BackendControl(typing.Protocol): + """How the TCK puts the backend under test into the states a scenario needs. + + Step definitions never talk to a backend directly. They talk to this + protocol, which is why the same Gherkin runs unchanged against a + containerised backend driven over HTTP and against a provider manipulated + in-process. Nothing below this line knows about ports, containers or + transports. + + **Which implementation is right for your provider.** If your provider talks + to a backend -- a server, a service, anything out of process -- drive it + over the HTTP control API described in ``control-api.yaml``. That API is the + normative contract for those providers, and it is what makes a conformance + claim portable: another language's TCK drives the same endpoints against the + same stack and must get the same answers. + + Do not write an in-process control that reaches into an external backend + through a side channel -- a test-only admin client, a shared database + handle, a hook inside the provider. It will pass, and it will prove nothing, + because the path it exercised is not the path the contract describes. + + In-process control exists for providers with *no* backend to contract with: + in-memory, environment-variable and file-based providers, where "the + backend" is a data structure in the same process. See + :class:`InProcessControl`. + """ + + def prepare_scenario(self) -> None: + """Bring the backend to the state every scenario starts from. + + Reachable, with flag state at the baseline of the canonical flag set. + Called once before each scenario. This is the TCK's only isolation + mechanism -- scenarios share one backend for the whole suite, and + containers are never restarted between them. + """ + + def change_flag(self) -> None: + """Mutate flag configuration so a conforming provider observes a change. + + Afterwards the provider must resolve a different value for + ``changing-flag``. Which value it changes to is deliberately + unspecified; the suite asserts only that the resolved value differs. + """ + + @property + def description(self) -> str: + """A short description of what is being controlled, for messages a human reads.""" + + @property + def control_api(self) -> ControlApi: + """Which path this control drove the backend through. + + ``"http"`` is the normative HTTP control API in ``control-api.yaml``. + ``"in-process"`` is the narrow allowance made for a provider with no + backend, where "the backend" is a data structure in this process -- see + :class:`InProcessControl`. + + **Required, and stated rather than inferred.** It is the one fact that + decides what everything else in a report is worth: the same scenarios + passing over the control API and passing through in-process manipulation + of a provider that *does* have a backend are not the same claim, and + this is the only field that separates them. Nothing outside a control can + tell the two apart -- a suite that guessed from the control's concrete + type would be right about the two controls in this package and silently + wrong about a custom one, which is exactly the case where the answer + matters. + + Nor would an absent value be neutral. Every run is one or the other, so + there is no third case an omitted value legitimately covers: it would + not be "no claim made" but an unfalsifiable one. A custom control states + it here and nothing downstream has to guess. + """ + + +@typing.runtime_checkable +class ConnectionControl(typing.Protocol): + """Implemented by a backend that can be cut off from the provider and restored. + + Separate from :class:`BackendControl` so a backend-less provider cannot + accidentally supply a no-op implementation: not implementing it at all is + the honest answer, and the TCK turns the resulting gap into an explicit, + reported skip. + """ + + def disconnect(self) -> None: + """Make the backend unreachable for the rest of the scenario, without stopping a container.""" + + def reconnect(self) -> None: + """Make the backend reachable again, preserving flag state. + + Preserving flag state is a requirement, not an implementation detail. An + outage must be observable as a change in availability and never as a + change in flag values, or the stale scenario cannot distinguish the two. + """ + + +def unsupported_control( + control: BackendControl, operation: str +) -> UnsupportedControlError: + """Build the error raised when a backend has no connection to control. + + The message names the fix, because the mistake it reports is always the same + one. + """ + return UnsupportedControlError( + f"{control.description} does not support {operation!r}. This is a " + f"test-configuration bug rather than a provider defect: a scenario needing " + f"connection control ran, so the suite declared Capability.STALE or " + f"Capability.UNAVAILABLE_INIT for a backend that cannot simulate an outage. " + f"Remove those capabilities from TckConfig.capabilities, or supply a " + f"BackendControl that also implements ConnectionControl." + ) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/extensions.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/extensions.py new file mode 100644 index 00000000..6424ee55 --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/extensions.py @@ -0,0 +1,363 @@ +"""Where the scenarios come from: the canonical set, plus whatever an adopter adds. + +A provider is rarely only a provider. flagd has ``fractional`` targeting, another +vendor has a proprietary rollout rule, and the behaviour of those is as worth +pinning as the contract they sit on top of. Verifying them used to mean standing +up a second harness: a second backend lifecycle, a second set of fixtures, a +second thing to keep working. The canonical suite ran, then something else ran, +and nothing tied the two together. + +So an adopter's own scenarios run **inside** the canonical suite instead -- +against the same provider instance, in the same backend lifecycle, with the same +step vocabulary available. Almost nothing is needed to make that happen, because +pytest already scans. It collects ``conftest.py`` on its own and pytest-bdd +resolves step definitions through the fixture system, so a step defined in the +adopter's ``conftest.py`` -- or in the test module itself -- is in scope for the +scenarios ``scenarios()`` generates there. The only thing pytest cannot find by +itself is the feature files, which is what this module finds: a directory named +``extensions`` beside the adopter's test module. + +That leaves one line, and it is the same line whether or not there are +extensions:: + + scenarios(*feature_paths()) + +**An extension can never stand in for a canonical scenario.** The two are told +apart by the uri each feature file is identified by, and this module derives that +uri from where the file *is* rather than taking what the runner offers: + +* ``gherkin/…`` is the packaged canonical assets, and nothing else; +* ``extensions/…`` is a discovered extension, whatever the adopter's own + directory layout under ``extensions`` looks like. + +The derivation is not decoration. pytest-bdd names a feature file by its parent +directory joined to its own name, so ``extensions/gherkin/errors.feature`` +arrives as ``gherkin/errors.feature`` -- the same uri as a canonical file. A +record of what ran holds one copy of a feature file per uri, so the second file +is never read and its scenarios are attributed to the first one's or to nothing +at all. + +The derivation is public, and the two problems it cannot rule out are reported +rather than raised, because the consumer of all of this is a conformance report +and that is not written here. Appendix F requires a report to say which scenarios +ran; nothing else can tell an adopter's question from the specification's. +""" + +from __future__ import annotations + +import importlib.resources +import inspect +import re +import typing +from pathlib import Path + +__all__ = [ + "CANONICAL_DIRECTORY", + "EXTENSIONS_DIRECTORY", + "EXTENSIONS_URI_PREFIX", + "canonical_root", + "canonical_tags", + "collision_problem", + "extension_root", + "feature_paths", + "is_canonical", + "is_canonical_uri", + "reserved_prefix_problem", + "uri_collisions", + "uri_for", +] + +_PACKAGE = "openfeature.contrib.tools.tck" + +_TAG = re.compile(r"@[\w-]+") +"""One Gherkin tag, as it appears on a tag line.""" + +CANONICAL_DIRECTORY = "gherkin" +"""The packaged directory the canonical feature files live in. + +Also the uri prefix they are identified by, which is why it is reserved: anyone +reading ``gherkin/errors.feature`` is entitled to assume it is the +specification's file rather than a local one that happened to land in a directory +of that name. + +The name is no longer chosen here. Appendix F fixes it: a canonical feature is +identified by its path **relative to the specification's asset directory**, and +``gherkin`` is the directory it occupies there. This suite used to vendor those +assets under a local name of its own and report that name instead, which is how +it came to answer ``features/errors.feature`` where Go -- consuming the same +assets as a module whose root *is* that directory -- answered +``gherkin/errors.feature``. A consumer joining two languages' results keys on the +uri and the scenario name, so the local name was the whole of the divergence. +""" + +EXTENSIONS_DIRECTORY = "extensions" +"""Where an adopter puts feature files of their own, beside their test module. + +Deliberately not the canonical name: a directory sharing it is how an extension +comes to occupy a canonical file's identity, and a convention that cannot collide +is worth more than one that reads slightly better. ``gherkin`` and ``extensions`` +are distinct, so that still holds. The name is the one Java's TCK scans for on the +classpath -- renamed to ``extensions`` there in the same round as here -- so an +adopter who ships a provider in both languages still puts the same directory in +both repositories. +""" + +EXTENSIONS_URI_PREFIX = "extensions" +"""The uri prefix an extension's scenarios are identified by. + +The Go and JavaScript suites mount extensions under the same prefix, so a +consumer holding reports from several languages applies one rule to tell an +adopter's scenario from the specification's. + +Equal to :data:`EXTENSIONS_DIRECTORY` today, and still a constant of its own: the +directory this suite scans and the prefix a report is keyed by are two separate +facts, and only the second is fixed by Appendix F. Collapsing them is exactly what +went wrong on the canonical half, where one name did both jobs and the reported +uri inherited a local choice. +""" + + +def _canonical_path() -> str: + """The packaged directory holding the canonical feature files. + + Deliberately not public. It used to be, as ``features_path()``, and the + public pair was a trap: ``scenarios(features_path())`` and + ``scenarios(*feature_paths())`` are both valid calls, differ by one + character at the call site, and the first one silently drops the extensions + directory. What that produces is a green run that examined fewer scenarios + than the adopter believes it did, which is the worst failure mode available + to a conformance suite -- worse than a red one, because nothing is there to + notice. Both flagd suites and the OFREP suite were calling it. + + :func:`canonical_root` is the supported way to reach the directory for + anything that is not "the scenarios to run". + """ + return str(importlib.resources.files(_PACKAGE) / CANONICAL_DIRECTORY) + + +def feature_paths() -> tuple[str, ...]: + """Return every feature directory this adoption should run. + + The canonical set, always, and an ``extensions`` directory beside the + calling module if there is one. Hand the result to pytest-bdd's + ``scenarios()``:: + + scenarios(*feature_paths()) + + That line does not change when an adopter adds an extension, which is what + makes adding one a matter of creating a directory rather than of configuring + anything. + + The calling module is located from the caller's frame, which is how + pytest-bdd locates it for ``scenarios()`` itself, so the two agree about + which module is adopting the suite. Call it from the test module rather than + from a helper: a helper's directory is what a helper would find. A caller + with no ``__file__`` -- an interactive session, an exec'd string -- gets the + canonical set alone. + """ + paths = [_canonical_path()] + directory = _caller_directory() + if directory is not None: + extensions = extension_root(directory) + if extensions is not None: + paths.append(str(extensions)) + return tuple(paths) + + +def extension_root(module_directory: Path) -> Path | None: + """The extension directory beside a test module, or ``None`` if there is none. + + ``None`` rather than a path that contributes nothing, so that an adopter + without extensions hands ``scenarios()`` exactly what they handed it before: + same scenarios, same count, same report. + """ + candidate = module_directory / EXTENSIONS_DIRECTORY + return candidate if candidate.is_dir() else None + + +def canonical_root() -> Path | None: + """The packaged canonical features directory, as a real path. + + ``None`` if the assets are not on the filesystem -- an installation from a + zipimport, say. Everything built on this degrades to "cannot tell", which is + the honest answer and never a false accusation. + """ + try: + return _resolve(Path(_canonical_path())) + except (OSError, TypeError): # pragma: no cover - assets outside a filesystem + return None + + +def canonical_tags() -> frozenset[str]: + """Every Gherkin tag the packaged canonical feature files carry. + + The specification's half of what a reservation is checked against: a tag is + reserved because no canonical scenario carries it, and this is the set that + says whether that is still true. Read off the packaged files rather than + off a collected run, so a ``-k`` or a ``--deselect`` cannot narrow a run + past it. An adopter's own features have no equivalent source -- the + directory is found from their test module -- so the other half of the check + reads them from what was collected. See + :func:`~.capability.expired_reservations`. + + **Tag lines only**, which is what tells a tag apart from the same word + written in prose. ``events.feature`` mentions ``@caching`` in a comment, + saying where those scenarios will go once they exist, so a scan that read + the whole file would report the reservation as expired on the strength of a + sentence about it -- and every adoption would then fail on a sentence. + + Recursive, because the shape of the canonical directory is the + specification's to change: a flat scan would answer "no tags" for a file + one directory down, which is silent under-collection, the failure mode the + rest of this module exists to stop. + + Empty when the assets are not reachable as files -- an installation from a + zipimport, say. Everything built on this then degrades to "cannot tell", + which is the honest answer and never a false accusation. + """ + root = canonical_root() + if root is None or not root.is_dir(): + return frozenset() + + tags: set[str] = set() + for feature in sorted(root.rglob("*.feature")): + for line in feature.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if stripped.startswith("@"): + tags.update(_TAG.findall(stripped)) + return frozenset(tags) + + +def is_canonical(path: Path) -> bool: + """Whether a feature file is one of the packaged canonical ones.""" + canonical = canonical_root() + return canonical is not None and _resolve(path).is_relative_to(canonical) + + +def is_canonical_uri(uri: str) -> bool: + """Whether a uri names a canonical feature file. + + The discriminator between a canonical scenario and an adopter's own wherever + it matters. Derived from the uri rather than carried beside it, so there is + no second fact to disagree with the first. + """ + return uri.startswith(f"{CANONICAL_DIRECTORY}/") + + +def uri_for(path: Path) -> str | None: + """The uri a feature file should be identified by. + + ``None`` when the file is neither canonical nor under an extension + directory, in which case the caller falls back to what pytest-bdd named it. + + Derived from the file's location rather than from pytest-bdd's + ``rel_filename``, which is the parent directory's name joined to the file's + own. That is what let ``extensions/gherkin/errors.feature`` present + itself as ``gherkin/errors.feature``: the same uri as a canonical file, and + a record of what ran holds one copy of a feature file per uri. + """ + resolved = _resolve(path) + + canonical = canonical_root() + if canonical is not None and resolved.is_relative_to(canonical): + return _uri(Path(CANONICAL_DIRECTORY) / resolved.relative_to(canonical)) + + for parent in resolved.parents: + if parent.name == EXTENSIONS_DIRECTORY: + return _uri(Path(EXTENSIONS_URI_PREFIX) / resolved.relative_to(parent)) + return None + + +def reserved_prefix_problem(uri: str, path: Path) -> str | None: + """Report a feature file claiming the canonical uri prefix without being canonical. + + The one thing the naming convention cannot rule out on its own: an adopter + who hands ``scenarios()`` a directory of their own named ``gherkin``. The + file is then named exactly as a canonical one would be, and a reader has no + way to tell that the specification did not write it. + + Returned rather than raised. The suite itself has no use for the answer -- + the scenarios run either way, and they are the adopter's to run -- so this is + for whatever writes a record of the run to refuse to publish one. + """ + if not is_canonical_uri(uri) or is_canonical(path): + return None + return ( + f"{uri} is not a canonical feature file -- it is {path} -- but it would be " + f"reported under the {CANONICAL_DIRECTORY}/ prefix, which is reserved for " + f"the packaged conformance assets. Move it into a directory named " + f"{EXTENSIONS_DIRECTORY} beside the test module, which feature_paths() " + f"finds on its own" + ) + + +def uri_collisions( + identified: typing.Iterable[tuple[str, Path]], +) -> dict[str, tuple[Path, ...]]: + """Feature files that would share one uri, keyed by that uri. + + Deriving the uri from the file's location removes the collision an adopter + is actually likely to hit, but it does not make one impossible. Two + extension roots contributing the same relative path to a single suite -- two + test modules sharing one ``tck_config`` from a conftest, each with an + ``extensions/vendor.feature`` -- still land on ``extensions/vendor.feature`` + twice, and so does an ``extensions`` directory nested inside another one. + + That has to be refused rather than resolved. A record of what ran holds one + copy of a feature file per uri, so the second file is never read: its + scenarios are attributed to the first file's where the names happen to + match, and go missing where they do not. The first is the silent form of + exactly the failure Java measured, and it is the one a consumer cannot + detect from the outside. + + Compared by resolved path, so the same file reached by two routes is one + file rather than a collision. + """ + files: dict[str, dict[Path, None]] = {} + for uri, path in identified: + files.setdefault(uri, {})[_resolve(path)] = None + return {uri: tuple(paths) for uri, paths in files.items() if len(paths) > 1} + + +def collision_problem(uri: str, paths: typing.Sequence[Path]) -> str: + """Say which files collided and what to do about it.""" + listed = ", ".join(str(path) for path in sorted(paths)) + return ( + f"{uri} is the uri of {len(paths)} different feature files -- {listed} -- " + f"and a record of what ran holds one copy of a feature file per uri, so " + f"one of them would be reported against the other's. Give them paths that " + f"differ below their {EXTENSIONS_DIRECTORY} directory" + ) + + +def _caller_directory() -> Path | None: + """The directory of the module two frames up, if it has a file.""" + frame = inspect.currentframe() + for _ in range(2): + if frame is None: # pragma: no cover - no Python frames to walk + return None + frame = frame.f_back + if frame is None: # pragma: no cover - called with no caller above + return None + file_name: typing.Any = frame.f_globals.get("__file__") + if not isinstance(file_name, str) or not file_name: + return None + return _resolve(Path(file_name)).parent + + +def _resolve(path: Path) -> Path: + try: + return path.resolve() + except OSError: # pragma: no cover - a path that cannot be resolved at all + return path + + +def _uri(path: Path) -> str: + """A relative path as a uri: slash-separated on every platform. + + These paths are assembled with ``pathlib``, so on Windows they arrive + backslash-separated. A uri is not, and the same string has to identify a + feature file wherever the suite ran or a run on Windows is not comparable + with one on Linux. + """ + return path.as_posix() diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/httpcontrol.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/httpcontrol.py new file mode 100644 index 00000000..2f6ef665 --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/httpcontrol.py @@ -0,0 +1,377 @@ +"""HTTP backend control: the normative control path for a provider with a real backend.""" + +from __future__ import annotations + +import threading +import time +import typing +import urllib.error +import urllib.parse +import urllib.request + +from .control import BackendControl, ConnectionControl, ControlApi + +__all__ = [ + "DEFAULT_CONFIGURATION", + "DEFAULT_STARTUP_TIMEOUT", + "ControlApiError", + "HttpControl", +] + +DEFAULT_CONFIGURATION = "default" +"""The configuration name every backend under test must support. + +It is the one that serves the canonical flag set the feature files assume. +""" + +DEFAULT_TIMEOUT = 30.0 +"""Seconds bounding a single control-API request. + +Control calls are local HTTP to a container on the same host; anything slower +than this is a wedged backend rather than a slow one. +""" + +DEFAULT_STARTUP_TIMEOUT = 60.0 +"""Seconds to wait for a stack and its control API to become reachable. + +The same default every language's TCK uses, so an adopter porting an adoption +between two of them does not find one of them more patient than the other. +""" + +_NOT_IMPLEMENTED = frozenset({404, 501}) +"""How a backend that does not implement ``/reset`` answers it, per the OpenAPI document.""" + +_SUPPORTED_SCHEMES = frozenset({"http", "https"}) + +_READY_PROBE_TIMEOUT = 5.0 +"""Seconds bounding a single readiness probe, so one wedged probe is not the whole wait.""" + +_READY_POLL_INTERVAL = 0.2 +"""Seconds between readiness probes.""" + + +class ControlApiError(RuntimeError): + """Raised when a control-API call fails or answers with an unexpected status. + + Always a defect in the stack under test or in its wiring, never a provider + defect -- so it is raised rather than swallowed. A control call that quietly + did nothing would leave the next scenario running against an unknown backend + state and reporting whatever it found as a conformance result. + """ + + +class HttpControl: + """Drives a backend under test over the HTTP control API in ``control-api.yaml``. + + This is the normative control path for any provider with a real backend, and + it is what makes a conformance claim portable: another language's TCK drives + the same endpoints against the same stack and must get the same answers. + + Built on :mod:`urllib.request` alone, so adopting the TCK pulls in no HTTP + client and no container library. Orchestrating the stack stays with the + adopting suite, where the vendor-specific knowledge already lives -- which + compose file, which services, which internal ports. + + **What it never does.** It never stops, kills or recreates a container. + Unavailability is simulated inside the running stack, through ``POST /stop``, + because container orchestrators assign host ports dynamically and cannot + reliably preserve them across a restart: a restarted backend generally comes + back on a different host port, silently invalidating every provider already + pointed at the old one, and the resulting failure looks like a flaky provider + rather than a broken test. Starting and stopping the stack itself belongs to + the adopting suite, once per session. + + **Scenario isolation.** :meth:`prepare_scenario` prefers ``POST /reset``, + which restores the flag baseline with no availability blip and therefore + cannot inject a spurious lifecycle event into the next scenario. That + operation is optional, and a backend that does not implement it answers 404 + or 501; the TCK then falls back to ``POST /start?config=...``, which also + resets flag state at the cost of a process restart. The fallback is probed + once and remembered for the rest of the suite. + + **No settle after a control call, ever.** Every state-changing endpoint -- + ``/start``, ``/change``, ``/reset`` -- owes the caller that the new state is + being served before it returns. A fixed delay here would buy silence rather + than correctness: it is un-tunable, because the window it covers is a + property of the backend and not of this client, and it hides the defect from + the one consumer positioned to notice. Where an adopter is stuck with a + backend that breaks the promise, the wait belongs in *that adoption*, set + explicitly and citing the defect, so that it disappears when the backend is + fixed instead of being inherited by every future adopter from here. + + **After a disconnect, ``/start`` rather than ``/reset``.** ``/reset`` is + specified to restore flag state, not to bring a stopped backend back up, so + a disconnect is recorded and the scenario that follows one is prepared with + ``/start``. + + Safe to share between suites, and it should be shared whenever they drive the + same backend: the disconnect bookkeeping is only correct if every operation + against one backend goes through one instance of this class. + """ + + def __init__( + self, + base_url: str, + *, + backend_configuration: str = DEFAULT_CONFIGURATION, + timeout: float = DEFAULT_TIMEOUT, + ) -> None: + """Build a control for the backend whose control API is rooted at ``base_url``. + + :param base_url: root of the control API, for example + ``http://localhost:32768``. It must be built from the dynamically + mapped host port of the control service, discovered after the stack + is up -- a stack under test must not pin host ports. + :param backend_configuration: the named flag configuration the backend + under test seeds. Defaults to :data:`DEFAULT_CONFIGURATION`, the + only name every backend must support and the one serving the + canonical flag set. Named for the backend because a report's + ``provider.configuration`` is a different thing entirely -- which + mode of the provider was tested. + :param timeout: seconds bounding a single control-API request. + """ + parsed = urllib.parse.urlsplit(base_url) + if parsed.scheme not in _SUPPORTED_SCHEMES or not parsed.netloc: + msg = ( + f"base_url {base_url!r} is not an http(s) URL. It is the root of the " + f"control API, built from the dynamically mapped host port of the " + f"control service, for example 'http://localhost:32768'" + ) + raise ValueError(msg) + + self._base_url = base_url.rstrip("/") + self._backend_configuration = backend_configuration + self._timeout = timeout + + self._lock = threading.Lock() + # None until the first /reset call tells us which way it went. + self._reset_supported: bool | None = None + # Set by any operation that may have left the backend down, so the next + # prepare_scenario starts it rather than merely resetting flag state. + self._backend_maybe_down = False + + @property + def control_api(self) -> ControlApi: + """Report that this control drives its backend over the HTTP control API. + + Every operation below is an HTTP request to ``control-api.yaml``, so + this is the one control in the package that can answer without + qualification. + """ + return "http" + + @property + def description(self) -> str: + return f"the backend at {self._base_url}, driven over the control API" + + @property + def base_url(self) -> str: + """The root of the control API this client drives.""" + return self._base_url + + def await_ready(self, timeout: float = DEFAULT_STARTUP_TIMEOUT) -> None: + """Block until the control API is ready to accept commands. + + A real readiness check against the control API itself rather than a + fixed pause, and the only wait in this class. ``GET /healthz`` is the + optional readiness path in ``control-api.yaml``; a backend that does not + implement it answers 404, which the document states *is* ready -- + readiness then rests on the control port accepting a connection, which + whatever started the stack has already established. A 503 is the control + API saying "not yet" and is retried. + + Called once, before the first command, by whatever brought the stack up. + There is deliberately no counterpart *after* a command: a pause there + would cover a window the control API is specified to close on its own, + and a suite that sleeps instead of holding the API to that promise stops + being able to detect when the promise breaks. + + :param timeout: seconds to keep probing before giving up. + :raises ControlApiError: if the control API is still not ready when the + timeout expires, quoting the last thing the probe saw. + """ + deadline = time.monotonic() + timeout + last = "no probe completed" + while True: + try: + status = self._probe_health() + except OSError as error: + last = f"not reachable: {error}" + else: + # 404 is "not implemented", which the document defines as ready. + if self._is_success(status) or status == 404: + return + last = f"answered HTTP {status}" + + if time.monotonic() >= deadline: + msg = ( + f"the control API at {self._base_url} was not ready within " + f"{timeout:g}s: GET /healthz {last}. The control API must be " + f"reachable before the first scenario and must stay reachable " + f"even while the backend is deliberately down" + ) + raise ControlApiError(msg) + time.sleep(_READY_POLL_INTERVAL) + + def _probe_health(self) -> int: + target = self._base_url + "/healthz" + # S310: __init__ rejects any base_url that is not http(s), and target is + # that validated base URL plus a literal path. + request = urllib.request.Request(target, method="GET") # noqa: S310 + try: + opened = urllib.request.urlopen(request, timeout=_READY_PROBE_TIMEOUT) # noqa: S310 + with opened as response: + response.read() + return int(response.status) + except urllib.error.HTTPError as error: + with error: + error.read() + return int(error.code) + + def prepare_scenario(self) -> None: + """Bring the backend to the state every scenario starts from. + + Prefers ``/reset`` and falls back to ``/start`` -- see the class + documentation for why, and for why a disconnect forces ``/start``. + """ + with self._lock: + must_start = self._backend_maybe_down or self._reset_supported is False + + if must_start: + self._start() + return + + status = self._call("/reset") + + if status in _NOT_IMPLEMENTED: + # The documented fallback. Remembered so the probe costs one request + # per suite rather than one per scenario. + with self._lock: + self._reset_supported = False + self._start() + return + + if not self._is_success(status): + msg = f"POST /reset on {self._base_url} returned {status}" + raise ControlApiError(msg) + + with self._lock: + self._reset_supported = True + + def change_flag(self) -> None: + """Mutate flag configuration so a conforming provider observes a change. + + ``/change`` must not return until the new value is actually being + served, and that promise is about the **backend**: once this returns, a + fresh evaluation against the backend resolves the new value. How long + the *provider under test* takes to notice is a property of its transport + -- streaming sees it in milliseconds, a poller may need most of an + interval -- and that is what the suite's event timeout is for. There is + deliberately no wait here: a backend that returns before it serves the + new value makes the provider's detection latency unmeasurable, because + the clock would start before there is anything to detect. + """ + self._require("/change") + + def disconnect(self) -> None: + """Make the backend unreachable, without touching any container. + + The backend *process* inside the still-running container is stopped. See + the class documentation for why that distinction is a requirement rather + than a preference. + """ + with self._lock: + self._backend_maybe_down = True + self._require("/stop") + + def reconnect(self) -> None: + """Make the backend reachable again, preserving flag state. + + Starting with the configuration already in effect restores the same + baseline, so the provider observes a change in availability and never a + change in flag values. + """ + self._start() + + # NO BINDING FOR ``POST /restart`` + # + # The endpoint simulates a *bounded* outage, and it is ``[OPTIONAL]`` in + # ``control-api.yaml`` because no shipped scenario reaches it. The + # disconnect/reconnect scenario is written as an unbounded outage -- "the + # connection is lost", then "the connection is restored" -- which is + # :meth:`disconnect` followed by :meth:`reconnect`, so a scenario ends the + # outage when it is ready rather than guessing in advance how long the + # provider needs to notice one. + # + # A binding nothing can call is dead surface that also misreports the + # contract, by implying every backend under test owes the endpoint. Go's + # client left it out for the same reason. What would bring it back is + # written down: a ``@caching`` scenario asserting what a stale provider + # serves *during* an outage needs the flag-state preservation that + # ``/restart`` has and ``/stop`` + ``/start`` does not. + + def _start(self) -> None: + self._require("/start", {"config": self._backend_configuration}) + with self._lock: + self._backend_maybe_down = False + + def _require(self, path: str, query: dict[str, str] | None = None) -> None: + """Perform a control call and fail on any non-2xx response.""" + status = self._call(path, query) + if not self._is_success(status): + msg = f"POST {path} on {self._base_url} returned {status}" + raise ControlApiError(msg) + + def _call(self, path: str, query: dict[str, str] | None = None) -> int: + """Perform one control-API request and return its status code. + + The response body is read and discarded: the control API's bodies are + human-readable messages the TCK is specified never to interpret, and + reading them lets the connection be released cleanly. + """ + target = self._base_url + path + if query: + target += "?" + urllib.parse.urlencode(query) + + # An empty body rather than none, so the request carries Content-Length + # even where a proxy in the stack insists on one. + # + # S310 wants the scheme audited before a URL is opened; __init__ rejects + # any base_url that is not http(s), and target is built from that + # validated base URL plus a literal path, so no other scheme can reach + # here. + request = urllib.request.Request(target, data=b"", method="POST") # noqa: S310 + + try: + with urllib.request.urlopen(request, timeout=self._timeout) as response: # noqa: S310 + response.read() + return int(response.status) + except urllib.error.HTTPError as error: + # A status the server chose to report as an error is still an answer, + # and /reset answering 404 is the documented way to say "not + # implemented" -- so this is a return, not a raise. + with error: + error.read() + return int(error.code) + except OSError as error: + msg = ( + f"control request POST {target} failed: {error}. The control API must " + f"stay reachable even while the backend is deliberately down, " + f"otherwise an outage cannot be ended" + ) + raise ControlApiError(msg) from error + + @staticmethod + def _is_success(status: int) -> bool: + return 200 <= status < 300 + + +if typing.TYPE_CHECKING: + # Static assertion, erased at runtime: HttpControl must satisfy both control + # protocols, the way Go's `var _ BackendControl = (*HTTPControl)(nil)` does. + # A method renamed out of the protocol fails type-checking rather than at the + # first scenario that needs it. + def _implements( + control: HttpControl, + ) -> tuple[BackendControl, ConnectionControl]: + return control, control diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/inprocess.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/inprocess.py new file mode 100644 index 00000000..f97d8b1a --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/inprocess.py @@ -0,0 +1,123 @@ +"""In-process backend control, for providers with no backend at all.""" + +from __future__ import annotations + +from openfeature.provider import FeatureProvider + +from .control import ControlApi +from .provider import ( + CHANGING_FLAG_KEY, + ControllableInMemoryProvider, + canonical_flag_set, + changing_flag, +) + +__all__ = ["InProcessControl"] + +_BASELINE = "foo" +_CHANGED = "bar" + + +class InProcessControl: + """Manipulates an in-process provider directly, with no backend and no HTTP. + + This exists so providers with nothing to connect to -- in-memory, + environment-variable and file-based providers -- can run the TCK. For those, + "the backend" is a data structure in the same process: seeding flags is + building a mapping, and changing one is an update on the live provider, so + the event the suite awaits is the provider's own + ``PROVIDER_CONFIGURATION_CHANGED`` rather than one the TCK synthesised. + + **This is not a shortcut for providers that do have a backend.** Reaching + into an external backend from inside the test process -- a test-only admin + client, a shared database handle, a hook in the provider -- produces a suite + that passes while proving nothing, because the path it exercised is not the + path the contract describes. Those providers drive the HTTP control API + instead. + + **Connection control.** :class:`InProcessControl` deliberately does not + implement :class:`~.control.ConnectionControl`. An in-memory provider has no + connection to lose, and pretending otherwise with a no-op would report the + ``@stale`` scenarios as passed. A suite using it leaves + :attr:`Capability.STALE` and :attr:`Capability.UNAVAILABLE_INIT` undeclared, + and those scenarios are skipped with the reason reported. + + **Ownership of the provider.** This type both seeds the flags and creates + the provider serving them, because in-process they are the same object: + :meth:`change_flag` has to reach the live instance to emit an event from it. + A suite therefore wires both through one control:: + + control = InProcessControl() + TckConfig( + name="in-memory", + control=control, + new_provider=control.new_provider, + capabilities={Capability.EVENTS, Capability.CONFIGURATION_CHANGE}, + ) + """ + + def __init__(self) -> None: + self._current: ControllableInMemoryProvider | None = None + self._changing_variant = _BASELINE + + @property + def description(self) -> str: + return "in-process control of an in-memory provider" + + @property + def control_api(self) -> ControlApi: + """Report how this backend was driven, for the conformance report. + + ``in-process`` is the narrow allowance for providers with no backend, + which is exactly what this control exists for. A provider that does have + a backend and reports this is claiming something it should not. + """ + return "in-process" + + def new_provider(self) -> FeatureProvider: + """Create the provider for the scenario about to run, at the baseline. + + Each call returns a fresh instance over a fresh copy of the canonical + flag set, which is what makes :meth:`prepare_scenario` nothing more than + dropping the previous reference. + """ + self._changing_variant = _BASELINE + self._current = ControllableInMemoryProvider(canonical_flag_set()) + return self._current + + def prepare_scenario(self) -> None: + """Drop the previous scenario's provider. + + That is the whole reset: the flag set is rebuilt per provider, so the + :meth:`new_provider` call that follows starts from an untouched + baseline. Clearing the reference rather than leaving it dangling means a + scenario that changes flags without creating a provider fails with a + clear message instead of mutating one that has already been shut down. + """ + self._current = None + + def change_flag(self) -> None: + """Flip ``changing-flag`` between its two variants on the live provider. + + The event the suite awaits is therefore the provider's own + ``PROVIDER_CONFIGURATION_CHANGED``, carrying ``changing-flag`` in + ``flags_changed``, and not a signal the TCK synthesised. + + Alternating rather than assigning a fixed variant keeps repeated calls + within one scenario meaningful; the suite asserts that the resolved + value differs, not what it became. + """ + if self._current is None: + msg = ( + "No in-memory provider exists for this scenario. In-process control " + "manipulates the provider itself, so the scenario must create one -- " + 'with "Given a stable provider" -- before any step that changes flag state.' + ) + raise RuntimeError(msg) + + self._changing_variant = ( + _BASELINE if self._changing_variant == _CHANGED else _CHANGED + ) + self._current.update_flag( + CHANGING_FLAG_KEY, changing_flag(self._changing_variant) + ) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/plugin.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/plugin.py new file mode 100644 index 00000000..6541ced4 --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/plugin.py @@ -0,0 +1,320 @@ +"""The pytest plugin: capability gating, scenario state, and the step vocabulary. + +Registered through the ``pytest11`` entry point, so installing this package is +all it takes for the step definitions to be available. pytest-bdd resolves steps +through the fixture system and fixtures from an installed plugin are visible to +every test, which is what keeps an adoption down to one fixture and one call to +``scenarios(*feature_paths())``. + +The same mechanism is what makes the suite extensible: a step an adopter defines +in their own ``conftest.py`` is resolved by the same fixture lookup as one this +plugin ships, so their scenarios need no glue and no second harness. See +:mod:`~.extensions`. + +It is also where the two things a run must refuse to do quietly are checked: +skipping a scenario whose capability was not declared happens loudly, with the +reason, and a scenario carrying a tag this package still calls reserved fails +the run outright rather than being skipped for a capability nobody may claim. +""" + +from __future__ import annotations + +import typing +from collections.abc import Iterable +from pathlib import Path + +import pytest + +from openfeature import api + +from .capability import Capability, capability_for_marker, expired_reservations +from .compose import ComposeBackend, RunningBackend, run_compose_backend +from .config import TckConfig +from .extensions import canonical_tags, is_canonical, uri_for +from .state import TckState + +# The step modules are registered as plugins in their own right, not merely +# imported. pytest-bdd's decorators inject a generated fixture name into the +# *defining* module's namespace, so a step is only visible to pytest once the +# module defining it is a registered plugin -- importing it here would run the +# decorators but leave those fixtures where pytest never looks. +pytest_plugins = [ + "openfeature.contrib.tools.tck.steps.provider_steps", + "openfeature.contrib.tools.tck.steps.flag_steps", + "openfeature.contrib.tools.tck.steps.event_steps", +] + + +def pytest_configure(config: pytest.Config) -> None: + """Register the capability tags as markers. + + pytest-bdd turns every Gherkin tag into a marker with + ``getattr(pytest.mark, tag)`` without registering it, which raises + ``PytestUnknownMarkWarning`` for each one -- noise at best, and a hard + failure in a project configured with ``-W error``. + """ + for capability in Capability: + config.addinivalue_line( + "markers", + f"{capability.value}: OpenFeature provider TCK capability {capability.tag}", + ) + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + """Fail the run if any scenario it collected carries a reserved tag. + + A reservation is a name held open for scenarios that do not exist yet, and + a reserved capability cannot be declared -- :class:`~.config.TckConfig` + refuses it. So a scenario carrying one reaches the capability gate below + and is skipped, for a capability nobody is permitted to claim: a question + put and silently withdrawn. Appendix F calls that the unclaimable + capability. The run stays green, the report stays well-formed, and nothing + else here notices, which is why it is checked rather than watched for. + + Two different mistakes end there. The check does not tell them apart, + because the consequence is identical and the message names both remedies: + + * the specification wrote the scenarios the tag was held open for and + :data:`~.capability.RESERVED_CAPABILITIES` has not followed -- + ``@targeting`` was reserved until spec revision ``26362f85`` gave it + three scenarios; + * an extension of the adopter's own used a reserved name for a tag of its + own, which is unclaimable from the day the file is written. + + This once fired for the canonical set alone, on the reasoning that only the + specification can expire a reservation. That much is true and it is beside + the point: an extension scenario carrying a reserved tag can never run and + can never be claimed either, which is exactly the failure being surfaced. + It arrives from the adopter rather than from upstream; the consequence does + not care. Go and Java check every scenario the run collected, and so does + this. + + Refused rather than worked around. Treating the tag as declarable here + would let a run claim a capability against a package that does not know the + tag exists, and the point of the check is that a human re-reads that set + against the specification -- or renames their own tag. + + **The two halves have different sources, and only one of them is + collected.** The canonical tags are read off the packaged files rather than + off the items, so a ``-k`` or a ``--deselect`` cannot narrow a run past the + specification's half. An extension's tags have no such source -- the + directory is found from the adopter's own test module, at the moment + :func:`~.extensions.feature_paths` is called -- so they come from what was + collected, as they do in every other language. In practice this hook is + handed the whole collection before anything is deselected, so a selection + does not narrow that half either; that is pytest's hook order rather than a + promise, and the half that must not be narrowable does not lean on it. + + What is read off the collection either way is whether this session runs the + conformance suite at all. This package is a ``pytest11`` plugin, so the + hook fires for every pytest run in an environment that merely has it + installed, and an unrelated test suite has no business failing over the + contents of these feature files. So the check waits for a canonical + scenario, which nothing but :func:`~.extensions.feature_paths` produces, + and it reads tags only from scenarios this suite is running. Go, Java and + JS have an explicit entry point and no equivalent problem. + """ + running = False + carried: set[str] = set() + for item in items: + feature = _suite_feature(item) + if feature is None: + continue + running = running or is_canonical(feature) + carried.update(f"@{marker.name}" for marker in item.iter_markers()) + + if not running: + return + + expired = expired_reservations(carried | canonical_tags()) + if not expired: + return + + named = " ".join(capability.tag for capability in expired) + raise pytest.UsageError( + f"reserved capabilities {named} are carried by scenarios this run " + f"collected, and a reserved capability cannot be declared -- so every " + f"scenario carrying one is skipped for a capability nobody is allowed " + f"to claim, and the report shows a gap the provider may not have. " + f"Either they came with the canonical feature files, and the scenarios " + f"they were held open for now exist: remove them from " + f"RESERVED_CAPABILITIES, so an adoption can declare them and be held " + f"to them. Or they came from a feature file of your own under " + f"extensions/, in which case pick tags of your own: a reserved tag " + f"gates nothing and can never be declared." + ) + + +def _suite_feature(item: pytest.Item) -> Path | None: + """The feature file behind a collected node, when this suite is what runs it. + + ``__scenario__`` is what pytest-bdd hangs on the function it generates, and + it is readable at collection without running a fixture. ``None`` for a node + that is not a scenario at all, and for a pytest-bdd scenario belonging to + some other suite that happens to share the session -- neither is this + suite's to fail. + + Which scenarios are this suite's is decided by + :func:`~.extensions.uri_for`, the same derivation a report identifies a + scenario by, so what is checked and what is recorded cannot disagree about + what the run consisted of. + """ + scenario = getattr(getattr(item, "function", None), "__scenario__", None) + filename = getattr(getattr(scenario, "feature", None), "filename", None) + if not filename: + return None + feature = Path(str(filename)) + return feature if uri_for(feature) is not None else None + + +@pytest.fixture(scope="session") +def tck_backend(compose_backend: ComposeBackend) -> typing.Iterator[RunningBackend]: + """The Compose stack under test, started once for the whole session. + + Depends on an adopter-supplied ``compose_backend`` fixture returning a + :class:`~.compose.ComposeBackend`, and yields the started stack's control + and endpoint. An adoption then reads:: + + @pytest.fixture(scope="session") + def compose_backend() -> ComposeBackend: + return ComposeBackend( + compose_file="tests/tck/docker-compose.yaml", + backend_ports=[8013], + ) + + @pytest.fixture(scope="session") + def tck_config(tck_backend: RunningBackend) -> TckConfig: + ... + + Session-scoped rather than module-scoped on purpose: two suites that drive + the same backend -- flagd's two resolvers, say -- must share one stack *and* + one control, because the control remembers whether a disconnect left the + backend down and two instances would each hold half of that knowledge. + + Lazy, like every fixture: a provider with no backend never requests it, + never defines ``compose_backend``, and never needs Docker or the ``compose`` + extra installed. See :mod:`~.compose`. + """ + yield from run_compose_backend(compose_backend) + + +@pytest.fixture +def tck_state(tck_config: TckConfig) -> typing.Iterator[TckState]: + """Per-scenario state, carried between step definitions.""" + # Resetting here rather than in an autouse fixture ties the reset to the + # scenarios that actually use the TCK, and guarantees it happens after the + # capability gate has had its say -- a skipped scenario never touches the + # backend. + tck_config.control.prepare_scenario() + state = TckState(config=tck_config) + yield state + state.teardown() + + +@pytest.fixture(autouse=True) +def _tck_capability_gate(request: pytest.FixtureRequest) -> None: + """Autouse wrapper around :func:`capability_gate`. + + A one-line fixture over a plain function, so the decision it makes can be + put under test without reaching inside a fixture object for the callable + pytest wrapped -- which is private, and has moved between pytest versions. + """ + capability_gate(request) + + +def capability_gate(request: pytest.FixtureRequest) -> None: + """Skip a scenario whose capability the provider did not declare. + + ``pytest.skip`` here reports the scenario as skipped **with the reason**, + which is exactly what the specification asks a TCK implementation to do. + Nothing about it can be mistaken for a pass. + + The gate keys off the node's markers rather than its requested fixtures. + pytest-bdd resolves a step's fixtures lazily, as each step runs, so + ``tck_config`` is not in ``request.fixturenames`` when this autouse fixture + is set up -- guarding on that silently disabled the gate and let + ``@unavailable`` scenarios run against a config that never declared it. + + Checking markers first also means the gate costs nothing, and instantiates + nothing, for tests that are not TCK scenarios. + + **A capability this SDK cannot express is skipped first, and says so.** Its + scenarios would be skipped anyway -- nothing may declare it, so nothing + does -- but with the wrong reason. "The provider does not declare it" reads + as a decision the provider made, and no provider in this language had one to + make; a reader of the report has to be able to tell those apart, because only + the first says anything about the provider. Checked before the declaration + loop rather than inside it so that a scenario gated by both kinds reports the + permanent, language-wide reason rather than whichever tag came first off the + marker iterator. Empty in Python; see + :data:`~.capability.INEXPRESSIBLE_CAPABILITIES`. + """ + gated = [ + capability + for marker in request.node.iter_markers() + if (capability := capability_for_marker(marker.name)) is not None + ] + if not gated: + return + + inexpressible = inexpressible_skip_reason(gated) + if inexpressible is not None: + pytest.skip(inexpressible) + + try: + config: TckConfig = request.getfixturevalue("tck_config") + except pytest.FixtureLookupError: + return + + undeclared = undeclared_skip_reason(gated, config) + if undeclared is not None: + pytest.skip(undeclared) + + +def inexpressible_skip_reason(gated: Iterable[Capability]) -> str | None: + """Why these scenarios cannot be run in this language at all, or ``None``. + + Says nothing about the provider, and says so, because the alternative + reading is the one a reader will reach for: a capability missing from a + report usually means the provider declined. Here nothing declined -- no + provider in this SDK could be asked -- and Appendix F makes telling those + two apart the implementation's job rather than the reader's. + + Deterministic when more than one applies: the tags are sorted, so the + message does not depend on the order markers come off a node. + """ + for capability in sorted(gated, key=lambda c: c.tag): + reason = capability.inexpressible_reason + if reason is not None: + return ( + f"{capability.tag} cannot be expressed by this SDK, so no provider " + f"in this language can be asked: {reason}. Nothing about the " + f"provider under test follows from this skip" + ) + return None + + +def undeclared_skip_reason( + gated: Iterable[Capability], config: TckConfig +) -> str | None: + """Why this provider is not being asked these scenarios, or ``None``. + + The other half of the pair, and the one that *is* about the provider: it + declined, and the declaration it did make is quoted so a reader can see what + was claimed instead. + """ + for capability in gated: + if not config.declares(capability): + return ( + f"provider does not declare capability {capability.tag}. " + f"Declared: {' '.join(config.sorted_capabilities) or '(none)'}" + ) + return None + + +@pytest.fixture(scope="session", autouse=True) +def _tck_release_providers() -> typing.Iterator[None]: + """Shut down whatever the suite registered once it is over.""" + yield + api.shutdown() + api.clear_providers() diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/provider.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/provider.py new file mode 100644 index 00000000..636b7e66 --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/provider.py @@ -0,0 +1,248 @@ +"""An in-memory provider that can be reconfigured at runtime, and the canonical flag set.""" + +from __future__ import annotations + +import importlib.resources +import json +import typing + +from openfeature.event import ProviderEventDetails +from openfeature.provider.in_memory_provider import ( + FlagStorage, + InMemoryFlag, + InMemoryProvider, +) + +__all__ = [ + "CHANGING_FLAG_KEY", + "ControllableInMemoryProvider", + "canonical_flag_set", + "canonical_flags_json", + "changing_flag", +] + +CHANGING_FLAG_KEY = "changing-flag" +"""The flag :meth:`BackendControl.change_flag` mutates.""" + +_CHANGING_BASELINE = "foo" +_CHANGING_CHANGED = "bar" + +_PACKAGE = "openfeature.contrib.tools.tck" + +_FLAG_DATA_DIRECTORY = "flag_data" +_CANONICAL_FLAGS_FILE = "canonical-flags.json" + +_COMMENT_KEY = "$comment" +"""The key the specification's assets carry prose under. + +Ignored at the document level, at a flag's level and among a flag's *variant +names* -- a "variant" called ``$comment`` is prose about the flag rather than a +variant of it -- and deliberately **not** inside a variant's value. A value is +opaque data the suite passes through: ``object-flag`` could perfectly well grow +a member of that name, and a loader that reached into a value to strip it would +serve an object no scenario expects. JavaScript's suite draws the line in the +same place, on purpose. +""" + + +class ControllableInMemoryProvider(InMemoryProvider): + """An in-memory provider whose flag set can be replaced at runtime. + + **Why this exists.** `Appendix A`_ of the specification requires an SDK's + in-memory provider to "support a means of updating the ``flag set``, + resulting in the emission of ``PROVIDER_CONFIGURATION_CHANGED`` events". The + Python SDK's :class:`~openfeature.provider.in_memory_provider.InMemoryProvider` + has no such method: it copies the flag mapping in its constructor and never + exposes a way to change it. + + Only half the machinery is missing, which is what makes this a small class + rather than a reimplementation. :class:`~openfeature.provider.AbstractProvider` + already supplies ``emit_provider_configuration_changed``, and the registry + already attaches the emitter, so all that is needed is a method that swaps + the mapping and emits. Everything about *resolution* -- variants, reasons, + ``FLAG_NOT_FOUND`` -- is still the SDK's. + + That makes this an honest reference for what the SDK's provider should grow, + rather than a competing implementation that could drift from it. + + .. _Appendix A: https://github.com/open-feature/spec/blob/main/specification/appendix-a-included-utilities.md + """ + + def update_flags(self, flags: FlagStorage) -> None: + """Replace the whole flag set and emit a configuration-change event. + + The event names the union of the previous and new keys, which is what + Appendix A asks for: a consumer caching evaluations needs to know + everything that might have changed, and a key that disappeared has + changed as much as one that was added. + """ + changed = sorted(set(self._flags) | set(flags)) + self._flags = dict(flags) + self.emit_provider_configuration_changed( + ProviderEventDetails( + flags_changed=changed, message="flag configuration changed" + ) + ) + + def update_flag(self, key: str, flag: InMemoryFlag[typing.Any]) -> None: + """Replace a single flag and emit a configuration-change event naming it.""" + updated = dict(self._flags) + updated[key] = flag + self._flags = updated + self.emit_provider_configuration_changed( + ProviderEventDetails( + flags_changed=[key], message="flag configuration changed" + ) + ) + + def flag(self, key: str) -> InMemoryFlag[typing.Any] | None: + """Return the flag currently registered under ``key``.""" + return self._flags.get(key) + + +def changing_flag(default_variant: str) -> InMemoryFlag[str]: + """Build ``changing-flag`` at one of its two variants. + + The one flag built by hand rather than decoded, because + :meth:`InProcessControl.change_flag` has to rebuild it at the *other* + variant and so has to name both. That the names here are the ones the + canonical file defines is asserted by the self-tests rather than assumed. + """ + return InMemoryFlag( + default_variant=default_variant, + variants={ + _CHANGING_BASELINE: _CHANGING_BASELINE, + _CHANGING_CHANGED: _CHANGING_CHANGED, + }, + ) + + +def canonical_flags_json() -> str: + """Return the canonical flag set as raw JSON, in the flagd flag-definition format. + + This is the flag set every scenario assumes, and a backend under test must + serve an equivalent one. The format is not what matters -- the keys, types, + variant names and resolved values are. Seed them however your backend seeds + flags. + + Exposed so an adopting provider can seed a backend from the canonical + definition rather than transcribing it, transcription being the usual way + the two drift apart. :func:`canonical_flag_set` takes its own advice. + """ + ref = ( + importlib.resources.files(_PACKAGE) + / _FLAG_DATA_DIRECTORY + / _CANONICAL_FLAGS_FILE + ) + return ref.read_text(encoding="utf-8") + + +def canonical_flag_set() -> FlagStorage: + """Return the canonical flag set as SDK in-memory flags. + + Decoded from ``flag_data/canonical-flags.json`` -- the JSON + :func:`canonical_flags_json` returns -- rather than transcribed, so that the + in-memory suites cannot drift from the file every other language seeds a + backend from. That file is published precisely so an adopter can "seed a + backend directly from the canonical definition rather than transcribing it, + transcription being the usual way the two drift apart"; this suite is an + adopter of it like any other. + + The drift it prevents is silent rather than loud. A fixture that has moved + away from the file makes the in-memory self-tests pass against a baseline + that is no longer the canonical one, so the suite verifies itself against + the wrong flags while reporting green -- and the report it publishes claims + the canonical set. + + Four properties of the file are load-bearing, and all four survive the + decoding: + + * ``missing-flag`` is absent, which is what the ``FLAG_NOT_FOUND`` scenario + tests. Adding it turns that scenario green for the wrong reason. + * no flag carries a ``context_evaluator``, so every evaluation reports reason + ``STATIC``. ``targeting-key-flag`` is the one flag in the file with a + ``targeting`` member, and this decoder reads only ``state``, ``variants`` + and ``defaultVariant`` -- so that flag is served at its ``miss`` default + whatever the context, like every other. That is deliberate rather than + pending: decoding a rule language would make this package a second + implementation of somebody else's evaluator, and the untargeted scenarios + are the ones it exists to serve. The consequence is that an in-memory + adoption must leave :attr:`~.capability.Capability.TARGETING` undeclared, + and its three scenarios are skipped with that reason. + * ``boolean-zero-flag``, ``integer-zero-flag`` and ``string-zero-flag`` + resolve to ``False``, ``0`` and ``""``. They are values, not absences, and + the falsy scenarios exist to catch a provider that cannot tell the + difference. Their ``zero``/``non-zero`` variant names are load-bearing + too, for an adoption declaring + :attr:`~.capability.Capability.VARIANTS`: the gated variant scenario + asserts the variant, where the falsy scenarios assert only the value. + * a number keeps the type it was written with. ``json.loads`` gives ``int`` + for ``10``, ``float`` for ``10.0`` and an arbitrary-precision ``int`` for + 2^53 - 1, and nothing here normalises either way, so + ``integral-float-flag`` stays the ``float`` ``10.0`` and + ``huge-integer-flag`` stays exact. Normalising integral floats to ``int`` + is the decoder bug that bit Java, and it makes the lossless-coercion + scenario pass without coercing anything. + + A variant's value is passed through untouched, which is both why the types + survive and why a ``$comment`` member *inside* an object value survives with + them -- see :data:`_COMMENT_KEY`. + + Raises: + ValueError: if the packaged file is not the shape this expects. + Unreachable for a pinned spec revision, because the file is copied + in from the submodule at build time: a failure here means the pinned + assets and this decoder disagree about the file's shape, which + moving the pin should have surfaced. + """ + return _decode_canonical_flags(canonical_flags_json()) + + +def _decode_canonical_flags(raw: str) -> FlagStorage: + """Turn the canonical flag file into in-memory flags.""" + document = json.loads(raw) + if not isinstance(document, dict): + msg = f"{_CANONICAL_FLAGS_FILE} is not a JSON object" + raise ValueError(msg) + + # Reading the one member this needs is what ignores $comment at the document + # level, along with every other part of the flagd format the suite has no + # use for. + definitions = document.get("flags") + if not isinstance(definitions, dict) or not definitions: + msg = f"{_CANONICAL_FLAGS_FILE} defines no flags" + raise ValueError(msg) + + return {key: _decode_flag(key, value) for key, value in definitions.items()} + + +def _decode_flag(key: str, definition: typing.Any) -> InMemoryFlag[typing.Any]: + """Turn one flag definition into an in-memory flag, or say why it cannot be.""" + if not isinstance(definition, dict): + msg = f"flag {key!r}: expected an object, got {type(definition).__name__}" + raise ValueError(msg) + + variants = definition.get("variants") + if not isinstance(variants, dict): + msg = f"flag {key!r}: variants is not an object" + raise ValueError(msg) + # Only the variant *names* are filtered. The values are not looked into. + variants = {name: value for name, value in variants.items() if name != _COMMENT_KEY} + + default_variant = definition.get("defaultVariant") + if not isinstance(default_variant, str) or default_variant not in variants: + msg = ( + f"flag {key!r}: default variant {default_variant!r} is not one of its " + f"variants ({', '.join(sorted(map(repr, variants)))})" + ) + raise ValueError(msg) + + raw_state = definition.get("state") + try: + state = InMemoryFlag.State(raw_state) + except ValueError: + allowed = ", ".join(member.value for member in InMemoryFlag.State) + msg = f"flag {key!r}: state {raw_state!r} is none of {allowed}" + raise ValueError(msg) from None + + return InMemoryFlag(default_variant=default_variant, variants=variants, state=state) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/state.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/state.py new file mode 100644 index 00000000..694858ee --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/state.py @@ -0,0 +1,233 @@ +"""Per-scenario state: what a scenario accumulates, and how it observes events. + +Separate from :mod:`plugin` so the step modules can import these types at the top +level. The step modules are loaded by the plugin as plugins in their own right, +and a step importing from the plugin module that loads it reads like a cycle even +where it is not one. +""" + +from __future__ import annotations + +import queue +import typing +from dataclasses import dataclass, field + +from openfeature.client import OpenFeatureClient +from openfeature.evaluation_context import EvaluationContext +from openfeature.event import EventDetails, ProviderEvent +from openfeature.flag_evaluation import FlagType +from openfeature.provider import FeatureProvider + +from .config import TckConfig + +__all__ = ["EvaluationRecord", "EventRecorder", "LifecycleRecord", "TckState"] + + +@dataclass +class EvaluationRecord: + """The outcome of one flag evaluation, flattened across the five typed calls.""" + + value: typing.Any = None + variant: str | None = None + reason: str | None = None + error_code: str | None = None + error_message: str | None = None + raised: BaseException | None = None + """The exception the call raised, if any. + + In Python an errored evaluation returns the code default in the details + rather than raising, so this stays ``None`` on the error paths the suite + exercises. It is what "no exception should have been thrown" asserts. + """ + + +@dataclass +class LifecycleRecord: + """The outcome of one direct call into the provider's lifecycle. + + The shutdown scenarios call the provider's own ``shutdown`` and + ``initialize`` rather than going through the SDK, because the SDK's + bookkeeping around them is Appendix B's business rather than this suite's. + Each call is recorded the same way an evaluation is -- what it raised, if + anything -- so that "no exception should have been thrown" reads one kind + of record for both, plus how long it took, which is what the prompt-shutdown + scenario bounds. + """ + + operation: str + """``shutdown`` or ``initialize``, for failure messages.""" + + duration: float + """Wall-clock seconds the call took to return, or to be given up on.""" + + raised: BaseException | None = None + """The exception the call raised, if any.""" + + +class EventRecorder: + """Captures the events of one type, in order, so a scenario consumes them one at a time. + + Consuming rather than merely observing is what makes the stale scenario + work: it awaits a ``PROVIDER_READY`` at the start and a second, different + ``PROVIDER_READY`` once the backend is back, and a recorder that only + remembered "ready has fired at some point" would report the second assertion + as satisfied by the first event. + + A queue rather than a list because a provider with a background thread -- + anything with a real backend -- delivers events from that thread while the + scenario waits on the main one. + """ + + def __init__(self, client: OpenFeatureClient, event: ProviderEvent) -> None: + self.event = event + self._client = client + self._events: queue.Queue[EventDetails] = queue.Queue() + self.last: EventDetails | None = None + + # The SDK replays a matching event on registration when the provider is + # already in the corresponding state, so a handler added after the + # provider became ready still observes its PROVIDER_READY. That is what + # lets the feature files register handlers after "Given a stable + # provider" without racing it. + client.add_handler(event, self._on_event) + + def _on_event(self, details: EventDetails) -> None: + self._events.put(details) + + def await_event(self, timeout: float) -> EventDetails: + """Consume the next event of this recorder's type.""" + try: + details = self._events.get(timeout=timeout) + except queue.Empty: + msg = ( + f"timed out after {timeout}s waiting for a {self.event.value} event. " + f"If the provider is simply slower than this to notice, raise " + f"TckConfig.event_timeout rather than treating it as a failure" + ) + raise AssertionError(msg) from None + self.last = details + return details + + def detach(self) -> None: + self._client.remove_handler(self.event, self._on_event) + + +@dataclass +class TckState: + """Everything one scenario accumulates.""" + + config: TckConfig + client: OpenFeatureClient | None = None + provider: FeatureProvider | None = None + """The provider under test, for the steps that call it directly. + + Everything else reaches the provider through :attr:`client`, which is how + an application would. The lifecycle and metadata steps are the exception: + they ask the provider itself, because what they verify is the provider's + own ``shutdown``, ``initialize`` and ``get_metadata`` rather than the SDK's + handling of them. + """ + flag_key: str | None = None + flag_type: FlagType | None = None + default_value: typing.Any = None + evaluation_context: EvaluationContext | None = None + """The context the scenario supplies to the evaluation, if it supplies one. + + ``None`` rather than an empty context, and the distinction is load-bearing: + one of the ``@targeting`` scenarios is specifically about a rule that cannot + match because no context was given at all, and a provider that would fall + over on an empty context rather than on an absent one is exactly what it is + looking for. So an unset context is passed to the SDK as ``None``, which is + what an application calling the two-argument form sends. + """ + last: EvaluationRecord | None = None + lifecycle: list[LifecycleRecord] = field(default_factory=list) + """Every direct lifecycle call this scenario made, in order.""" + remembered: typing.Any = None + has_memory: bool = False + recorders: dict[ProviderEvent, EventRecorder] = field(default_factory=dict) + + def require_client(self) -> OpenFeatureClient: + if self.client is None: + msg = ( + "no provider has been registered in this scenario: a " + '"Given a stable provider" or "Given a unavailable provider" step ' + "must come first" + ) + raise AssertionError(msg) + return self.client + + def require_provider(self) -> FeatureProvider: + if self.provider is None: + msg = ( + "no provider has been registered in this scenario: a " + '"Given a stable provider" or "Given a unavailable provider" step ' + "must come first" + ) + raise AssertionError(msg) + return self.provider + + def require_shutdown(self) -> LifecycleRecord: + """The most recent direct ``shutdown`` call, for the steps that bound it.""" + for record in reversed(self.lifecycle): + if record.operation == "shutdown": + return record + msg = ( + "the provider has not been shut down in this scenario: a " + '"When the provider is shut down" step must come first' + ) + raise AssertionError(msg) + + def raised(self) -> list[tuple[str, BaseException]]: + """Every call into the provider that raised, as (what was called, exception). + + The evaluation and the lifecycle calls are recorded separately, since + they carry different things, but "did anything the scenario asked of + the provider raise" is one question and this is where it is answered. + """ + raised: list[tuple[str, BaseException]] = [ + (record.operation, record.raised) + for record in self.lifecycle + if record.raised is not None + ] + if self.last is not None and self.last.raised is not None: + raised.append(("the evaluation", self.last.raised)) + return raised + + def has_called_provider(self) -> bool: + """Whether the scenario has asked anything of the provider yet.""" + return self.last is not None or bool(self.lifecycle) + + def require_flag(self) -> tuple[str, FlagType, typing.Any]: + if self.flag_key is None or self.flag_type is None: + msg = ( + "no flag has been declared in this scenario: a " + '"Given a -flag with key ... and a default value ..." step ' + "must come first" + ) + raise AssertionError(msg) + return self.flag_key, self.flag_type, self.default_value + + def require_evaluation(self) -> EvaluationRecord: + if self.last is None: + msg = ( + "no flag has been evaluated in this scenario: a " + '"When the flag was evaluated with details" step must come first' + ) + raise AssertionError(msg) + return self.last + + def require_recorder(self, event: ProviderEvent) -> EventRecorder: + recorder = self.recorders.get(event) + if recorder is None: + msg = ( + f"no handler was registered for {event.value} in this scenario: a " + '"Given a event handler" step must come first' + ) + raise AssertionError(msg) + return recorder + + def teardown(self) -> None: + for recorder in self.recorders.values(): + recorder.detach() + self.recorders.clear() diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/steps/__init__.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/steps/__init__.py new file mode 100644 index 00000000..6581ee86 --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/steps/__init__.py @@ -0,0 +1,11 @@ +"""The shared step vocabulary. + +Each module here is registered as a pytest plugin by the TCK's own plugin, which +is what makes the steps visible: pytest-bdd's decorators inject a generated +fixture name into the *defining* module's namespace, so a step only reaches +pytest once its module is a registered plugin. + +Deliberately empty of imports. Pulling the submodules in here would import them +before pytest loads them as plugins, and pytest cannot rewrite assertions in a +module that is already imported. +""" diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/steps/event_steps.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/steps/event_steps.py new file mode 100644 index 00000000..6b635699 --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/steps/event_steps.py @@ -0,0 +1,165 @@ +"""Steps covering provider events, connection loss and client status.""" + +from __future__ import annotations + +from pytest_bdd import given, parsers, then, when + +from openfeature.event import ProviderEvent +from openfeature.provider import ProviderStatus + +from ..control import ConnectionControl, unsupported_control +from ..state import EventRecorder, TckState + +__all__ = [ + "an_event_handler", + "an_event_was_fired", + "the_client_should_be_in_state", + "the_connection_is_lost", + "the_connection_is_restored", + "the_event_handler_should_have_been_executed", + "the_event_handler_should_have_been_executed_within", + "the_flag_should_be_part_of_the_event_payload", +] + +_EVENT_BY_NAME: dict[str, ProviderEvent] = { + "ready": ProviderEvent.PROVIDER_READY, + "stale": ProviderEvent.PROVIDER_STALE, + "error": ProviderEvent.PROVIDER_ERROR, + "change": ProviderEvent.PROVIDER_CONFIGURATION_CHANGED, +} + +_STATUS_BY_NAME: dict[str, ProviderStatus] = { + "ready": ProviderStatus.READY, + "stale": ProviderStatus.STALE, + "error": ProviderStatus.ERROR, +} + + +def _event(name: str) -> ProviderEvent: + try: + return _EVENT_BY_NAME[name] + except KeyError: + msg = f"unknown event kind {name!r}" + raise AssertionError(msg) from None + + +@given(parsers.re(r"^an? (?Pready|stale|error|change) event handler$")) +def an_event_handler(tck_state: TckState, kind: str) -> None: + """Attach a recorder for one event type. + + Handlers are attached after the provider is registered, which the SDK + handles by replaying a matching event on registration when the provider is + already in the corresponding state. That is why "Given a stable provider" + followed by "And a ready event handler" is not a race. + """ + event = _event(kind) + if event in tck_state.recorders: + return + client = tck_state.require_client() + tck_state.recorders[event] = EventRecorder(client, event) + + +@when(parsers.re(r"^a (?Pready|stale|error|change) event was fired$")) +def an_event_was_fired(tck_state: TckState, kind: str) -> None: + """Consume an event, so a later assertion observes the next one rather than this. + + The stale scenario depends on it: it consumes the initial ``PROVIDER_READY`` + here and then asserts a second, distinct one once the backend is back. + """ + recorder = tck_state.require_recorder(_event(kind)) + recorder.await_event(tck_state.config.event_timeout) + + +@then( + parsers.re( + r"^the (?Pready|stale|error|change) event handler should have been executed$" + ) +) +def the_event_handler_should_have_been_executed(tck_state: TckState, kind: str) -> None: + recorder = tck_state.require_recorder(_event(kind)) + recorder.await_event(tck_state.config.event_timeout) + + +@then( + parsers.re( + r"^the (?Pready|stale|error|change) event handler should have been " + r"executed within (?P\d+)ms$" + ) +) +def the_event_handler_should_have_been_executed_within( + tck_state: TckState, kind: str, millis: str +) -> None: + """Bound the wait explicitly. + + The scenarios using this assert promptness, not merely eventual arrival: a + provider that cannot reach its backend has to report that fact quickly, + because an application blocked on provider registration is down. The bound + therefore overrides ``event_timeout`` rather than being clamped by it. + """ + recorder = tck_state.require_recorder(_event(kind)) + recorder.await_event(int(millis) / 1000.0) + + +@then("the flag should be part of the event payload") +def the_flag_should_be_part_of_the_event_payload(tck_state: TckState) -> None: + """Assert the configuration-change event named the flag that changed. + + Naming the changed flags is what makes the event actionable: a consumer + caching evaluations needs to know what to invalidate, and an event carrying + no keys forces it to invalidate everything. + """ + key, _flag_type, _default = tck_state.require_flag() + recorder = tck_state.require_recorder(ProviderEvent.PROVIDER_CONFIGURATION_CHANGED) + + if recorder.last is None: + msg = ( + "no configuration-change event has been consumed in this scenario: a " + '"the change event handler should have been executed" step must come first' + ) + raise AssertionError(msg) + + changed = recorder.last.flags_changed or [] + if key in changed: + return + + if not changed: + msg = ( + f"the configuration-change event carried no changed flags, expected it to " + f"name {key!r}" + ) + else: + msg = f"the configuration-change event named {changed}, expected it to include {key!r}" + raise AssertionError(msg) + + +def _connection_control(tck_state: TckState, operation: str) -> ConnectionControl: + control = tck_state.config.control + if not isinstance(control, ConnectionControl): + raise unsupported_control(control, operation) + return control + + +@when("the connection is lost") +def the_connection_is_lost(tck_state: TckState) -> None: + _connection_control(tck_state, "disconnect").disconnect() + + +@when("the connection is restored") +def the_connection_is_restored(tck_state: TckState) -> None: + _connection_control(tck_state, "reconnect").reconnect() + + +@then(parsers.re(r"^the client should be in (?Pready|stale|error) state$")) +def the_client_should_be_in_state(tck_state: TckState, name: str) -> None: + """Assert the provider status the client reports. + + Checked after the corresponding event has been consumed, and the SDK writes + provider status before running handlers, so no polling is needed: if the + event arrived, the status is already current. + """ + client = tck_state.require_client() + expected = _STATUS_BY_NAME[name] + actual = client.get_provider_status() + if actual != expected: + msg = f"client reports status {actual}, expected {expected}" + raise AssertionError(msg) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/steps/flag_steps.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/steps/flag_steps.py new file mode 100644 index 00000000..98ecf0de --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/steps/flag_steps.py @@ -0,0 +1,320 @@ +"""Steps that declare, evaluate and assert flags.""" + +from __future__ import annotations + +import typing +from collections.abc import Callable + +from pytest_bdd import given, parsers, then, when + +from openfeature.evaluation_context import EvaluationContext +from openfeature.flag_evaluation import FlagType + +from ..capability import Capability +from ..state import EvaluationRecord, TckState +from ..values import describe, parse_flag_type, parse_value, values_equal + +__all__ = [ + "a_context_containing_a_targeting_key", + "a_flag_with_key_and_default", + "no_exception_should_have_been_thrown", + "the_error_code_should_be", + "the_error_message_should_be_empty", + "the_flag_was_evaluated_with_details", + "the_flag_was_modified", + "the_reason_should_be", + "the_resolved_object_value_should_contain", + "the_resolved_value_is_remembered", + "the_resolved_value_should_be", + "the_resolved_value_should_have_changed", + "the_variant_should_be", +] + + +@given( + parsers.re( + r'^an? (?P[A-Za-z]+)-flag with key "(?P[^"]*)" ' + r'and a default value "(?P[^"]*)"$' + ) +) +def a_flag_with_key_and_default( + tck_state: TckState, flag_type: str, key: str, default: str +) -> None: + """Declare the flag the scenario is about, and the type it is requested as. + + The two are independent on purpose: most of ``errors.feature`` asks for a + flag as a type it is not. + """ + parsed_type = parse_flag_type(flag_type) + tck_state.flag_key = key + tck_state.flag_type = parsed_type + tck_state.default_value = parse_value(parsed_type, default) + + +@given( + parsers.re(r'^a context containing a targeting key with value "(?P[^"]*)"$') +) +def a_context_containing_a_targeting_key(tck_state: TckState, value: str) -> None: + """Supply the evaluation context the resolve call is made with. + + The wording is `Appendix B + `_'s, + which the flagd testkit in this repository already carries a step definition + for, because a second way to say "a context containing a targeting key" is + the divergence Appendix F exists to prevent. + + Only the targeting key, for now. ``targeting-key-flag``'s rule keys on it, + so it is what the canonical set can observe arriving; a custom attribute + would need a second flag whose rule keys on one, which Appendix F lists as + a known gap. + """ + context = tck_state.evaluation_context + if context is None: + tck_state.evaluation_context = EvaluationContext(targeting_key=value) + else: + context.targeting_key = value + + +@when("the flag was evaluated with details") +def the_flag_was_evaluated_with_details(tck_state: TckState) -> None: + """Resolve the declared flag through the typed client call matching its type.""" + client = tck_state.require_client() + key, flag_type, default = tck_state.require_flag() + + # Annotated explicitly: the five typed getters have different signatures, so + # an unannotated mapping infers a value type mypy will not let us call. + calls: dict[ + FlagType, Callable[[str, typing.Any, EvaluationContext | None], typing.Any] + ] = { + FlagType.BOOLEAN: client.get_boolean_details, + FlagType.STRING: client.get_string_details, + FlagType.INTEGER: client.get_integer_details, + FlagType.FLOAT: client.get_float_details, + FlagType.OBJECT: client.get_object_details, + } + + record = EvaluationRecord() + try: + # Passed positionally and unconditionally, ``None`` included: the SDK's + # own default for the parameter is ``None``, so a scenario that declared + # no context sends what a two-argument call would. Requirement 2.2.1 + # makes the context a parameter of every resolve method, and until the + # evaluation-context scenarios landed no scenario supplied one -- so a + # provider that threw on any context, or serialised it into a malformed + # request, passed the whole suite. + details = calls[flag_type](key, default, tck_state.evaluation_context) + except BaseException as exc: # recorded here, asserted on by its own step + record.raised = exc + record.value = default + else: + record.value = details.value + record.variant = details.variant + record.reason = str(details.reason) if details.reason is not None else None + record.error_code = ( + details.error_code.value if details.error_code is not None else None + ) + record.error_message = details.error_message + + tck_state.last = record + + +@then(parsers.re(r'^the resolved details value should be "(?P[^"]*)"$')) +def the_resolved_value_should_be(tck_state: TckState, expected: str) -> None: + _key, flag_type, _default = tck_state.require_flag() + record = tck_state.require_evaluation() + wanted = parse_value(flag_type, expected) + + if not values_equal(wanted, record.value): + detail = ( + f" (the client also reported: {record.error_message})" + if record.error_message + else "" + ) + msg = ( + f"flag {tck_state.flag_key!r} resolved to {describe(record.value)}, " + f"expected {describe(wanted)}{detail}" + ) + raise AssertionError(msg) + + +@then(parsers.re(r'^the variant should be "(?P[^"]*)"$')) +def the_variant_should_be(tck_state: TckState, expected: str) -> None: + """Assert the resolved variant, for an adoption that declared ``@variants``. + + Reached only through the gated scenario, so the failure message says which + of the two readings applies: a variant lost in transit is a defect, while a + backend with no variant concept at all should withhold the capability rather + than fail here. Requirement 2.2.4 is a ``SHOULD``. + """ + record = tck_state.require_evaluation() + if record.variant != expected: + msg = ( + f"variant was {record.variant!r}, expected {expected!r}. A variant that " + f"does not survive the trip from the backend is one of the easiest parts " + f"of the contract to drop -- but if this backend has no variant concept " + f"for a plain flag, withhold {Capability.VARIANTS.tag} instead: " + f"requirement 2.2.4 is a SHOULD and types.md types the field as optional" + ) + raise AssertionError(msg) + + +@then(parsers.re(r'^the reason should be "(?P[^"]*)"$')) +def the_reason_should_be(tck_state: TckState, expected: str) -> None: + record = tck_state.require_evaluation() + if record.reason != expected: + msg = f"reason was {record.reason!r}, expected {expected!r}" + raise AssertionError(msg) + + +@then(parsers.re(r'^the error-code should be "(?P[^"]*)"$')) +def the_error_code_should_be(tck_state: TckState, expected: str) -> None: + """Assert the reported error code, where the empty string means none at all. + + The empty case matters as much as the populated ones. A provider that + reports a plausible value with no error code is the failure mode the suite + is most concerned with, because the application has no way to notice. + """ + record = tck_state.require_evaluation() + actual = record.error_code or "" + + if actual == expected: + return + + if expected == "": + msg = f"error-code was {actual!r}, expected none" + elif actual == "": + msg = ( + f"no error-code was reported, expected {expected!r}. Returning a value " + f"without an error code leaves the application unable to tell that " + f"anything went wrong" + ) + else: + msg = f"error-code was {actual!r}, expected {expected!r}" + raise AssertionError(msg) + + +@then("the error message should be empty") +def the_error_message_should_be_empty(tck_state: TckState) -> None: + """Assert no error message was reported (requirement 2.3.2). + + Asserted on the success paths, where a message contradicts the value beside + it: an application reading the message will believe the wrong one of the + two signals. ``None`` and ``""`` are both "none": the SDK's resolution + details default the field to ``None`` and a provider that writes the empty + string has said the same thing. + """ + record = tck_state.require_evaluation() + if record.error_message: + msg = ( + f"an error message was reported alongside a successful evaluation: " + f"{record.error_message!r}. A value and an error message are two " + f"contradictory signals, and the application cannot tell which to believe" + ) + raise AssertionError(msg) + + +@then("no exception should have been thrown") +def no_exception_should_have_been_thrown(tck_state: TckState) -> None: + """Assert that nothing the scenario asked of the provider raised. + + That is the evaluation, if there was one, and every direct lifecycle call: + each records what it raised rather than propagating it, and this is the + one step that reads those records back. + + In Python an errored evaluation returns the code default in the details and + does not raise, so this holds on the error paths too. A provider that raises + instead takes the calling application down with it -- and one that raises + from ``shutdown`` does so from the application's own shutdown, where an + exception is least welcome. Both are what the feature files forbid. + """ + if not tck_state.has_called_provider(): + msg = ( + "nothing has been asked of the provider in this scenario: a " + '"When the flag was evaluated with details" or "When the provider is ' + 'shut down" step must come first' + ) + raise AssertionError(msg) + + raised = tck_state.raised() + if raised: + what, exc = raised[0] + msg = ( + f"{what} raised {exc!r}. A provider must return from an evaluation with a " + f"value and an error code, and from a lifecycle call quietly -- never raise" + ) + raise AssertionError(msg) from exc + + +@then("the resolved object value should contain") +def the_resolved_object_value_should_contain( + tck_state: TckState, datatable: list[list[str]] +) -> None: + """Assert members of a structured value, each with its own expected type.""" + record = tck_state.require_evaluation() + header, *rows = datatable + + if header != ["key", "type", "value"]: + msg = f"expected a data table with columns key, type, value; got {header}" + raise AssertionError(msg) + + if not isinstance(record.value, dict): + msg = ( + f"resolved object value is {describe(record.value)}, which has no members " + f"to check" + ) + raise AssertionError(msg) + + for key, raw_type, raw_value in rows: + wanted = parse_value(parse_flag_type(raw_type), raw_value) + if key not in record.value: + msg = f"resolved object value has no member {key!r}" + raise AssertionError(msg) + actual = record.value[key] + if not values_equal(wanted, actual): + msg = f"object member {key!r} was {describe(actual)}, expected {describe(wanted)}" + raise AssertionError(msg) + + +@when("the resolved value is remembered") +def the_resolved_value_is_remembered(tck_state: TckState) -> None: + """Store the current value so a later step can assert it changed.""" + record = tck_state.require_evaluation() + tck_state.remembered = record.value + tck_state.has_memory = True + + +@then("the resolved details value should have changed") +def the_resolved_value_should_have_changed(tck_state: TckState) -> None: + """Assert that re-evaluation produced a different value. + + This is the half of the configuration-change contract providers actually get + wrong. Emitting ``PROVIDER_CONFIGURATION_CHANGED`` and then continuing to + resolve the old value is worse than emitting nothing, because the + application acted on a signal that was not true. + """ + record = tck_state.require_evaluation() + if not tck_state.has_memory: + msg = ( + "no value was remembered in this scenario: a " + '"the resolved value is remembered" step must come first' + ) + raise AssertionError(msg) + + if values_equal(tck_state.remembered, record.value): + msg = ( + f"the resolved value is still {describe(record.value)} after the " + f"configuration changed. The change was signalled but not applied, so the " + f"event told the application something untrue" + ) + raise AssertionError(msg) + + +@when("the flag was modified") +def the_flag_was_modified(tck_state: TckState) -> None: + """Change flag configuration on the backend.""" + control = tck_state.config.control + try: + control.change_flag() + except Exception as exc: + msg = f"could not change flag configuration on {control.description}: {exc}" + raise AssertionError(msg) from exc diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/steps/provider_steps.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/steps/provider_steps.py new file mode 100644 index 00000000..cf84f8ea --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/steps/provider_steps.py @@ -0,0 +1,251 @@ +"""Steps that put a provider under test, and the ones that talk to it directly.""" + +from __future__ import annotations + +import concurrent.futures +import contextlib +import time +from collections.abc import Callable + +from pytest_bdd import given, parsers, then, when + +from openfeature import api +from openfeature.evaluation_context import EvaluationContext + +from ..state import LifecycleRecord, TckState + +__all__ = [ + "a_stable_provider", + "an_unavailable_provider", + "the_provider_is_initialized_again", + "the_provider_is_shut_down", + "the_provider_metadata_name_should_not_be_empty", + "the_shutdown_should_have_completed_within", +] + + +@given(parsers.re(r"^an? stable provider$")) +def a_stable_provider(tck_state: TckState) -> None: + """Register the provider under test against the running, seeded backend. + + ``api.set_provider_and_wait`` initialises the provider before it returns and + dispatches ``PROVIDER_READY``, so by the time this step returns the provider + is ready and every scenario that follows can assume it. A suite that started + evaluating before that would report races in the TCK as defects in the + provider. + + It has to be the waiting variant. Plain ``api.set_provider`` registers and + initialises on a worker thread, returning long before the provider is up, so + the very first evaluation of every scenario answers ``PROVIDER_NOT_READY`` -- + a TCK defect that reads exactly like a provider that cannot resolve + anything. + """ + config = tck_state.config + provider = config.new_provider() + if provider is None: + msg = "TckConfig.new_provider returned None" + raise AssertionError(msg) + + try: + _call_within( + lambda: api.set_provider_and_wait(provider, config.domain), + config.ready_timeout, + ) + except TimeoutError: + msg = ( + f"the provider did not become ready within {config.ready_timeout}s. The backend " + f"is up and seeded at this point, so either initialisation is genuinely hanging " + f"or TckConfig.ready_timeout is too short" + ) + raise AssertionError(msg) from None + except Exception as exc: + msg = ( + f"registering the provider raised {exc!r}. The backend is up and seeded " + f"at this point, so this is a genuine initialisation failure rather than " + f"the unavailable-backend case" + ) + raise AssertionError(msg) from exc + + tck_state.provider = provider + tck_state.client = api.get_client(config.domain) + + +@given(parsers.re(r"^an? unavailable provider$")) +def an_unavailable_provider(tck_state: TckState) -> None: + """Register a provider pointed at a backend that does not exist. + + Neither a failed initialisation nor a raised exception during registration + is a failure here: what the contract requires is that the provider settles + into an observable error state promptly, which the scenario asserts through + the event and the client status. The SDK's registry already converts a + raising ``initialize`` into ``PROVIDER_ERROR``, so registration itself is + expected to return normally -- but a provider that raises anyway must not + take the scenario down with it, which is why this is caught rather than + propagated. + """ + config = tck_state.config + + if config.new_unavailable_provider is None: + msg = ( + "TckConfig.new_unavailable_provider is None but an @unavailable scenario " + "ran. This is a test-configuration bug rather than a provider defect: the " + "suite declared Capability.UNAVAILABLE_INIT without supplying a provider " + "that cannot reach its backend. Remove that capability, or supply the factory" + ) + raise AssertionError(msg) + + provider = config.new_unavailable_provider() + if provider is None: + msg = "TckConfig.new_unavailable_provider returned None" + raise AssertionError(msg) + + # The waiting variant for the same reason as the stable provider: plain + # set_provider initialises on a worker thread, so registration would return + # before the provider had even tried to reach its backend, and the scenario + # would assert an error state that had not happened yet. + # + # A raising initialize is already converted to PROVIDER_ERROR by the SDK's + # registry, so the suppression is belt and braces: a provider that raises + # anyway must not take the scenario down with it, because the contract is + # about the observable error state rather than about how registration + # returned. + with contextlib.suppress(Exception): + api.set_provider_and_wait(provider, config.domain) + + tck_state.provider = provider + tck_state.client = api.get_client(config.domain) + + +@when("the provider is shut down") +def the_provider_is_shut_down(tck_state: TckState) -> None: + """Call the provider's own ``shutdown``, directly. + + Not through the SDK. The SDK shuts a provider down when it is replaced or + when the API is shut down, but going that way would test the registry's + bookkeeping as much as the provider, and Appendix B already does that. + Calling ``shutdown`` on the instance is also what lets a scenario call it + twice: the registry only ever calls it once per registration. + + The registry is not told. The client still points at the same instance, so + an evaluation after "the provider is initialized again" reaches the very + object that was shut down and brought back, which is what that scenario + asserts. And when the scenario ends the SDK shuts the provider down once + more on its own -- a second call, which requirement 2.5.3 makes harmless. + """ + _record_lifecycle_call(tck_state, "shutdown", tck_state.require_provider().shutdown) + + +@when("the provider is initialized again") +def the_provider_is_initialized_again(tck_state: TckState) -> None: + """Call the provider's own ``initialize`` after it was shut down. + + With an empty context, as the SDK would with none set. Direct for the same + reason as the shutdown step: re-registering through the SDK would create a + new registration around the same instance, and what is under test is that + the instance itself reverts to an initialisable state. + """ + provider = tck_state.require_provider() + _record_lifecycle_call( + tck_state, "initialize", lambda: provider.initialize(EvaluationContext()) + ) + + +@then(parsers.re(r"^the shutdown should have completed within (?P\d+)ms$")) +def the_shutdown_should_have_completed_within(tck_state: TckState, millis: str) -> None: + """Bound the most recent shutdown. + + The scenario using this runs against a backend that will never answer, so + what it asserts is that shutdown returns rather than waiting for a graceful + close that cannot happen. A shutdown that was given up on because it + outlasted ``TckConfig.ready_timeout`` fails here too: its recorded duration + is however long the suite waited before moving on. + """ + record = tck_state.require_shutdown() + bound = int(millis) / 1000.0 + if record.duration > bound: + msg = ( + f"shutdown took {record.duration * 1000:.0f}ms, expected it to complete within " + f"{millis}ms. A shutdown that waits on a backend that is gone hangs the host " + f"application's own shutdown" + ) + raise AssertionError(msg) + + +@then("the provider metadata name should not be empty") +def the_provider_metadata_name_should_not_be_empty(tck_state: TckState) -> None: + """Assert the provider identifies itself (requirement 2.1.1). + + Asked of the provider rather than of ``api.get_provider_metadata``, which + would answer for whatever the registry holds under the domain: the same + object here, but the question is about the provider. + """ + provider = tck_state.require_provider() + try: + metadata = provider.get_metadata() + except Exception as exc: + msg = f"get_metadata raised {exc!r}: the provider cannot say what it is" + raise AssertionError(msg) from exc + + name = getattr(metadata, "name", None) + if not isinstance(name, str) or not name.strip(): + msg = ( + f"the provider metadata name is {name!r}, expected a non-empty string. A " + f"conformance report keyed on the name cannot be attributed without one" + ) + raise AssertionError(msg) + + +def _record_lifecycle_call( + tck_state: TckState, operation: str, call: Callable[[], object] +) -> None: + """Make one direct lifecycle call and record how it went, raising nothing. + + An exception is recorded rather than propagated, for the same reason an + evaluation's is: "no exception should have been thrown" is a step of its + own, and a scenario that wants a raise to fail says so there. Only + ``Exception`` is caught, though. The shutdown scenario that matters most is + the one against a backend that is gone, which is exactly where somebody + might reach for Ctrl-C, and a ``KeyboardInterrupt`` recorded as "shutdown + raised" would carry the run on past the thing they interrupted. + + A call that outlasts ``TckConfig.ready_timeout`` is given up on and recorded + as a ``TimeoutError`` with the time waited, so a hanging shutdown fails its + scenario with a message rather than hanging the session. + """ + started = time.perf_counter() + raised: BaseException | None = None + try: + _call_within(call, tck_state.config.ready_timeout) + except TimeoutError: + raised = TimeoutError( + f"{operation} did not return within {tck_state.config.ready_timeout}s" + ) + except Exception as exc: # recorded here, asserted on by its own step + raised = exc + duration = time.perf_counter() - started + tck_state.lifecycle.append( + LifecycleRecord(operation=operation, duration=duration, raised=raised) + ) + + +def _call_within(call: Callable[[], object], timeout: float) -> None: + """Make a call into the provider, giving up if it has not returned in time. + + Neither ``api.set_provider_and_wait`` nor a provider's own ``shutdown`` has a + timeout of its own, so one that hangs while talking to its backend would hang + the whole session with no useful message. Running it on a worker thread bounds + it. + + The worker is deliberately not cancelled on timeout -- Python cannot interrupt a + thread blocked in a socket call -- so it is left to finish or die with the process. + That is acceptable here because a timeout already means the scenario is failing. + """ + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(call) + try: + future.result(timeout=timeout) + except concurrent.futures.TimeoutError: + raise TimeoutError from None + finally: + # Do not block __exit__ on a worker that is still stuck. + pool.shutdown(wait=False) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/values.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/values.py new file mode 100644 index 00000000..4459d466 --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/values.py @@ -0,0 +1,125 @@ +"""Turning Gherkin strings into typed values, and comparing them with what a provider resolved.""" + +from __future__ import annotations + +import json +import typing + +from openfeature.flag_evaluation import FlagType + +__all__ = ["describe", "parse_flag_type", "parse_value", "values_equal"] + +_BY_NAME: dict[str, FlagType] = { + "boolean": FlagType.BOOLEAN, + "string": FlagType.STRING, + "integer": FlagType.INTEGER, + "float": FlagType.FLOAT, + "object": FlagType.OBJECT, +} + + +def parse_flag_type(raw: str) -> FlagType: + """Resolve the type named in a scenario, case-insensitively.""" + try: + return _BY_NAME[raw.strip().lower()] + except KeyError: + names = ", ".join(sorted(n.capitalize() for n in _BY_NAME)) + msg = f"unknown flag type {raw!r}: expected one of {names}" + raise ValueError(msg) from None + + +def _parse_bool(raw: str) -> bool: + lowered = raw.strip().lower() + if lowered in {"true", "t", "yes", "1"}: + return True + if lowered in {"false", "f", "no", "0"}: + return False + msg = f"{raw!r} is not a boolean" + raise ValueError(msg) + + +def parse_value(flag_type: FlagType, raw: str) -> typing.Any: + """Convert a value written in a scenario into the type the API uses. + + Everything in Gherkin is a string, so this is where ``"0.5"`` becomes a + float and ``"{}"`` becomes an empty object. Parsing per declared type rather + than guessing is what keeps the integer and float scenarios + distinguishable: ``"1"`` is an ``int`` in an Integer scenario and a ``float`` + in a Float one. + """ + if flag_type is FlagType.BOOLEAN: + return _parse_bool(raw) + if flag_type is FlagType.STRING: + return raw + if flag_type is FlagType.INTEGER: + return int(raw) + if flag_type is FlagType.FLOAT: + return float(raw) + if flag_type is FlagType.OBJECT: + # Gherkin escapes quotes in table cells; pytest-bdd keeps the backslash, + # so strip it before handing the text to json. + return json.loads(raw.replace('\\"', '"')) + msg = f"unknown flag type {flag_type!r}" + raise ValueError(msg) + + +def _as_number(value: typing.Any) -> float | None: + """Return a numeric value as a float, or None if it is not numeric. + + Booleans are deliberately excluded. Python makes ``bool`` a subclass of + ``int``, so an unguarded numeric comparison would quietly report ``True`` and + ``1`` as equal -- which is the exact confusion several of these scenarios + exist to detect. + """ + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return float(value) + return None + + +def values_equal(expected: typing.Any, actual: typing.Any) -> bool: + """Compare an expected value from a scenario with what a provider resolved. + + Numbers are compared numerically rather than by Python type. A provider that + deserialises its backend's JSON hands back ``float`` for every number, so the + ``100`` inside ``object-flag`` arrives as ``100.0`` from one provider and + ``100`` from another while both are correct. Type distinctness is asserted + where it belongs -- by requesting a flag as a specific type and checking the + error code -- not by accident of how a number was decoded. + """ + # A boolean only ever equals a boolean. Without this, Python's bool-is-an-int + # rule would make True == 1 and quietly satisfy the scenario that exists to + # catch exactly that confusion. + if isinstance(expected, bool) or isinstance(actual, bool): + return ( + isinstance(expected, bool) + and isinstance(actual, bool) + and expected == actual + ) + + expected_number = _as_number(expected) + if expected_number is not None: + actual_number = _as_number(actual) + return actual_number is not None and expected_number == actual_number + + if isinstance(expected, dict) and isinstance(actual, dict): + if set(expected) != set(actual): + return False + return all(values_equal(v, actual[k]) for k, v in expected.items()) + + if isinstance(expected, list) and isinstance(actual, list): + return len(expected) == len(actual) and all( + values_equal(e, a) for e, a in zip(expected, actual, strict=True) + ) + + return bool(expected == actual) + + +def describe(value: typing.Any) -> str: + """Render a value for a failure message, including its type. + + "expected 100 but got 100" is the single most confusing failure a + cross-language conformance suite can produce. + """ + return f"{value!r} ({type(value).__name__})" diff --git a/tools/openfeature-tck/tests/conftest.py b/tools/openfeature-tck/tests/conftest.py new file mode 100644 index 00000000..a5e6726f --- /dev/null +++ b/tools/openfeature-tck/tests/conftest.py @@ -0,0 +1,39 @@ +"""Known deviations of the Python SDK, recorded rather than hidden. + +A conformance suite that quietly goes green on scenarios it did not run is worse +than no suite at all -- and the same is true of one that quietly goes green on a +scenario it *did* run and fail. So the one scenario the Python SDK cannot +currently satisfy is marked ``xfail(strict=True)`` here, which: + +* keeps it visible in the report, as XFAIL with the reason attached; +* fails the suite if it ever *passes*, so the marker is removed the moment the + SDK is fixed rather than lingering as a lie. + +This lives in the TCK's own self-test rather than in the shared package. It is a +fact about the SDK under test, not part of the conformance definition, and +Appendix F deliberately leaves a general "known deviations" concept as an open +question (spec#417, Q4). If that concept lands, this moves into it. +""" + +from __future__ import annotations + +import pytest + +# The Scenario Outline row that asks for boolean-flag as an Integer. +_BOOL_AS_INT = ( + "test_requesting_the_wrong_type_returns_the_code_default[boolean-flag-Integer-1]" +) + +_REASON = ( + "python-sdk: a boolean satisfies an Integer request. The client type-checks with " + "isinstance(value, int) and bool is a subclass of int in Python, so boolean-flag " + "requested as an Integer returns True with reason STATIC and no error code, where " + "the specification requires the code default and TYPE_MISMATCH. " + "See https://github.com/open-feature/python-sdk/issues/619" +) + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + for item in items: + if item.name == _BOOL_AS_INT: + item.add_marker(pytest.mark.xfail(reason=_REASON, strict=True)) diff --git a/tools/openfeature-tck/tests/test_compose.py b/tools/openfeature-tck/tests/test_compose.py new file mode 100644 index 00000000..f1b28b21 --- /dev/null +++ b/tools/openfeature-tck/tests/test_compose.py @@ -0,0 +1,352 @@ +"""What an adopter declares about a container stack, and what the harness owes them. + +The stack lifecycle itself needs Docker and is proved by the flagd adoption, +which runs the whole canonical suite against a real testbed through this +harness. What is checked here is everything that can be wrong *without* a +container, and all of it was adopter code until now: the declaration's defaults, +the refusals that turn a mistake into a message instead of a provider that +cannot connect three scenarios later, and the port resolution the provider +factory is handed. + +``ComposeStack`` is a protocol rather than ``DockerCompose`` itself, which is +what makes that possible: a stub satisfying two methods stands in for Docker, so +the resolution logic is examined rather than assumed. +""" + +from __future__ import annotations + +import dataclasses +import typing +from pathlib import Path + +import pytest + +from openfeature.contrib.tools.tck import ( + DEFAULT_BACKEND_SERVICE, + DEFAULT_CONFIGURATION, + DEFAULT_CONTROL_PORT, + DEFAULT_STARTUP_TIMEOUT, + BackendEndpoint, + ComposeBackend, + ControlApiError, + run_compose_backend, +) + +BACKEND_PORT = 8013 +"""A container-internal port a provider connects to. flagd's RPC port, for realism.""" + + +@dataclasses.dataclass +class _StubStack: + """A Compose stack that publishes exactly what it is told to. + + Keyed by ``(service, container port) -> host port``, because that is the + whole of what the harness asks a stack for -- and getting the service half + wrong is how a multi-service stack resolves a port against the wrong + container. + """ + + published: dict[tuple[str, int], int] + host: str = "127.0.0.1" + + def get_service_host( + self, service_name: str | None = None, port: int | None = None + ) -> str | None: + # Testcontainers resolves a host *through* a published port, and asked + # without one it insists the service publish exactly one. Reproduced + # rather than waved through: a stub that answered anyway is why the + # no-port form reached a real three-port stack and raised there. + assert service_name is not None + if port is None: + ports = [key for key in self.published if key[0] == service_name] + assert len(ports) == 1, ( + f"service {service_name!r} publishes {len(ports)} ports, so the " + f"host cannot be resolved without naming one of them" + ) + return self.host + + def get_service_port( + self, service_name: str | None = None, port: int | None = None + ) -> int | None: + assert service_name is not None + assert port is not None + return self.published.get((service_name, port)) + + +def _endpoint( + published: dict[tuple[str, int], int], + service: str = DEFAULT_BACKEND_SERVICE, +) -> BackendEndpoint: + """An endpoint over a stub stack, told which ports each service publishes. + + Which is what the real one is told, from the declaration: the host is + resolved through one of them. + """ + ports: dict[str, list[int]] = {} + for name, internal in published: + ports.setdefault(name, []).append(internal) + return BackendEndpoint( + stack=typing.cast("typing.Any", _StubStack(published=published)), + backend_service=service, + service_ports=ports, + ) + + +# -- the declaration --------------------------------------------------------- + + +def test_the_defaults_are_the_ones_every_language_fixes() -> None: + """Pinned as literals, because the Compose file is what an adoption shares. + + A provider shipped in two languages writes one Compose file and two + declarations against it, so a default that differs between two TCKs makes + the file wrong in one of them -- and the symptom is a stack that starts and + a control API nothing can reach. Every other assertion in this file uses the + constants and would stay green through a change to them. + """ + backend = ComposeBackend(compose_file="docker-compose.yaml", backend_ports=[8013]) + + assert backend.backend_service == "backend" + assert backend.control_port == 8080 + assert backend.additional_ports == {} + assert backend.backend_configuration == "default" + assert backend.startup_timeout == 60.0 + + assert DEFAULT_BACKEND_SERVICE == "backend" + assert DEFAULT_CONTROL_PORT == 8080 + assert DEFAULT_CONFIGURATION == "default" + assert DEFAULT_STARTUP_TIMEOUT == 60.0 + + +def test_backend_ports_is_required_and_says_the_control_port_is_not_one() -> None: + """The one refusal an adopter is most likely to meet, so it has to teach. + + An empty ``backend_ports`` is a declaration that says nothing about which + ports the provider needs, which turns a startup check into nothing and the + first scenario into an unexplained connection failure. + """ + with pytest.raises(ValueError, match="backend_ports is required") as raised: + ComposeBackend(compose_file="docker-compose.yaml", backend_ports=[]) + + assert "control port (8080) is handled automatically" in str(raised.value) + + +def test_the_control_port_may_not_be_declared_as_a_backend_port() -> None: + """Listing it is a sign of a misunderstanding rather than a duplicate entry. + + The control API is the harness's, not the provider's. A provider pointed at + it is not exercising the contract this suite tests, and a declaration that + lists it reads as though it might be. + """ + with pytest.raises(ValueError, match="lists the control port 8080"): + ComposeBackend( + compose_file="docker-compose.yaml", + backend_ports=[BACKEND_PORT, DEFAULT_CONTROL_PORT], + ) + + +def test_a_relocated_control_port_may_be_a_backend_port_elsewhere() -> None: + """The check is against the declared control port, not against 8080. + + A stack serving its control API somewhere else is entitled to have 8080 be + an ordinary backend port. + """ + backend = ComposeBackend( + compose_file="docker-compose.yaml", + backend_ports=[DEFAULT_CONTROL_PORT], + control_port=9090, + ) + assert backend.exposed_ports == {"backend": (9090, DEFAULT_CONTROL_PORT)} + + +def test_a_declaration_reports_every_problem_it_has_at_once() -> None: + """One traceback naming everything wrong, not one per round trip.""" + with pytest.raises(ValueError) as raised: + ComposeBackend( + compose_file="docker-compose.yaml", + backend_ports=[], + backend_service="", + startup_timeout=0, + ) + + message = str(raised.value) + assert "backend_ports is required" in message + assert "backend_service must name a Compose service" in message + assert "startup_timeout must be positive" in message + + +def test_the_declaration_is_normalised_to_tuples() -> None: + """So a declaration written as a list is stored as what it is checked as. + + Normalised before validation rather than after, which is the order that + matters: the control-port check reads ``backend_ports``, and reading it + first would consume a one-shot iterable before it reached the field. + """ + backend = ComposeBackend( + compose_file="docker-compose.yaml", + backend_ports=[8013, 8015], + additional_ports={"proxy": [9212]}, + ) + assert backend.backend_ports == (8013, 8015) + assert backend.additional_ports == {"proxy": (9212,)} + + +def test_the_control_port_is_exposed_ahead_of_everything_an_adopter_named() -> None: + """It is the harness's own port and is never declared, so it is added here. + + First, because a stack that publishes nothing else still has to answer + control calls, and the startup failure should say so before it says anything + about a provider port. + """ + backend = ComposeBackend( + compose_file="docker-compose.yaml", + backend_ports=[8013, 8015], + additional_ports={"proxy": [9212]}, + ) + assert backend.exposed_ports == { + "backend": (DEFAULT_CONTROL_PORT, 8013, 8015), + "proxy": (9212,), + } + + +def test_additional_ports_on_the_backend_service_join_rather_than_replace() -> None: + """A stack may name the backend service again without losing the control port.""" + backend = ComposeBackend( + compose_file="docker-compose.yaml", + backend_ports=[8013], + additional_ports={DEFAULT_BACKEND_SERVICE: [8015, 8013]}, + ) + assert backend.exposed_ports == {"backend": (DEFAULT_CONTROL_PORT, 8013, 8015)} + + +# -- resolving the compose file ---------------------------------------------- + + +def test_a_relative_compose_path_resolves_against_the_package_directory( + tmp_path: Path, +) -> None: + """Which is where pytest runs from, so ``tests/tck/docker-compose.yaml`` works.""" + backend = ComposeBackend( + compose_file="tests/tck/docker-compose.yaml", backend_ports=[BACKEND_PORT] + ) + assert ( + backend.resolved_compose_file(tmp_path) + == tmp_path / "tests/tck/docker-compose.yaml" + ) + + +def test_an_absolute_compose_path_is_used_as_given(tmp_path: Path) -> None: + absolute = tmp_path / "elsewhere" / "docker-compose.yaml" + backend = ComposeBackend(compose_file=absolute, backend_ports=[BACKEND_PORT]) + assert backend.resolved_compose_file(tmp_path / "ignored") == absolute + + +def test_a_missing_compose_file_fails_before_anything_is_started( + tmp_path: Path, +) -> None: + """And says where it looked, because the answer is usually "not where I meant". + + Raised from the generator before Docker is touched, so an adopter who + mistyped the path does not wait for a stack to come up first -- and never + needs Docker to find out. + """ + backend = ComposeBackend( + compose_file="tests/tck/docker-compose.yaml", backend_ports=[BACKEND_PORT] + ) + with pytest.raises(FileNotFoundError) as raised: + next(run_compose_backend(backend, root=tmp_path)) + + message = str(raised.value) + assert "docker-compose.yaml" in message + assert "resolved relative to the package directory" in message + + +# -- the endpoint the provider factory is handed ----------------------------- + + +def test_a_mapped_port_is_looked_up_by_container_port() -> None: + """Which is what an adopter knows: 8013 is in their Compose file, 32769 is not.""" + endpoint = _endpoint({(DEFAULT_BACKEND_SERVICE, BACKEND_PORT): 32769}) + assert endpoint.port(BACKEND_PORT) == 32769 + + +def test_a_port_may_be_qualified_by_service_for_a_multi_service_stack() -> None: + """Two services may publish the same container port, and usually do. + + Resolving one against the other is silent: the provider connects to + something that answers, and the scenario fails on whatever it answers with. + """ + endpoint = _endpoint( + { + (DEFAULT_BACKEND_SERVICE, BACKEND_PORT): 32769, + ("proxy", BACKEND_PORT): 32770, + } + ) + assert endpoint.port(BACKEND_PORT) == 32769 + assert endpoint.port(BACKEND_PORT, service="proxy") == 32770 + + +def test_the_host_is_the_stacks_own_rather_than_localhost() -> None: + """A remote daemon, Docker Desktop or a rootless setup can serve any address.""" + endpoint = BackendEndpoint( + stack=typing.cast( + "typing.Any", + _StubStack( + published={(DEFAULT_BACKEND_SERVICE, BACKEND_PORT): 32769}, + host="192.168.64.2", + ), + ), + service_ports={DEFAULT_BACKEND_SERVICE: [BACKEND_PORT]}, + ) + assert endpoint.host == "192.168.64.2" + + +def test_the_host_of_a_service_publishing_several_ports_still_resolves() -> None: + """flagd's stack publishes three on one service, which is the common shape. + + Resolved through one of the declared ports, because testcontainers resolves + a host *through* a published port and, asked without one, requires the + service to publish exactly one. The no-port form therefore worked on every + single-port stack and raised ``NoSuchPortExposed`` on the first real one -- + with a message about the port rather than about the call. + """ + endpoint = _endpoint( + { + (DEFAULT_BACKEND_SERVICE, DEFAULT_CONTROL_PORT): 32768, + (DEFAULT_BACKEND_SERVICE, BACKEND_PORT): 32769, + (DEFAULT_BACKEND_SERVICE, 8015): 32770, + } + ) + assert endpoint.host == "127.0.0.1" + assert endpoint.port(8015) == 32770 + + +def test_an_undeclared_service_says_so_rather_than_failing_inside_docker() -> None: + """A service the declaration does not name has nothing published for it. + + So there is nothing to resolve a host through and nothing for a provider to + connect to. Answered here, naming the services that *were* declared, rather + than passed down to testcontainers to answer as a port problem. + """ + endpoint = _endpoint({(DEFAULT_BACKEND_SERVICE, BACKEND_PORT): 32769}) + with pytest.raises(ControlApiError) as raised: + endpoint.host_of("proxy") + + message = str(raised.value) + assert "no port is declared for service 'proxy'" in message + assert "additional_ports" in message + + +def test_an_unpublished_port_says_the_compose_file_has_to_list_it() -> None: + """The error an adopter actually hits, and the fix is in the Compose file. + + Testcontainers' own answer here is a ``NoSuchPortExposed`` naming the port + and nothing else, which reads as though the harness were at fault. + """ + endpoint = _endpoint({(DEFAULT_BACKEND_SERVICE, DEFAULT_CONTROL_PORT): 32768}) + with pytest.raises(ControlApiError) as raised: + endpoint.port(BACKEND_PORT) + + message = str(raised.value) + assert f"no host port for {BACKEND_PORT}" in message + assert "`ports:`" in message diff --git a/tools/openfeature-tck/tests/test_controllable_conformance.py b/tools/openfeature-tck/tests/test_controllable_conformance.py new file mode 100644 index 00000000..108587e3 --- /dev/null +++ b/tools/openfeature-tck/tests/test_controllable_conformance.py @@ -0,0 +1,79 @@ +"""Run the conformance suite against the TCK's own updatable in-memory provider. + +This is the suite that exercises the configuration-change path, and it exists +because the SDK's in-memory provider cannot: it has no way to update a flag set, +so ``test_in_memory_conformance`` necessarily skips those scenarios. Without +this suite the change-event step definitions would ship with no coverage at all, +and a break in them would first surface in a containerised provider suite where +it looks like a provider defect. + +It is also the reference for what an in-process control path looks like when the +provider does support updates, which is what a file-based or +environment-variable provider should be able to do. +""" + +from __future__ import annotations + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.tck import ( + Capability, + InProcessControl, + TckConfig, + feature_paths, +) + + +@pytest.fixture(scope="session") +def tck_config() -> TckConfig: + """Declare the provider under test and what it can do. + + ``STALE`` and ``UNAVAILABLE_INIT`` stay undeclared: there is still no + connection to lose, and ``InProcessControl`` does not implement + ``ConnectionControl``. ``CONFIGURATION_CHANGE`` is what this suite adds over + the plain in-memory one, and it is the whole point of it. + + ``LIFECYCLE`` stays undeclared for the same reason as in + ``test_in_memory_conformance``: there is no backend to reach during + initialisation, so the readiness scenario would pass here without testing + anything. It did exactly that while the feature was gated on ``@events``. + + ``NUMERIC_COERCION`` stays undeclared for the reason given there too. + ``ControllableInMemoryProvider`` changes nothing about resolution, so it + inherits the SDK provider's refusal to coerce: ``10.0`` requested as an + integer is a ``TYPE_MISMATCH`` rather than ``10``. ``LARGE_INTEGERS`` is + declared, since a Python ``int`` is exact at 2^53 - 1. + + ``TARGETING`` stays undeclared for the reason given there as well: this is + the same decoded flag set, and it ignores ``targeting-key-flag``'s rule. + ``VARIANTS`` is declared, since the flag set is keyed by variant name. + + ``DISABLED_FLAGS`` stays undeclared for the reason given there too, and this + class inherits it rather than choosing it: ``ControllableInMemoryProvider`` + changes only how the flag set is *replaced*, and resolution -- including the + fact that ``InMemoryFlag.resolve`` never reads ``state`` -- is still the + SDK's. Measured the same way, with the same four failures. + + ``STANDARD_REASONS`` is declared, for the reason given there and on the same + measurement: resolution is the SDK's, so the four ``STATIC`` rows and the two + error scenarios pass here identically. The three that compose the tag with + ``TARGETING`` or ``DISABLED_FLAGS`` skip, since neither is declared. + """ + control = InProcessControl() + return TckConfig( + name="controllable-in-memory", + control=control, + new_provider=control.new_provider, + capabilities={ + Capability.EVENTS, + Capability.CONFIGURATION_CHANGE, + Capability.OBJECT, + Capability.VARIANTS, + Capability.LARGE_INTEGERS, + Capability.STANDARD_REASONS, + }, + ) + + +scenarios(*feature_paths()) diff --git a/tools/openfeature-tck/tests/test_declaration.py b/tools/openfeature-tck/tests/test_declaration.py new file mode 100644 index 00000000..d23d6fc8 --- /dev/null +++ b/tools/openfeature-tck/tests/test_declaration.py @@ -0,0 +1,795 @@ +"""What an adoption may declare about its provider, and what it may not. + +A ``TckConfig`` is two things at once. It is the configuration a run needs, and +it is a *declaration*: the set of claims an adopter makes about the provider, +which is what turns a skipped scenario from a hole in the run into a recorded +answer. Everything checked here belongs to the second role, so none of it is +observable in the pass/fail of a suite -- which is exactly why it is pinned by +tests of its own rather than by the conformance suites. + +The declaration vocabulary is public because something outside this package has +to read it back. A reporter deciding whether a skip was legitimate needs the tag +lookup and the reserved set; a comparison page needs to tell a declined +capability from an impossible one. None of those consumers is here, and the API +is complete for them anyway -- a follow-up that adds one should widen nothing. + +The one property that makes "reserved" mean anything is checked against the +packaged assets rather than asserted: a reserved tag is reserved because no +canonical scenario carries it, and that stops being true the moment the spec +adds one. +""" + +from __future__ import annotations + +import types +import typing +from pathlib import Path + +import pytest + +from openfeature.contrib.tools.tck import ( + DECLARABLE_CAPABILITIES, + INEXPRESSIBLE_CAPABILITIES, + RESERVED_CAPABILITIES, + BackendControl, + Capability, + ControlApi, + InProcessControl, + KnownDeviation, + TckConfig, + canonical_root, + plugin, +) +from openfeature.contrib.tools.tck import capability as capability_module +from openfeature.contrib.tools.tck import config as config_module +from openfeature.contrib.tools.tck.capability import ( + capability_for_marker, + capability_for_tag, + expired_reservations, +) +from openfeature.contrib.tools.tck.extensions import canonical_tags + + +class _StubControl: + """A control that says nothing it is not obliged to say. + + Which no longer includes ``control_api``: it is a required member of + ``BackendControl``, so even a stub has to answer it. + """ + + def prepare_scenario(self) -> None: ... + + def change_flag(self) -> None: ... + + @property + def description(self) -> str: + return "a stub" + + @property + def control_api(self) -> ControlApi: + return "in-process" + + +def _config(**overrides: typing.Any) -> TckConfig: + """A configuration that is valid but declares nothing in particular.""" + settings: dict[str, typing.Any] = { + "name": "stub", + "control": _StubControl(), + "new_provider": lambda: None, + "capabilities": frozenset(), + } + settings.update(overrides) + return TckConfig(**settings) + + +# -- the vocabulary ---------------------------------------------------------- + + +def test_every_capability_is_declarable_reserved_or_inexpressible() -> None: + """Three sets, one enum, and no member in two of them or in none. + + ``DECLARABLE_CAPABILITIES`` is derived from the other two rather than listed + beside them, so this is really a check that the derivation is the one the + documentation promises. It is worth pinning precisely because the third set + is empty here: a derivation that quietly dropped it would look right in + Python forever and be wrong the day an entry is added. + + Nothing may be both reserved and inexpressible. A reservation says no + scenario anywhere carries the tag; inexpressibility says the scenarios exist + and this SDK cannot put their question. The second presupposes what the first + denies. + """ + inexpressible = frozenset(INEXPRESSIBLE_CAPABILITIES) + assert frozenset(Capability) == ( + DECLARABLE_CAPABILITIES | RESERVED_CAPABILITIES | inexpressible + ) + assert not DECLARABLE_CAPABILITIES & RESERVED_CAPABILITIES + assert not DECLARABLE_CAPABILITIES & inexpressible + assert not RESERVED_CAPABILITIES & inexpressible + assert RESERVED_CAPABILITIES, "the whole rule is vacuous if nothing is reserved" + + for capability in Capability: + assert capability.reserved is (capability in RESERVED_CAPABILITIES) + assert capability.inexpressible is (capability in inexpressible) + + +def test_a_reserved_capability_is_one_no_canonical_scenario_carries() -> None: + """The fact the rule rests on, read off the assets rather than asserted. + + "Reserved" claims that nothing carries the tag, so declaring it cannot be + verified. When the specification gives a reserved tag a scenario, that stops + being true and this fails -- which is the moment the capability should become + declarable, and the moment somebody has to notice. It has already happened + once: ``@targeting`` gained three scenarios at spec revision ``26362f85`` + and moved out of the reserved set, which is what this half is for. + + This half is now also enforced on every adoption's run, by the plugin, and + not only here -- a self-test of this package is read by whoever changes this + package, and the reservation expires somewhere else. What stays here is the + converse, which is the half that catches a tag added to the enum and never + wired to anything: every declarable capability must be carried by some + canonical scenario, or declaring it examines nothing. + """ + carried = canonical_tags() + assert not expired_reservations(carried), ( + "the canonical assets now carry a reserved tag, so it can be verified " + "and should be declarable" + ) + for capability in DECLARABLE_CAPABILITIES: + assert capability.tag in carried, ( + f"{capability.tag} is declarable but no canonical scenario carries " + f"it, so declaring it would be a claim nothing examines" + ) + + +def test_the_packaged_tags_are_read_off_tag_lines_and_not_out_of_prose() -> None: + """The one way this scan can be wrong, pinned against the real assets. + + ``events.feature`` mentions ``@caching`` in a comment, saying where those + scenarios will go once they exist. A scan that read the whole file rather + than its tag lines would call the reservation expired on the strength of + that sentence, and since the plugin fails a run over an expiry, every + adoption would fail over a sentence. + + The second assertion is what keeps the first from being vacuous: it checks + that the prose mention is still there to be mis-read. + """ + tags = canonical_tags() + assert "@events" in tags, "a tag line is read" + assert "@caching" not in tags, "prose is not" + + root = canonical_root() + assert root is not None, "the packaged canonical features are not on a filesystem" + mentions = [ + feature + for feature in sorted(root.rglob("*.feature")) + if "@caching" in feature.read_text(encoding="utf-8") + ] + assert mentions, "nothing mentions @caching any more, so this proves nothing" + + +def test_a_reservation_expires_when_a_scenario_carries_it() -> None: + """The detection itself, which is all the plugin adds to it. + + Deduplicated, because the tags arrive from every scenario of every feature + file and one carried twice is not two expiries. + """ + reserved = sorted(RESERVED_CAPABILITIES, key=lambda c: c.tag) + + assert expired_reservations(["@events", "@object"]) == () + assert expired_reservations(c.tag for c in reserved) == tuple(reserved) + + first = reserved[0] + assert expired_reservations([first.tag, first.tag, "@events"]) == (first,) + + +def _scenario_item(filename: str, *tags: str) -> typing.Any: + """A collected node shaped the way pytest-bdd shapes one. + + ``__scenario__`` on the generated function, carrying the feature it came + from, and the tags as the markers pytest-bdd turns them into. Those two are + the whole of what the check reads off a node. + + Markers rather than the scenario's own ``tags``, which is the same choice + the capability gate makes and for the same reason: pytest-bdd applies a + marker for every tag on the scenario, on its feature *and* on its rule, so + a feature-level tag is absent from ``Scenario.tags`` and gates every + scenario in the file regardless. Reading what the gate reads is what keeps + the two from disagreeing about which scenarios are unclaimable. + """ + feature = types.SimpleNamespace(filename=filename) + function = types.SimpleNamespace() + function.__scenario__ = types.SimpleNamespace(feature=feature) + markers = [types.SimpleNamespace(name=tag.lstrip("@")) for tag in tags] + return types.SimpleNamespace(function=function, iter_markers=lambda: iter(markers)) + + +def _a_canonical_feature() -> str: + root = canonical_root() + assert root is not None, "the packaged canonical features are not on a filesystem" + features = sorted(root.rglob("*.feature")) + assert features, "the packaged canonical features are missing" + return str(features[0]) + + +def test_the_plugin_fails_a_run_over_an_expired_reservation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An adopter's run, not merely this package's own tests. + + The self-test above is read by whoever changes this package, and a + reservation expires in the specification repository instead -- so an + adoption that re-pinned the assets and ran the suite would see the new + scenarios skipped, for a capability it is refused permission to declare, + and nothing would say so. This is what says so. + + The upstream half of the check: the tags come off the packaged files, so + nothing an adopter's run does to its selection gets past it. The half that + arrives from the adopter's own side is below. + """ + items = [_scenario_item(_a_canonical_feature())] + + # Nothing has expired, which is the state every real run is in, and the + # hook is then silent. + plugin.pytest_collection_modifyitems(items) + + reserved = sorted(RESERVED_CAPABILITIES, key=lambda c: c.tag) + monkeypatch.setattr( + plugin, "canonical_tags", lambda: frozenset(c.tag for c in reserved) + ) + + with pytest.raises(pytest.UsageError) as raised: + plugin.pytest_collection_modifyitems(items) + + message = str(raised.value) + for capability in reserved: + assert capability.tag in message + # Named, because the fix is to edit that set against the specification and + # nothing the run can do stands in for it. + assert "RESERVED_CAPABILITIES" in message + + +def test_the_plugin_fails_a_run_over_an_adopters_own_reserved_tag( + tmp_path: Path, +) -> None: + """The same failure, arriving from the adopter's side instead of upstream. + + A reserved capability cannot be declared, so the capability gate skips + every scenario carrying its tag -- an extension's included. That scenario + can never run and can never be claimed, which is precisely what this check + exists to surface, so where the tag came from changes the remedy and not + the consequence. The check used to read the canonical set alone and let + this one through. + + Nothing is monkeypatched here: the tag is reserved for real and it is the + adopter's own scenario carrying it, which is the whole of the case. + """ + reserved = sorted(RESERVED_CAPABILITIES, key=lambda c: c.tag) + items = [ + _scenario_item(_a_canonical_feature()), + _scenario_item( + str(tmp_path / "extensions" / "vendor.feature"), + *(capability.tag for capability in reserved), + ), + ] + + with pytest.raises(pytest.UsageError) as raised: + plugin.pytest_collection_modifyitems(items) + + message = str(raised.value) + for capability in reserved: + assert capability.tag in message + # Both remedies, because the check cannot tell which mistake it caught. + assert "RESERVED_CAPABILITIES" in message, "the upstream remedy is named" + assert "extensions/" in message, "so is the adopter's" + + +def test_a_scenario_this_suite_does_not_run_is_not_read(tmp_path: Path) -> None: + """One pytest session can hold more than this suite. + + A project adopting the TCK may have a pytest-bdd suite of its own, and it + is collected into the same session. Its tags are not this suite's business: + a scenario of theirs tagged ``@caching`` is gated by nothing here, runs + normally, and is claimed by nobody -- so failing their run over it would be + an accusation about a file this package has no say in. + + The discriminator is the uri derivation, which answers for the canonical + assets and for an ``extensions`` directory and for nothing else. + """ + items = [ + _scenario_item(_a_canonical_feature()), + _scenario_item( + str(tmp_path / "features" / "billing.feature"), + *(capability.tag for capability in RESERVED_CAPABILITIES), + ), + ] + + plugin.pytest_collection_modifyitems(items) + + +def test_the_plugin_leaves_a_session_that_is_not_running_the_suite_alone( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The plugin is installed for every pytest run in the environment. + + Which makes the check's trigger part of its correctness: an unrelated test + suite in a project that happens to depend on this package has no business + failing over the contents of these feature files. So the check waits for a + canonical scenario to be collected, and nothing but ``feature_paths()`` + produces one of those. A node that is not a scenario at all is not one. + + Neither is an adopter's extension, and that is the one asymmetry worth + stating: an extension's tags are read, but an extension is not what says + the suite is running. Nothing an adopter selects reaches that distinction + anyway -- the hook is handed the whole collection before anything is + deselected -- so this is about a session that genuinely collected no + canonical scenario: somebody else's pytest-bdd suite, in an environment + that merely has this package installed. + """ + monkeypatch.setattr( + plugin, + "canonical_tags", + lambda: frozenset(c.tag for c in RESERVED_CAPABILITIES), + ) + + not_a_scenario: typing.Any = types.SimpleNamespace() + + plugin.pytest_collection_modifyitems([]) + plugin.pytest_collection_modifyitems([not_a_scenario]) + plugin.pytest_collection_modifyitems([_scenario_item(__file__)]) + plugin.pytest_collection_modifyitems( + [_scenario_item(str(tmp_path / "extensions" / "vendor.feature"))] + ) + + +def test_a_tag_maps_onto_the_capability_it_gates() -> None: + """The lookup a reporter outside this package needs, in the tag form. + + The tag form rather than the marker form, because that is the form a + scenario's tags are recorded in: deciding whether a skip was legitimate + means reading them back as the feature files spell them. Nothing in this + package calls it -- it is exported for the consumer that does. + """ + for capability in Capability: + assert capability_for_tag(capability.tag) is capability + assert capability_for_marker(capability.value) is capability + + # An organisational tag gates nothing, and must not be mistaken for a + # capability: the feature files carry them freely. + assert capability_for_tag("@smoke") is None + assert capability_for_tag("events") is None, "the at-sign is part of the tag" + + +# -- declaring a capability set ---------------------------------------------- + + +def test_the_default_is_every_declarable_capability() -> None: + """And so cannot pick up a reserved tag on the way past. + + "Declare everything, then narrow it" is the advice, which makes the default + the one place a reserved tag would otherwise get declared by accident. One + implementation's published report asserts ``@targeting`` and ``@caching`` + for precisely that reason -- back when both were reserved. + """ + # Not routed through ``_config``, which narrows the set: the field default is + # the whole point of this one. It needs an unavailable-provider factory + # because ``@unavailable`` is declarable, so the default declares it. + settings: dict[str, typing.Any] = { + "name": "stub", + "control": _StubControl(), + "new_provider": lambda: None, + "new_unavailable_provider": lambda: None, + } + declared = TckConfig(**settings).capabilities + assert declared == DECLARABLE_CAPABILITIES + for capability in RESERVED_CAPABILITIES: + assert capability not in declared + + +def test_a_capability_set_is_normalised_however_it_was_written() -> None: + """A list, a set or a generator all arrive as the same frozenset.""" + expected = frozenset({Capability.EVENTS, Capability.OBJECT}) + written = [ + [Capability.EVENTS, Capability.OBJECT, Capability.EVENTS], + {Capability.EVENTS, Capability.OBJECT}, + (c for c in (Capability.EVENTS, Capability.OBJECT)), + ] + for capabilities in written: + assert _config(capabilities=capabilities).capabilities == expected + + +def test_something_that_is_not_a_capability_is_refused() -> None: + with pytest.raises(ValueError, match="unknown capabilities"): + _config(capabilities={"events"}) + + +def test_a_reserved_capability_cannot_be_declared() -> None: + """A tag no scenario carries is a claim nothing can check, so it is refused. + + At construction rather than at the point something reads the declaration: + the adopter wrote it down and meant something by it, so a configuration + silently different from the one they wrote is worse than one that will not + build -- and construction is where their own code is still on the stack to + say which line to fix. + """ + for reserved in RESERVED_CAPABILITIES: + with pytest.raises(ValueError, match=f"reserved capabilities {reserved.tag}"): + _config(capabilities={Capability.EVENTS, reserved}) + + +def test_the_refusal_says_what_may_be_declared_instead() -> None: + """A message that names the rule and not just the violation. + + The offending capability is taken from ``RESERVED_CAPABILITIES`` rather than + named, because naming one is how this test went stale: it asked about + ``@targeting``, which stopped being reserved the moment the specification + gave it scenarios, and the refusal it was asserting became correct + behaviour's absence. + """ + reserved = next(iter(sorted(RESERVED_CAPABILITIES, key=lambda c: c.tag))) + with pytest.raises(ValueError) as raised: + _config(capabilities={reserved}) + message = str(raised.value) + assert "DECLARABLE_CAPABILITIES" in message + for capability in DECLARABLE_CAPABILITIES: + assert capability.tag in message + + +# -- what this SDK cannot express -------------------------------------------- +# +# INEXPRESSIBLE_CAPABILITIES is empty in Python, and that was measured: `int` is +# arbitrary-precision and `get_integer_details` and `get_float_details` are +# separate accessors reaching separate provider methods, so all four questions +# the two tagged groups ask can be put, and were. The machinery is here anyway, +# because the rule belongs to Appendix F rather than to this package and the next +# capability may hit it -- and a mechanism nothing exercises is indistinguishable +# from a mechanism that does not work. So these tests supply an entry rather than +# skipping for want of one, and the fabricated entry is Java's real case. + + +_AS_IN_JAVA = ( + "the integer accessor is a 32-bit Integer, so 2^53 - 1 cannot be asked for" +) + + +@pytest.fixture +def one_inexpressible(monkeypatch: pytest.MonkeyPatch) -> Capability: + """Pretend, for one test, that this SDK cannot express ``@large-integers``. + + Patched on the module rather than injected, because the production code + reads the mapping through the module global at call time and an injected + copy would test a seam nothing else uses. + """ + monkeypatch.setattr( + capability_module, + "INEXPRESSIBLE_CAPABILITIES", + types.MappingProxyType({Capability.LARGE_INTEGERS: _AS_IN_JAVA}), + ) + return Capability.LARGE_INTEGERS + + +def test_an_inexpressible_capability_is_one_whose_scenarios_exist() -> None: + """The property that tells it from a reservation, read off the assets. + + A capability nothing carries is reserved, whatever any SDK could express + about it -- so an entry here whose tag no canonical scenario carries is + misfiled, and the two would then differ only in their wording. Every entry + also has to say *which* property of the SDK puts the question out of reach, + because that is the half of the message an adopter could not have worked out. + + Vacuous while the mapping is empty, and kept for the pass where it is not. + """ + carried = canonical_tags() + for capability, reason in INEXPRESSIBLE_CAPABILITIES.items(): + assert capability.tag in carried, ( + f"{capability.tag} is recorded as inexpressible but no canonical " + f"scenario carries it, which makes it a reservation instead" + ) + assert reason.strip(), ( + f"{capability.tag} does not say what puts it out of reach" + ) + + +def test_a_capability_this_sdk_cannot_express_cannot_be_declared( + one_inexpressible: Capability, +) -> None: + """Refused by the implementation, rather than left for adopters to remember. + + Which is the whole change: the fact is about the language, so an adopter + should not have to know it, and three suites each remembering it separately + is three chances to put an unverifiable claim in a report. + """ + with pytest.raises(ValueError) as raised: + _config(capabilities={Capability.EVENTS, one_inexpressible}) + + message = str(raised.value) + assert f"{one_inexpressible.tag} cannot be declared in this language" in message + # The property of the SDK, not the rule. An adopter reaching this has done + # nothing wrong and needs to be told something they could not have known. + assert _AS_IN_JAVA in message + assert "nothing for you to fix" in message + + +def test_the_two_refusals_do_not_read_the_same(one_inexpressible: Capability) -> None: + """A reader has to be able to tell a reservation from an impossibility. + + Reserved: global, temporary, expires when the specification writes a + scenario. Inexpressible: one language's, permanent, and the scenarios + already exist and pass elsewhere. Both end in a refusal and nothing else + about them is the same, so neither message may be reachable from the other's + predicate. + """ + reserved = next(iter(sorted(RESERVED_CAPABILITIES, key=lambda c: c.tag))) + + with pytest.raises(ValueError) as raised: + _config(capabilities={reserved}) + reserved_message = str(raised.value) + + with pytest.raises(ValueError) as raised: + _config(capabilities={one_inexpressible}) + inexpressible_message = str(raised.value) + + assert "no scenario carries them" in reserved_message + assert "no scenario carries" not in inexpressible_message, ( + "its scenarios do exist -- that is what makes it not a reservation" + ) + assert "is not a reservation" in inexpressible_message + assert _AS_IN_JAVA not in reserved_message + + # And they are produced by separate predicates, so neither can start + # answering for the other. + assert config_module.reserved_problems([one_inexpressible]) == [] + assert config_module.inexpressible_problems([reserved]) == [] + + +def test_a_deviation_may_not_name_a_capability_this_sdk_cannot_express( + one_inexpressible: Capability, +) -> None: + """A deviation is about this provider; this gap belongs to the language. + + Refused for the opposite reason a reserved one is. There the scenarios do + not exist, so there is nothing to deviate from; here they exist and no + provider in this SDK can attempt them, so the entry would attribute to one + provider something none of them could have done. + """ + with pytest.raises(ValueError) as raised: + _config( + known_deviations=[ + KnownDeviation.untracked(summary="a gap", capability=one_inexpressible) + ] + ) + + message = str(raised.value) + assert f"names {one_inexpressible.tag}, which cannot be expressed" in message + assert _AS_IN_JAVA in message + assert "belongs to the language" in message + + +def test_the_skip_reason_says_no_provider_here_could_have_been_asked( + one_inexpressible: Capability, +) -> None: + """The second half, and the one a report's reader actually sees. + + Refusing the declaration is not enough on its own: the scenarios are skipped + either way, and a skip reading "provider does not declare capability + @large-integers" describes a decision no provider in this language had the + chance to make. + """ + reason = plugin.inexpressible_skip_reason([one_inexpressible]) + assert reason is not None + assert _AS_IN_JAVA in reason + assert "no provider in this language can be asked" in reason + assert "Nothing about the provider under test follows" in reason + + declined = plugin.undeclared_skip_reason( + [Capability.EVENTS], _config(capabilities=frozenset()) + ) + assert declined is not None + assert "provider does not declare capability @events" in declined + assert declined != reason + + +def test_the_gate_prefers_the_language_reason_over_the_declaration_one( + one_inexpressible: Capability, +) -> None: + """A scenario gated by both kinds reports the permanent one. + + ``@large-integers`` and ``@events`` on one scenario, neither declared: the + marker iterator decides which comes first, and the message must not. The + language-wide reason is the true one -- the provider's declaration could not + have made this scenario run. + """ + node = types.SimpleNamespace( + iter_markers=lambda: iter( + [ + types.SimpleNamespace(name=Capability.EVENTS.value), + types.SimpleNamespace(name=one_inexpressible.value), + ] + ) + ) + request = types.SimpleNamespace( + node=node, + getfixturevalue=lambda name: _config(capabilities=frozenset()), + ) + + with pytest.raises(pytest.skip.Exception) as raised: + plugin.capability_gate(typing.cast("pytest.FixtureRequest", request)) + + assert _AS_IN_JAVA in str(raised.value) + + +def test_the_gate_still_skips_for_a_declaration_nobody_made() -> None: + """The unchanged half, checked here because the branch above is new. + + Nothing is patched: ``@events`` is expressible and undeclared, which is the + ordinary case and the one that must keep its own wording. + """ + node = types.SimpleNamespace( + iter_markers=lambda: iter([types.SimpleNamespace(name=Capability.EVENTS.value)]) + ) + request = types.SimpleNamespace( + node=node, + getfixturevalue=lambda name: _config(capabilities={Capability.OBJECT}), + ) + + with pytest.raises(pytest.skip.Exception) as raised: + plugin.capability_gate(typing.cast("pytest.FixtureRequest", request)) + + message = str(raised.value) + assert "provider does not declare capability @events" in message + assert "Declared: @object" in message + + +# -- acknowledging a gap ----------------------------------------------------- + + +ISSUE = "https://github.com/open-feature/python-sdk/issues/619" + + +def test_a_known_deviation_carries_its_capability_only_when_it_has_one() -> None: + """A deviation against a mandatory, ungated scenario belongs to no capability. + + Omitted rather than null, because the field is the answer to "which + capability does this concern" and there is not always one. + """ + mandatory = KnownDeviation.tracked( + summary="a boolean satisfies an Integer", issue=ISSUE + ) + assert mandatory.as_json() == {"issue": ISSUE, "summary": mandatory.summary} + + attributed = KnownDeviation.tracked( + summary="a lossy float satisfies an Integer", + issue=ISSUE, + capability=Capability.NUMERIC_COERCION, + ) + assert attributed.as_json() == { + "issue": ISSUE, + "summary": attributed.summary, + "capability": Capability.NUMERIC_COERCION.tag, + } + + +def test_an_untracked_deviation_is_a_form_of_its_own() -> None: + """Because a gap with nowhere to point at is still worth naming. + + Naming the defect is what separates it from a capability the provider chose + to withhold; a declaration that merely omits the tag cannot say which of the + two happened. This suite had no way to record one until now -- ``issue`` was + required -- and was the only one of the four that had not. + + ``issue`` is left out of the payload rather than sent as null: the report + schema types it as a uri-formatted string when present, and requires only + ``summary``. + """ + untracked = KnownDeviation.untracked( + summary="the lossy half of the coercion rule is not enforced", + capability=Capability.NUMERIC_COERCION, + ) + + assert not untracked.is_tracked + assert untracked.issue is None + assert untracked.as_json() == { + "summary": untracked.summary, + "capability": Capability.NUMERIC_COERCION.tag, + } + + assert KnownDeviation.tracked(summary="a gap", issue=ISSUE).is_tracked + + +def test_a_deviation_with_no_summary_is_refused() -> None: + """It records that something is wrong without saying what. + + Which leaves a reader worse off than the bare skip or failure it + accompanies, and it is the one field the report schema requires. Refused at + construction, where the adopter's own code is still on the stack to say + which line to fix. + """ + with pytest.raises(ValueError, match="known_deviations\\[0\\] has no summary"): + _config(known_deviations=[KnownDeviation.untracked(summary=" ")]) + + +def test_a_deviation_may_not_name_a_reserved_capability() -> None: + """No scenario carries the tag, so there is nothing to deviate from. + + The same reason declaring one is refused: there is no failure and no skip + for the deviation to explain, so the entry would tell a reader only that + something was claimed about something nothing examined. + """ + reserved = next(iter(RESERVED_CAPABILITIES)) + with pytest.raises(ValueError) as raised: + _config( + known_deviations=[ + KnownDeviation.untracked(summary="a gap", capability=reserved) + ] + ) + + message = str(raised.value) + assert f"names the reserved capability {reserved.tag}" in message + assert "nothing was failed or skipped" in message + + +def test_known_deviations_are_normalised_and_change_nothing_about_the_run() -> None: + """Declared as any sequence; read as a tuple. + + And that is all they do. A deviation is an acknowledgement, not a licence: + nothing here makes a scenario pass, skip, or be collected differently, which + is why a suite declaring one still fails on it. + """ + deviation = KnownDeviation.tracked( + summary="a gap", issue="https://example.invalid/1" + ) + config = _config(known_deviations=[deviation]) + assert config.known_deviations == (deviation,) + assert _config().known_deviations == () + assert _config(known_deviations=[deviation]).capabilities == _config().capabilities + + +# -- saying how the backend is driven ---------------------------------------- + + +def test_a_control_that_does_not_say_is_not_a_backend_control() -> None: + """``control_api`` is required, and the protocol is where that is enforced. + + Nothing outside a control can tell whether it spoke the normative HTTP API + or reached into this process, which is the argument for making the control + say rather than for letting the field be absent: every run is one or the + other, so an omitted value is not "no claim made" but an unfalsifiable one. + + ``BackendControl`` is runtime-checkable, so this is checked at the seam as + well as by the type checker -- which matters for an adopter who writes a + custom control in an untyped test module. + """ + + class _Quiet: + def prepare_scenario(self) -> None: ... + + def change_flag(self) -> None: ... + + @property + def description(self) -> str: + return "a control that will not say" + + assert not isinstance(_Quiet(), BackendControl) + assert isinstance(_StubControl(), BackendControl) + + +def test_in_process_control_says_it_is_in_process() -> None: + """The narrow allowance, and the control that exists to take it. + + A provider that does have a backend and reports this is claiming something + it should not, which is only detectable if the honest case says so plainly. + """ + control = InProcessControl() + assert isinstance(control, BackendControl) + assert control.control_api == "in-process" + + +def test_the_control_api_type_is_closed_to_the_two_values_the_schema_allows() -> None: + """Closed, so a third value is a type error rather than an invalid report. + + ``ControlApi`` is exported for exactly this: a custom control annotates its + own property with it and the type checker refuses ``"HTTP"`` or ``"grpc"`` + before either becomes a report that fails schema validation with nothing to + point at locally. + """ + assert typing.get_args(ControlApi) == ("http", "in-process") diff --git a/tools/openfeature-tck/tests/test_extensions.py b/tools/openfeature-tck/tests/test_extensions.py new file mode 100644 index 00000000..a84bbeb0 --- /dev/null +++ b/tools/openfeature-tck/tests/test_extensions.py @@ -0,0 +1,542 @@ +"""What an adopter's own scenarios may and may not do. + +An adopter with provider-specific behaviour -- flagd's ``fractional`` targeting, a +proprietary rollout rule -- has to be able to pin it in the same run as the +contract it sits on top of, or they end up maintaining a second harness beside +the one the TCK gives them. So the properties checked here are the ones that make +that safe rather than merely possible: + +* an extension scenario runs **inside** the canonical suite -- same session, same + provider registration, same backend control -- with a step definition the + adopter wrote in their own ``conftest.py`` and nothing else registered; +* an adoption without extensions runs exactly what it ran before, scenario for + scenario and outcome for outcome; +* a feature file's identity comes from where the file is, so an extension cannot + take a canonical scenario's. + +The last is not hypothetical. Java's suite discovered a same-named feature file +in a second classpath root silently *replacing* the canonical one, and the run +went green having asked the adopter's questions instead of the specification's. +The Python route to the same place is narrower and just as quiet: pytest-bdd +names a feature file by its parent directory joined to its own name, so a file at +``extensions/gherkin/errors.feature`` arrives under the uri the canonical +``errors.feature`` already occupies. + +The first three are properties of how a whole session runs rather than of what a +function returns, so they are checked against real pytest sessions in +subprocesses, read back through pytest's own JUnit XML. Reading them back from a +conformance report would be circular here and impossible anyway: this package +writes none. +""" + +from __future__ import annotations + +import dataclasses +import subprocess +import sys +import xml.etree.ElementTree as ElementTree +from pathlib import Path + +import pytest + +from openfeature.contrib.tools.tck import ( + EXTENSIONS_DIRECTORY, + canonical_root, + feature_paths, +) +from openfeature.contrib.tools.tck.extensions import ( + CANONICAL_DIRECTORY, + EXTENSIONS_URI_PREFIX, + collision_problem, + extension_root, + is_canonical, + is_canonical_uri, + reserved_prefix_problem, + uri_collisions, + uri_for, +) + + +def _canonical_root() -> Path: + """The packaged canonical directory, or a failure that says how to get one. + + ``canonical_root()`` answers ``None`` when the assets are not on a + filesystem, which is the honest answer for a zipimport and a missing build + step everywhere else. + """ + root = canonical_root() + assert root is not None, ( + "the packaged canonical features must be on a filesystem for this file " + "to have anything to say; run `poe sync-spec-assets` first" + ) + return root + + +CANONICAL_ROOT = _canonical_root() + +CANONICAL_FEATURE = "errors.feature" +"""The canonical file the collision cases are written against, chosen because it +is the one whose scenarios an extension could most plausibly want to restate.""" + +VENDOR_SCENARIO = "A vendor rule resolves through the suite's own provider" + + +# -- the generated adoption -------------------------------------------------- + +_SUITE_MODULE = '''\ +"""A one-fixture adoption, generated so extensions can be checked end to end.""" + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.tck import ( + Capability, + InProcessControl, + TckConfig, + feature_paths, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="{name}", + control=control, + new_provider=control.new_provider, + capabilities={{Capability.EVENTS, Capability.OBJECT}}, + ) + + +scenarios(*feature_paths()) +''' + +# The step the adopter writes, in the adopter's own conftest.py and nowhere else. +# It asks the TCK's own per-scenario state what happened, which is what makes +# "the same session and the same provider" checkable rather than asserted: a +# second harness would have a second provider, or none, and none of the canonical +# steps would have run. +_CONFTEST_MODULE = """\ +import pytest +from pytest_bdd import then + +from openfeature.contrib.tools.tck import TckState + +DEVIATION = "[boolean-flag-Integer-1]" + + +@then("the vendor rule ran against the provider the suite registered") +def vendor_rule_ran(tck_state: TckState) -> None: + assert tck_state.client is not None, "no provider was registered" + assert tck_state.last is not None, "the canonical steps did not run here" + assert tck_state.last.value == "hi", tck_state.last + + +def pytest_collection_modifyitems(items): + for item in items: + if item.name.endswith(DEVIATION): + item.add_marker(pytest.mark.xfail(reason="python-sdk#619")) +""" + +# Deliberately reuses the canonical step vocabulary and adds exactly one step of +# its own, which is the shape an adopter's feature file actually takes. +_VENDOR_FEATURE = """\ +Feature: Vendor rules + + Background: + Given a stable provider + + Scenario: A vendor rule resolves through the suite's own provider + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" + And the vendor rule ran against the provider the suite registered +""" + + +# -- reading a run back ------------------------------------------------------ + + +@dataclasses.dataclass(frozen=True) +class Run: + """One subprocess run of a generated adoption.""" + + directory: Path + result: subprocess.CompletedProcess[str] + outcomes: dict[str, str] + + +def _outcomes(report: Path) -> dict[str, str]: + """Read a JUnit XML report into ``node id -> passed | failed | skipped``. + + pytest's own results format, because the question is what the session did + and pytest is the thing that knows. It records one ``testcase`` per test with + the file it came from, which is what lets two suites in one directory be + told apart. + """ + outcomes: dict[str, str] = {} + # Not untrusted input: the file is one pytest wrote seconds ago, in a + # temporary directory this test made, from a subprocess this test started. + root = ElementTree.parse(report).getroot() # noqa: S314 + for case in root.iter("testcase"): + statuses = { + "failure": "failed", + "error": "failed", + "skipped": "skipped", + } + status = "passed" + for tag, named in statuses.items(): + if case.find(tag) is not None: + status = named + break + node = f"{case.get('classname', '')}::{case.get('name', '')}" + outcomes[node] = status + return outcomes + + +def _run( + tmp_path_factory: pytest.TempPathFactory, + modules: dict[str, str], + features: dict[str, str] | None = None, +) -> Run: + """Write an adoption, run it in a subprocess, and read the results back. + + Both mappings are keyed by a path relative to the adoption directory, so a + case can put a module or a feature file wherever the property under test + needs it. + """ + directory = tmp_path_factory.mktemp("adoption") + for relative, body in {**modules, **(features or {})}.items(): + path = directory / Path(relative) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(body, encoding="utf-8") + + report = directory / "results.xml" + result = subprocess.run( # noqa: S603 + [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "no:cacheprovider", + f"--junitxml={report}", + str(directory), + ], + capture_output=True, + text=True, + check=False, + ) + outcomes = _outcomes(report) if report.exists() else {} + return Run(directory=directory, result=result, outcomes=outcomes) + + +def _suite(name: str) -> str: + return _SUITE_MODULE.format(name=name) + + +BASELINE_DIRECTORY = "baseline" +"""Where the adoption that has no extensions of its own lives. + +A subdirectory rather than a second module beside the first, because both +adoptions now write the *same* line -- ``scenarios(*feature_paths())`` -- and +what distinguishes them is where they are written. ``feature_paths()`` looks for +an ``extensions`` directory beside the calling module, so a module one level +down is an adopter with none, in a session where one exists a directory away. + +There used to be a second public call, ``features_path()``, which returned the +canonical set alone and was what ``before`` used. It is gone: the two calls +differed by one character and the shorter one silently dropped the extensions +directory, so an adopter who reached for it got a green run over fewer scenarios +than they believed they had run. +""" + + +@pytest.fixture(scope="module") +def adoption(tmp_path_factory: pytest.TempPathFactory) -> Run: + """One session running two adoptions of the same provider. + + ``before`` is an adopter with no extensions directory of their own; ``after`` + is the same adoption with an ordinary extension beside it. Having both in one + run is what lets "an extension adds and does not alter" be a comparison + rather than a number written down here. + + One session rather than two, because a subprocess pytest run is by far the + most expensive thing in this file and the two suites are independent: each + resolves its own ``TckConfig`` under its own OpenFeature domain. + """ + return _run( + tmp_path_factory, + { + f"{BASELINE_DIRECTORY}/test_before.py": _suite("before"), + "test_after.py": _suite("after"), + "conftest.py": _CONFTEST_MODULE, + }, + {f"{EXTENSIONS_DIRECTORY}/vendor.feature": _VENDOR_FEATURE}, + ) + + +def _of(run: Run, module: str) -> dict[str, str]: + """The outcomes belonging to one of the generated suites, keyed by test name. + + JUnit XML names the module in dotted form, so the two generated suites in one + directory are told apart by the last segment of it. + """ + return { + node.split("::", 1)[1]: status + for node, status in run.outcomes.items() + if node.split("::", 1)[0].rsplit(".", 1)[-1] == module + } + + +def _contributed(run: Run) -> dict[str, str]: + """What the extension added: the tests ``after`` ran and ``before`` did not. + + Identified by difference rather than by name. pytest-bdd derives a test + function's name from the scenario name by a munging of its own -- an + apostrophe disappears where a space becomes an underscore -- and reproducing + that here would pin pytest-bdd's spelling rather than this package's + behaviour. + """ + before = _of(run, "test_before") + return { + name: status + for name, status in _of(run, "test_after").items() + if name not in before + } + + +# -- an extension runs inside the canonical suite ---------------------------- + + +def test_an_extension_scenario_runs_in_the_canonical_suite(adoption: Run) -> None: + """One suite, one session, both sets of scenarios in it. + + The extension scenario passes only if it reached the provider the suite + registered and the canonical steps ran in it, because that is what its own + step asserts -- so this is not a presentational fact about collection. It is + the same suite, which is the same provider registration and the same backend + control. + """ + assert adoption.result.returncode == 0, adoption.result.stdout + + contributed = _contributed(adoption) + assert len(contributed) == 1, adoption.outcomes + [(name, status)] = contributed.items() + assert "vendor_rule" in name, name + assert status == "passed" + + canonical = _of(adoption, "test_after") + assert len(canonical) > 1, "the canonical scenarios must have run too" + assert "passed" in canonical.values() + + +def test_the_extension_step_came_from_the_adopters_conftest(adoption: Run) -> None: + """Nothing was registered, imported or configured to make that step resolve. + + pytest collects ``conftest.py`` on its own and pytest-bdd resolves steps + through the fixture system, so a step defined beside the test module is in + scope for the scenarios generated into it. An unresolved step is a *failure* + rather than an omission, which is why asserting that the scenario passed is + enough to pin this. + """ + conftest = (adoption.directory / "conftest.py").read_text(encoding="utf-8") + assert "the vendor rule ran against the provider the suite registered" in conftest + assert set(_contributed(adoption).values()) == {"passed"} + + +# -- and changes nothing for an adopter who has none ------------------------- + + +def test_an_extension_adds_scenarios_and_alters_none(adoption: Run) -> None: + """``before`` is what an adopter ran before extensions existed. + + Every test it generated, ``after`` generated too, with the same outcome, and + the only difference between the two is the one scenario the extension added. + An adopter who has no extensions is the same comparison with the right-hand + side empty, which is what ``feature_paths()`` returning the canonical path + alone makes true by construction rather than by luck. + """ + before = _of(adoption, "test_before") + after = _of(adoption, "test_after") + assert before, "the baseline suite generated nothing" + + assert len(_contributed(adoption)) == 1 + assert {name: after[name] for name in before} == before + + +def test_a_module_with_no_extensions_directory_beside_it_sees_no_extension( + adoption: Run, +) -> None: + """An extension belongs to the module it sits beside, and to no other. + + ``before`` is one directory below the ``extensions`` directory this session + has, which is as close as an adopter with none can get to having one. It runs + the canonical set and nothing else, so the directory a module is in -- not + the session it runs in -- is what decides what it collects. + """ + assert (adoption.directory / EXTENSIONS_DIRECTORY).is_dir() + assert not (adoption.directory / BASELINE_DIRECTORY / EXTENSIONS_DIRECTORY).exists() + assert set(_of(adoption, "test_before")) < set(_of(adoption, "test_after")) + + +# -- finding an adopter's feature files -------------------------------------- + + +def test_feature_paths_is_the_canonical_set_when_there_is_no_extension_directory() -> ( + None +): + """This test module has no ``extensions`` beside it, and gets one path.""" + assert not (Path(__file__).parent / EXTENSIONS_DIRECTORY).exists() + assert [Path(path).resolve() for path in feature_paths()] == [CANONICAL_ROOT] + + +def test_an_extension_directory_counts_only_when_it_is_a_directory( + tmp_path: Path, +) -> None: + """``None`` rather than a path that contributes nothing. + + So that an adopter without extensions hands ``scenarios()`` exactly what + they handed it before -- and so that a *file* of that name, which pytest-bdd + would choke on, is not offered as a feature directory. + """ + assert extension_root(tmp_path) is None + + (tmp_path / EXTENSIONS_DIRECTORY).write_text("not a directory", encoding="utf-8") + assert extension_root(tmp_path) is None + + (tmp_path / EXTENSIONS_DIRECTORY).unlink() + (tmp_path / EXTENSIONS_DIRECTORY).mkdir() + assert extension_root(tmp_path) == tmp_path / EXTENSIONS_DIRECTORY + + +def test_the_canonical_features_are_found_inside_the_distribution() -> None: + """No submodule and no directory layout of the adopter's own.""" + packaged = CANONICAL_ROOT + assert (packaged / CANONICAL_FEATURE).is_file() + assert is_canonical(packaged / CANONICAL_FEATURE) + assert not is_canonical(Path(__file__)) + + +# -- deriving the uri -------------------------------------------------------- + + +def test_the_two_prefixes_are_the_ones_appendix_f_names() -> None: + """Pinned as literals, because every other assertion here uses the constants. + + Those assertions hold whatever the constants say, so renaming one would leave + the suite green while the uris it emits stopped joining with another + language's -- which is the failure that happened. Appendix F fixes both + strings: a canonical feature is identified by its path relative to the + specification's asset directory, and ``gherkin`` is the directory it occupies + there; an extension mounts under ``extensions``. + """ + assert CANONICAL_DIRECTORY == "gherkin" + assert EXTENSIONS_URI_PREFIX == "extensions" + assert CANONICAL_DIRECTORY != EXTENSIONS_DIRECTORY, ( + "an extensions directory sharing the canonical name is how an extension " + "comes to occupy a canonical file's identity" + ) + + +def test_the_canonical_assets_keep_the_reserved_prefix() -> None: + canonical = CANONICAL_ROOT / CANONICAL_FEATURE + assert uri_for(canonical) == f"{CANONICAL_DIRECTORY}/{CANONICAL_FEATURE}" + assert is_canonical_uri(f"{CANONICAL_DIRECTORY}/{CANONICAL_FEATURE}") + assert ( + reserved_prefix_problem(f"{CANONICAL_DIRECTORY}/{CANONICAL_FEATURE}", canonical) + is None + ) + + +def test_an_extension_keeps_its_layout_below_the_extensions_prefix( + tmp_path: Path, +) -> None: + """Whatever the adopter's own directory layout under the root looks like. + + Including one that reproduces the canonical name, which is the collision the + derivation exists for: pytest-bdd would have called the second of these + ``gherkin/errors.feature``. + """ + root = tmp_path / EXTENSIONS_DIRECTORY + assert uri_for(root / "vendor.feature") == "extensions/vendor.feature" + assert ( + uri_for(root / CANONICAL_DIRECTORY / CANONICAL_FEATURE) + == f"{EXTENSIONS_URI_PREFIX}/{CANONICAL_DIRECTORY}/{CANONICAL_FEATURE}" + ) + assert ( + uri_for(root / "a" / "b" / "vendor.feature") == "extensions/a/b/vendor.feature" + ) + assert not is_canonical_uri( + f"{EXTENSIONS_URI_PREFIX}/{CANONICAL_DIRECTORY}/{CANONICAL_FEATURE}" + ) + + +def test_a_derived_uri_is_slash_separated_on_every_platform(tmp_path: Path) -> None: + """A uri identifies a feature file, so it cannot depend on where it ran. + + The paths these are built from are ``pathlib`` paths, which are + backslash-separated on Windows. A run there has to be comparable with one on + Linux, and it is not if the same file is identified two ways. + """ + nested = tmp_path / EXTENSIONS_DIRECTORY / "a" / "b" / "vendor.feature" + derived = uri_for(nested) + assert derived == "extensions/a/b/vendor.feature" + assert derived is not None and "\\" not in derived + + +def test_a_file_that_is_neither_is_left_to_pytest_bdd(tmp_path: Path) -> None: + """``None`` rather than a guess: the caller falls back to what the runner said.""" + assert uri_for(tmp_path / "loose.feature") is None + + +def test_a_local_file_under_the_reserved_prefix_is_a_problem(tmp_path: Path) -> None: + """The one route to a canonical-looking uri the convention cannot close. + + An adopter may still hand ``scenarios()`` a directory of their own named + ``gherkin``, and its files are then named exactly as canonical ones would + be. Reported rather than raised: the scenarios are the adopter's to run, and + it is publishing them as the specification's that has to be refused. + """ + local = tmp_path / CANONICAL_DIRECTORY / "local.feature" + problem = reserved_prefix_problem(f"{CANONICAL_DIRECTORY}/local.feature", local) + assert problem is not None + assert EXTENSIONS_DIRECTORY in problem, "the message must say the fix" + assert str(local) in problem + + +def test_two_files_that_would_share_one_uri_are_reported(tmp_path: Path) -> None: + """Deriving the uri from the location narrows the collision; it does not end it. + + An ``extensions`` directory nested inside another one reaches the same uri + as its namesake at the root, and so would two test modules sharing one + ``tck_config``. + """ + root = tmp_path / EXTENSIONS_DIRECTORY + nested = root / "nested" / EXTENSIONS_DIRECTORY / "vendor.feature" + collisions = uri_collisions( + [ + ("extensions/vendor.feature", root / "vendor.feature"), + ("extensions/vendor.feature", nested), + ] + ) + assert set(collisions) == {"extensions/vendor.feature"} + + problem = collision_problem( + "extensions/vendor.feature", collisions["extensions/vendor.feature"] + ) + assert "2 different feature files" in problem + assert EXTENSIONS_DIRECTORY in problem + + +def test_distinct_extension_paths_do_not_collide(tmp_path: Path) -> None: + """The layouts that are fine, including the one that only looks like a clash.""" + root = tmp_path / EXTENSIONS_DIRECTORY + assert not uri_collisions( + [ + ("extensions/vendor.feature", root / "vendor.feature"), + ("extensions/a/vendor.feature", root / "a" / "vendor.feature"), + # One file reached by two routes is one file, not a collision. + ("extensions/vendor.feature", root / "a" / ".." / "vendor.feature"), + ] + ) diff --git a/tools/openfeature-tck/tests/test_http_control.py b/tools/openfeature-tck/tests/test_http_control.py new file mode 100644 index 00000000..97bd5ef0 --- /dev/null +++ b/tools/openfeature-tck/tests/test_http_control.py @@ -0,0 +1,333 @@ +"""What the Gherkin cannot assert about the HTTP control path. + +Every scenario's isolation rests on :meth:`HttpControl.prepare_scenario` doing +the right thing against a backend that implements only part of the control API, +and on a disconnect being remembered. Both are invisible from inside a scenario: +a control that silently did nothing would leave each scenario running against +whatever state the previous one left behind, and the suite would report those +results as conformance. + +So the control API is stubbed with :mod:`http.server` -- no Docker, no network +beyond loopback -- and the requests it actually made are asserted. +""" + +from __future__ import annotations + +import threading +import typing +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +from openfeature.contrib.tools.tck import ( + ControlApiError, + HttpControl, +) + + +class _StubControlApi: + """A control API that records every request and answers a scripted status.""" + + def __init__( + self, + statuses: dict[str, int] | None = None, + sequences: dict[str, list[int]] | None = None, + ) -> None: + self.requests: list[tuple[str, str, str]] = [] + """(method, path, query) of every request, in order.""" + + self.statuses = statuses or {} + self.sequences = sequences or {} + """Per-path statuses consumed one per request, for "not yet, then yes".""" + + stub = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: + self._answer("POST") + + def do_GET(self) -> None: + self._answer("GET") + + def _answer(self, method: str) -> None: + path, _, query = self.path.partition("?") + stub.requests.append((method, path, query)) + status = stub.statuses.get(path, 200) + # Scripted as a list to answer differently on each call, which + # is how "not ready, then ready" is expressed. + sequence = stub.sequences.get(path) + if sequence: + status = sequence.pop(0) + body = b'{"status":"stub"}' + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args: typing.Any) -> None: + """Silence the default stderr logging.""" + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + + @property + def base_url(self) -> str: + host, port = self._server.server_address[:2] + return f"http://{host!s}:{port}" + + @property + def paths(self) -> list[str]: + return [path for _, path, _ in self.requests] + + def __enter__(self) -> _StubControlApi: + self._thread.start() + return self + + def __exit__(self, *_exc: object) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join(timeout=5) + + +@pytest.fixture +def stub() -> typing.Iterator[_StubControlApi]: + with _StubControlApi() as api: + yield api + + +def test_prepare_scenario_prefers_reset_when_the_backend_implements_it( + stub: _StubControlApi, +) -> None: + """The preferred primitive, because it causes no availability blip. + + A ``/start`` between scenarios restarts the backend process, which a + provider observes as an outage and may report as a lifecycle event in the + scenario that follows. + """ + control = HttpControl(stub.base_url) + + control.prepare_scenario() + control.prepare_scenario() + + assert stub.paths == ["/reset", "/reset"] + + +def test_prepare_scenario_falls_back_to_start_and_remembers_the_answer() -> None: + """The path flagd-testbed actually takes: its launchpad has no ``/reset``. + + The fallback must be probed once rather than once per scenario -- a wasted + 404 before every scenario is a slow suite, and hiding the probe entirely + would mean a backend that grows ``/reset`` never gets used properly. + """ + with _StubControlApi({"/reset": 404}) as stub: + control = HttpControl(stub.base_url) + + control.prepare_scenario() + control.prepare_scenario() + control.prepare_scenario() + + assert stub.paths == ["/reset", "/start", "/start", "/start"] + + +@pytest.mark.parametrize("status", [404, 501]) +def test_both_documented_not_implemented_statuses_trigger_the_fallback( + status: int, +) -> None: + """The OpenAPI document permits either, so neither may be treated as a failure.""" + with _StubControlApi({"/reset": status}) as stub: + HttpControl(stub.base_url).prepare_scenario() + + assert stub.paths == ["/reset", "/start"] + + +def test_the_scenario_after_a_disconnect_starts_the_backend( + stub: _StubControlApi, +) -> None: + """``/reset`` restores flag state; it is not specified to start a stopped backend. + + Without this the scenario following a disconnect would prepare a backend + that is still down, register a provider against it, and report the failure + as a provider defect. + """ + control = HttpControl(stub.base_url) + control.prepare_scenario() # settles on /reset, which this stub supports + stub.requests.clear() + + control.disconnect() + control.prepare_scenario() + + assert stub.paths == ["/stop", "/start"] + + +def test_reconnect_starts_the_backend_and_clears_the_disconnect( + stub: _StubControlApi, +) -> None: + """A scenario that ended its own outage leaves the backend up, so ``/reset`` is fine again.""" + control = HttpControl(stub.base_url) + control.prepare_scenario() + control.disconnect() + control.reconnect() + stub.requests.clear() + + control.prepare_scenario() + + assert stub.paths == ["/reset"] + + +def test_start_names_the_configuration_under_test() -> None: + """``default`` is the only name every backend must support, and it serves the canonical set.""" + with _StubControlApi({"/reset": 404}) as stub: + HttpControl(stub.base_url).prepare_scenario() + + assert ("POST", "/start", "config=default") in stub.requests + + +def test_a_custom_backend_configuration_is_carried_through() -> None: + with _StubControlApi({"/reset": 404}) as stub: + HttpControl(stub.base_url, backend_configuration="ssl").prepare_scenario() + + assert ("POST", "/start", "config=ssl") in stub.requests + + +def test_change_flag_posts_to_change(stub: _StubControlApi) -> None: + HttpControl(stub.base_url).change_flag() + + assert stub.paths == ["/change"] + + +def test_a_failed_control_call_raises_rather_than_passing_silently() -> None: + """A control call that did nothing would leave the next scenario in an unknown state.""" + with ( + _StubControlApi({"/change": 500}) as stub, + pytest.raises(ControlApiError, match="500"), + ): + HttpControl(stub.base_url).change_flag() + + +def test_an_unreachable_control_api_raises_with_the_reason() -> None: + """The control API must stay up even while the backend is deliberately down.""" + # Bound and immediately closed, so the port is almost certainly free. + with _StubControlApi() as stub: + base_url = stub.base_url + control = HttpControl(base_url, timeout=2.0) + + with pytest.raises(ControlApiError, match="control request POST"): + control.change_flag() + + +@pytest.mark.parametrize( + "base_url", + ["", "localhost:8080", "file:///etc/passwd", "ftp://localhost:8080"], +) +def test_a_base_url_that_is_not_an_http_url_is_rejected_at_construction( + base_url: str, +) -> None: + """Rejected early, and by scheme, so no other URL scheme can reach ``urlopen``.""" + with pytest.raises(ValueError, match="not an http"): + HttpControl(base_url) + + +def test_a_trailing_slash_does_not_produce_a_double_slash_path() -> None: + with _StubControlApi() as stub: + HttpControl(stub.base_url + "/").change_flag() + + assert stub.paths == ["/change"] + + +def test_the_control_reports_which_api_it_drives_the_backend_through() -> None: + """The required ``BackendControl`` member, answered here without qualification. + + Every operation on this class is an HTTP request to the normative control + API, so this is the one control that can answer the question flatly. A + report whose ``backend.controlApi`` says otherwise for a provider with a + real backend is claiming something it should not. + """ + with _StubControlApi() as stub: + assert HttpControl(stub.base_url).control_api == "http" + + +def test_there_is_no_binding_for_restart() -> None: + """``/restart`` is optional in the control API and no scenario reaches it. + + A binding nothing can call would imply every backend under test owes the + endpoint, which is what the specification's own description wrongly claimed + before it was corrected. The disconnect/reconnect scenario is an unbounded + outage: ``disconnect`` then ``reconnect``. + """ + assert not hasattr(HttpControl, "restart") + + +# -- waiting for the control API --------------------------------------------- + + +def test_await_ready_returns_as_soon_as_healthz_answers( + stub: _StubControlApi, +) -> None: + """One probe against the control API itself, not a pause of a fixed length. + + The readiness check is what replaced Java's post-command settle. It probes + the thing whose readiness is in question, so a control API that is slow to + come up is waited for and one that never does is reported. + """ + control = HttpControl(stub.base_url) + + control.await_ready(timeout=5.0) + + assert stub.requests == [("GET", "/healthz", "")] + + +def test_await_ready_treats_an_unimplemented_healthz_as_ready() -> None: + """404 is "not implemented", which ``control-api.yaml`` defines as ready. + + Readiness then rests on the control port accepting a connection, which the + Compose harness has already established before it gets here. The reference + backend -- flagd-testbed's launchpad -- serves no ``/healthz`` at all, so + this is the normal path today rather than an edge case. + """ + with _StubControlApi({"/healthz": 404}) as stub: + HttpControl(stub.base_url).await_ready(timeout=5.0) + assert stub.paths == ["/healthz"] + + +def test_await_ready_keeps_probing_while_the_control_api_says_not_yet() -> None: + """503 is the control API saying "not ready", so it is retried, not accepted.""" + with _StubControlApi(sequences={"/healthz": [503, 503, 200]}) as stub: + HttpControl(stub.base_url).await_ready(timeout=10.0) + assert stub.paths == ["/healthz", "/healthz", "/healthz"] + + +def test_await_ready_gives_up_with_what_the_last_probe_saw() -> None: + """A stack that never becomes ready has to say what it was answering. + + "did not become ready" on its own sends an adopter to the wrong place: a + connection refused is a stack that is not up, a 503 is one that is up and + not finished. + """ + with _StubControlApi({"/healthz": 503}) as stub: + control = HttpControl(stub.base_url) + with pytest.raises(ControlApiError) as raised: + control.await_ready(timeout=0.3) + + message = str(raised.value) + assert "was not ready within" in message + assert "HTTP 503" in message + + +def test_await_ready_reports_a_control_api_that_is_not_there_at_all() -> None: + """Which is a stack that did not start, and reads differently from a 503.""" + with _StubControlApi() as stub: + base_url = stub.base_url + # The server is closed, so nothing is listening on that port any more. + + control = HttpControl(base_url) + with pytest.raises(ControlApiError) as raised: + control.await_ready(timeout=0.3) + + assert "not reachable" in str(raised.value) + + +def test_base_url_is_reportable_without_rebuilding_it() -> None: + """Whatever brought the stack up logs where the control API ended up.""" + with _StubControlApi() as stub: + assert HttpControl(stub.base_url + "/").base_url == stub.base_url diff --git a/tools/openfeature-tck/tests/test_in_memory_conformance.py b/tools/openfeature-tck/tests/test_in_memory_conformance.py new file mode 100644 index 00000000..03bba615 --- /dev/null +++ b/tools/openfeature-tck/tests/test_in_memory_conformance.py @@ -0,0 +1,178 @@ +"""Run the conformance suite against the SDK's own in-memory provider. + +This is the TCK's self-test, and it earns its keep twice over. + +It is the **reference adoption** for a provider with no backend. Everything a +file-based or environment-variable provider has to write is here: one fixture +and one call. + +It is also the **Docker-free canary**. Needing no container and no network, it +runs in a fraction of a second, which makes it the fast check that catches a +broken step definition, a mis-wired capability gate or a regression in the +shared harness long before a containerised suite would. + +What it does not do is license providers that have a backend to test themselves +this way -- see ``BackendControl`` for why. +""" + +from __future__ import annotations + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.tck import ( + Capability, + ControlApi, + TckConfig, + canonical_flag_set, + feature_paths, +) +from openfeature.provider import FeatureProvider +from openfeature.provider.in_memory_provider import InMemoryProvider + + +class PlainMemoryControl: + """Backend control for the SDK's stock in-memory provider. + + ``prepare_scenario`` is a no-op because the provider is rebuilt from the + canonical flag set for every scenario, so each one already starts from an + untouched baseline. + + ``change_flag`` cannot be implemented at all, and the error says why. + Appendix A of the specification requires an SDK's in-memory provider to + "support a means of updating the flag set, resulting in the emission of + PROVIDER_CONFIGURATION_CHANGED events"; the Python SDK's copies its mapping + in the constructor and exposes no way to change it. The suite below + therefore leaves ``CONFIGURATION_CHANGE`` undeclared and the scenario is + reported as skipped with its reason, which is the honest outcome. Reaching + this error would mean the capability had been declared anyway. + """ + + @property + def description(self) -> str: + return "the Python SDK's InMemoryProvider, rebuilt per scenario" + + @property + def control_api(self) -> ControlApi: + """In-process, and honestly so: there is no backend to speak HTTP to.""" + return "in-process" + + def prepare_scenario(self) -> None: + return None + + def change_flag(self) -> None: + msg = ( + "openfeature.provider.in_memory_provider.InMemoryProvider cannot change its " + "flag set: it copies the mapping in its constructor and exposes no update " + "method, so a configuration change can be neither applied nor signalled. " + "Appendix A of the specification requires it. See " + "ControllableInMemoryProvider for what the SDK's provider is missing" + ) + raise NotImplementedError(msg) + + +def _new_provider() -> FeatureProvider: + return InMemoryProvider(canonical_flag_set()) + + +@pytest.fixture(scope="session") +def tck_config() -> TckConfig: + """Declare the provider under test and what it can do. + + Each omission is a fact about the provider rather than a convenience: + + * ``CONFIGURATION_CHANGE`` -- omitted because the SDK's in-memory provider + cannot update its flag set. That is a finding, not a configuration choice; + see ``PlainMemoryControl``. Withheld for a defect under the same Appendix F + self-test carve-out as ``DISABLED_FLAGS`` below, and pinned differently: + ``test_controllable_conformance`` *runs* these scenarios against + ``ControllableInMemoryProvider``, which supplies what the SDK lacks, so the + skip here is not the only record of them. + * ``STALE`` and ``UNAVAILABLE_INIT`` -- omitted because there is no + connection to lose. ``PlainMemoryControl`` does not implement + ``ConnectionControl`` for the same reason, and the two omissions keep each + other honest: the scenarios are skipped before any step can reach an + operation the control cannot perform. + * ``TARGETING`` -- omitted because this flag set has no targeting rule to + resolve. ``canonical-flags.json`` gives ``targeting-key-flag`` one, and + ``_decode_canonical_flags`` deliberately ignores the member: decoding a + rule language would make this package a second implementation of somebody + else's evaluator. So the flag is served at its ``miss`` default whatever + the context, the matching-context scenario would fail, and withholding + the capability is the honest report. That is a property of this in-memory + flag set rather than a defect in the SDK's provider, which is why nothing + here is a ``KnownDeviation``. + * ``CACHING`` -- omitted because no scenario carries the tag yet, so leaving + it out skips nothing. It is also reserved, so declaring it is refused. + * ``LIFECYCLE`` -- omitted because there is no backend to reach. The + capability asserts that initialisation actually contacts a backend and + that the outcome is observable; this provider's ``initialize`` is a no-op + and the SDK dispatches ``PROVIDER_READY`` around it regardless, so the + readiness scenario would pass here without testing anything. It passed + vacuously while the feature was gated on ``EVENTS``, which is precisely + the failure mode the split of ``@lifecycle`` from ``@events`` exists to + end. A skip with a reason is the honest outcome. + * ``DISABLED_FLAGS`` -- omitted because the SDK's in-memory provider ignores + a flag's ``state``. ``InMemoryFlag`` has a ``State`` enum with a + ``DISABLED`` member, ``_decode_canonical_flags`` reads the canonical + file's ``"state": "DISABLED"`` and passes it through faithfully, and + ``InMemoryFlag.resolve`` never looks at it -- so all four ``disabled-*`` + flags are served at their own default variant with reason ``STATIC``, + where the scenarios expect the caller's default. Measured before it was + gated: the four rows failed on the value, ``disabled-boolean-flag`` + resolving to ``True`` against a caller default of ``false``. + + Unlike ``NUMERIC_COERCION`` this is a field the SDK offers and does not + honour, so it is a defect (finding 4 in the README) -- and **withholding a + capability for a defect is what Appendix F's self-test carve-out + licenses, not something an adoption may copy**. This suite is a fixture + for the harness rather than a report about a third party, and it runs in + the ordinary build where a permanently failing scenario is a broken build + rather than a finding. The carve-out's condition is that the defect be + pinned by a test of its own, and it is: + ``test_every_packaged_flag_resolves_to_its_packaged_default_variant`` + sweeps the four flags with the rest and asserts each resolves to its own + default variant, so the behaviour is asserted rather than only skipped and + the sweep turns red the day the SDK honours ``DISABLED``. + * ``NUMERIC_COERCION`` -- omitted because the SDK's in-memory provider does + not coerce. It hands each variant back untouched, and the client's type + check is ``isinstance``-based, so ``integral-float-flag`` (``10.0``) + requested as an integer is a ``TYPE_MISMATCH`` rather than ``10``, and + ``integer-flag`` (``10``) requested as a float is one rather than + ``10.0``. The lossy scenario passes for the wrong reason -- every float + is rejected -- which is exactly what the two lossless scenarios exist to + catch, and declaring the tag would have them catch it here. This is the + withholding Appendix F still calls right after its correction: a provider + that does not attempt the coercion at all, rather than one that attempts + it and gets a direction wrong. It needs no carve-out and no deviation. + + ``LARGE_INTEGERS`` is declared: a Python ``int`` is unbounded and nothing + in this provider routes a value through a float. ``VARIANTS`` is declared + too: the in-memory flag set is keyed by variant name, so the provider has + one to report for every flag and does. + + ``STANDARD_REASONS`` is declared, and it was measured before it was: the six + scenarios this provider can reach all pass. ``InMemoryFlag.resolve`` reports + ``Reason.STATIC`` for every flag in the decoded set, so the four rule-less + rows hold; a missing flag and a type mismatch both arrive with reason + ``ERROR`` beside their error code. The other three scenarios in + ``reason.feature`` compose the tag with ``TARGETING`` and ``DISABLED_FLAGS``, + neither of which is declared here, so they skip with that reason -- which is + the capability working as intended rather than a gap: a reason cannot be + observed without the behaviour that produces it. + """ + return TckConfig( + name="in-memory", + control=PlainMemoryControl(), + new_provider=_new_provider, + capabilities={ + Capability.EVENTS, + Capability.OBJECT, + Capability.VARIANTS, + Capability.LARGE_INTEGERS, + Capability.STANDARD_REASONS, + }, + ) + + +scenarios(*feature_paths()) diff --git a/tools/openfeature-tck/tests/test_in_process_control.py b/tools/openfeature-tck/tests/test_in_process_control.py new file mode 100644 index 00000000..df890e14 --- /dev/null +++ b/tools/openfeature-tck/tests/test_in_process_control.py @@ -0,0 +1,493 @@ +"""Things the Gherkin cannot assert about itself. + +Each of these is a way the in-process control path could look correct while +quietly making the conformance suites meaningless. +""" + +from __future__ import annotations + +import json +import typing +from collections.abc import Callable + +import pytest + +from openfeature.contrib.tools.tck import ( + CHANGING_FLAG_KEY, + ConnectionControl, + ControllableInMemoryProvider, + InProcessControl, + canonical_flag_set, + canonical_flags_json, +) +from openfeature.contrib.tools.tck.provider import ( + _decode_canonical_flags, + changing_flag, +) +from openfeature.contrib.tools.tck.values import describe, values_equal +from openfeature.event import ProviderEvent +from openfeature.flag_evaluation import FlagType, Reason +from openfeature.provider.in_memory_provider import InMemoryFlag + +_NOT_SEEDED = "this default must never be what a seeded flag resolves to" +"""The default value handed to every resolver below. + +A flag the seeding dropped resolves to it rather than to anything from the file, +which is what the variant and error-code assertions are looking for. +""" + + +def _resolve_changing(provider: ControllableInMemoryProvider) -> str: + return provider.resolve_string_details(CHANGING_FLAG_KEY, "unset").value + + +def test_change_flag_actually_changes_the_resolved_value() -> None: + """The assumption every configuration-change scenario rests on. + + If ``change_flag`` emitted an event without altering what the provider + resolves, the scenario would still pass its event assertion and the suite + would be certifying a signal with nothing behind it. + """ + control = InProcessControl() + provider = control.new_provider() + assert isinstance(provider, ControllableInMemoryProvider) + + before = _resolve_changing(provider) + control.change_flag() + after = _resolve_changing(provider) + + assert before != after, "change_flag did not change the resolved value" + + +def test_change_flag_emits_a_configuration_change_event_naming_the_flag() -> None: + """The event the scenarios await is the provider's own, and it names the flag.""" + control = InProcessControl() + provider = control.new_provider() + + seen: list[tuple[ProviderEvent, list[str] | None]] = [] + + def record(_provider: object, event: ProviderEvent, details: object) -> None: + seen.append((event, getattr(details, "flags_changed", None))) + + # attach() is how the SDK registry wires a provider's emitter; doing it by + # hand keeps this a unit test of the provider rather than of the registry. + provider.attach(record) + control.change_flag() + + assert seen, "no event was emitted" + event, flags_changed = seen[-1] + assert event is ProviderEvent.PROVIDER_CONFIGURATION_CHANGED + assert flags_changed == [CHANGING_FLAG_KEY] + + +def test_change_does_not_leak_into_the_next_scenario() -> None: + """Scenario isolation. + + A leak here would make the suite order-dependent: a scenario running after + the configuration-change one would start with ``changing-flag`` already + flipped, and the failure would look like a provider defect. + """ + control = InProcessControl() + + first = control.new_provider() + assert isinstance(first, ControllableInMemoryProvider) + baseline = _resolve_changing(first) + + control.change_flag() + assert _resolve_changing(first) != baseline, ( + "precondition: change_flag had no effect" + ) + + control.prepare_scenario() + + second = control.new_provider() + assert isinstance(second, ControllableInMemoryProvider) + assert _resolve_changing(second) == baseline, ( + "the next scenario did not start from the baseline" + ) + + +def test_change_flag_without_a_provider_fails_clearly() -> None: + """In-process the flag store and the provider are the same object, so there is + nothing to change before one exists. Saying so beats an AttributeError.""" + control = InProcessControl() + with pytest.raises(RuntimeError, match="must create one"): + control.change_flag() + + +def test_in_process_control_does_not_pretend_to_have_a_connection() -> None: + """The load-bearing one. + + A no-op ``disconnect`` would report the ``@stale`` scenarios as passed + against a provider that cannot go stale -- precisely the silent-green + failure a conformance suite must never have. ``InProcessControl`` therefore + does not implement ``ConnectionControl`` at all, and the TCK turns that into + a skip with a reason. + """ + assert not isinstance(InProcessControl(), ConnectionControl), ( + "InProcessControl implements ConnectionControl: an in-memory provider has no " + "connection to lose, and a no-op implementation would make the @stale " + "scenarios pass without testing anything" + ) + + +def test_canonical_flag_set_omits_missing_flag() -> None: + """The property the FLAG_NOT_FOUND scenario depends on. + + Seeding ``missing-flag`` would turn that scenario green for the wrong + reason, and nothing else in the suite would notice. + """ + assert "missing-flag" not in canonical_flag_set() + + +def _flag_type_of(value: typing.Any) -> FlagType: + """The type a scenario would request a flag of this value as. + + ``bool`` first, because Python makes it a subclass of ``int`` and would + otherwise route ``boolean-zero-flag`` through the integer accessor. + """ + if isinstance(value, bool): + return FlagType.BOOLEAN + if isinstance(value, int): + return FlagType.INTEGER + if isinstance(value, float): + return FlagType.FLOAT + if isinstance(value, str): + return FlagType.STRING + return FlagType.OBJECT + + +_DISABLED_FLAGS: tuple[tuple[str, str], ...] = ( + ("disabled-boolean-flag", "boolean-flag"), + ("disabled-string-flag", "string-flag"), + ("disabled-integer-flag", "integer-flag"), + ("disabled-float-flag", "float-flag"), +) +"""The flags the canonical file marks ``DISABLED``, paired with what each mirrors. + +Written out rather than derived from the file's ``state`` members, because the +tests below read those members to decide what to assert and a check that reads +the member deciding its own answer checks nothing. Four names and their +counterparts are cheap to keep; a silently empty set is not. +""" + +_IGNORES_STATE = ( + ". That flag is DISABLED in the canonical file and is served anyway, because " + "InMemoryFlag carries a state and InMemoryFlag.resolve never reads it -- " + "which is why neither in-memory suite declares @disabled-flags. If this row " + "has begun to fail, the SDK has started honouring DISABLED and the " + "capability has become declarable for both of them" +) +"""Appended to a failure about one of the four disabled flags. + +They are in the sweep below rather than excluded from it, so a reader who hits +one is told what the flag is and what its failing means, instead of finding an +exclusion list and no reason for it. +""" + + +def test_every_packaged_flag_resolves_to_its_packaged_default_variant() -> None: + """The whole of what seeding from the file has to achieve. + + Every flag the packaged ``canonical-flags.json`` defines is served, under + the variant name the file gives as its ``defaultVariant``, with that + variant's value -- read back through the same typed resolver a scenario + would use, and compared the way the ``Then`` steps compare. + + Asserting the variant and the absence of an error code is what makes this + more than an equality check: a flag the seeding dropped resolves to the + default value with no variant and ``FLAG_NOT_FOUND``, and for the falsy + flags that fallback value can equal what was expected. + + **Including the four ``DISABLED`` flags, which is a finding rather than an + oversight.** Since spec revision ``009afe06`` the canonical set marks four + flags disabled, and the ``@disabled-flags`` scenarios expect each to resolve + to the caller's default. This provider resolves them to their own default + variant instead: the state survives decoding faithfully and + ``InMemoryFlag.resolve`` ignores it, so a disabled flag is indistinguishable + from an enabled one here. Sweeping them with everything else is therefore + the accurate statement of what this flag set does -- and the assertion that + obliges both in-memory suites to withhold the capability. + """ + canonical = json.loads(canonical_flags_json())["flags"] + provider = ControllableInMemoryProvider(canonical_flag_set()) + ignores_state = {disabled for disabled, _ in _DISABLED_FLAGS} + + # Annotated explicitly, as in the evaluation step: the five typed resolvers + # have different signatures, so an unannotated mapping infers a value type + # mypy will not let us call. + resolvers: dict[FlagType, Callable[[str, typing.Any], typing.Any]] = { + FlagType.BOOLEAN: provider.resolve_boolean_details, + FlagType.STRING: provider.resolve_string_details, + FlagType.INTEGER: provider.resolve_integer_details, + FlagType.FLOAT: provider.resolve_float_details, + FlagType.OBJECT: provider.resolve_object_details, + } + + assert canonical, "the packaged flag file defines no flags" + for key, definition in canonical.items(): + variant = definition["defaultVariant"] + expected = definition["variants"][variant] + note = _IGNORES_STATE if key in ignores_state else "" + + details = resolvers[_flag_type_of(expected)](key, _NOT_SEEDED) + + assert details.error_code is None, f"{key}: {details.error_message}{note}" + assert details.variant == variant, f"{key}{note}" + assert details.reason == Reason.STATIC, f"{key}{note}" + assert values_equal(expected, details.value), ( + f"{key}/{variant}: packaged {describe(expected)}, " + f"resolved {describe(details.value)}{note}" + ) + + +# The variants whose Python *type* the scenarios depend on, and what that type +# and value have to be. Not a second copy of the flag set: every key, variant +# and value in it is already checked against the file by the test above, and +# these rows say the one thing a comparison with the file cannot -- that a +# decoder has not normalised a number on its way through. `10 == 10.0` and +# `0 == False` in Python, so the integral float and the falsy values compare +# equal to exactly the mistranslations they exist to catch. +_LOAD_BEARING: tuple[tuple[str, str, type, typing.Any], ...] = ( + ("integer-flag", "ten", int, 10), + ("float-flag", "half", float, 0.5), + ("large-integer-flag", "max-int32", int, 2147483647), + # 2^53 - 1. A Python int is arbitrary-precision, so being an int is being + # exact; arriving as a float would round it. + ("huge-integer-flag", "max-safe", int, 9007199254740991), + # The trailing .0 is the whole point: as an int, the lossless half of + # @numeric-coercion passes without coercing anything. This is the row that + # bit Java. + ("integral-float-flag", "ten", float, 10.0), + ("boolean-zero-flag", "zero", bool, False), + ("integer-zero-flag", "zero", int, 0), + ("string-zero-flag", "zero", str, ""), +) + + +@pytest.mark.parametrize(("key", "variant", "expected_type", "expected"), _LOAD_BEARING) +def test_the_decoded_flag_keeps_the_python_type_the_file_wrote( + key: str, variant: str, expected_type: type, expected: typing.Any +) -> None: + """A number keeps the type it was written with, stated independently of the file.""" + value = canonical_flag_set()[key].variants[variant] + + assert type(value) is expected_type, ( + f"{key}/{variant} decoded to {describe(value)}, expected an " + f"{expected_type.__name__}" + ) + assert value == expected, f"{key}/{variant} decoded to {describe(value)}" + + +def test_a_number_inside_an_object_keeps_its_type_too() -> None: + """A structured flag has to decode the same way on both sides of a comparison. + + ``object-flag``'s expected value reaches the assertion through + ``json.loads`` of the Gherkin table cell. The seeded value reaches it + through ``json.loads`` of the flag file, and nothing converts either, so a + member of the object is the same Python type in both. + """ + template = canonical_flag_set()["object-flag"].variants["template"] + + assert isinstance(template, dict) + assert type(template["imagesPerPage"]) is int, describe(template["imagesPerPage"]) + assert template["imagesPerPage"] == 100 + + +def test_no_decoded_flag_carries_targeting() -> None: + """Every untargeted scenario expects reason ``STATIC``. + + The TCK tests a provider's mapping of a response, not a backend's + evaluation logic, so a flag that evaluated its context would report + ``TARGETING_MATCH`` and fail scenarios that are about something else. + + Stated about the *decoded* flags rather than about the file, because since + spec revision ``26362f85`` the file is not free of targeting: it gives + ``targeting-key-flag`` a rule so that a real backend can show a context + arriving. ``_decode_canonical_flags`` reads only ``state``, ``variants`` and + ``defaultVariant``, so the member is inert here -- which is what this + asserts, and what obliges an in-memory adoption to leave ``@targeting`` + undeclared rather than fail its scenarios. + """ + decoded = canonical_flag_set() + assert "targeting-key-flag" in decoded, ( + "the packaged flag file no longer defines targeting-key-flag, so this " + "test no longer checks anything: the flag with the one targeting rule " + "is what makes the member's inertness observable" + ) + for key, flag in decoded.items(): + assert flag.context_evaluator is None, f"{key} has targeting" + + +def test_exactly_the_four_disabled_flags_are_decoded_disabled() -> None: + """``state`` reaches the flag set, and reaches only the four flags it should. + + Both halves are load-bearing. A decoder that dropped the member would leave + the canonical set with nothing disabled, and every scenario that assumes a + flag serves its own value would go on passing while ``@disabled-flags`` + became untestable; a state that leaked onto any other flag would break those + scenarios instead, which Appendix F calls out as the way to break the set + silently. + """ + decoded = canonical_flag_set() + disabled = { + key + for key, flag in decoded.items() + if flag.state is InMemoryFlag.State.DISABLED + } + + assert disabled == {key for key, _ in _DISABLED_FLAGS} + + +@pytest.mark.parametrize(("disabled", "enabled"), _DISABLED_FLAGS) +def test_a_disabled_flag_mirrors_its_enabled_counterpart( + disabled: str, enabled: str +) -> None: + """Differing only in state is what makes each row falsifiable. + + Every ``@disabled-flags`` row passes a caller default that is the flag's + *other* variant, so a provider ignoring the state returns the configured + value and fails on the value alone. Bring the two variants together -- give + ``disabled-string-flag`` a ``greeting`` of ``bye`` -- and the row passes + whether the state was honoured or not, which is the one way this outline can + be made vacuous without changing a single scenario. + + Stated as a mirror of the enabled counterpart rather than as four literal + values, because that is the property the file's own ``$comment`` and + Appendix F both claim, and it is the one a future edit would be reasoning + about. + """ + decoded = canonical_flag_set() + + assert decoded[disabled].variants == decoded[enabled].variants + assert decoded[disabled].default_variant == decoded[enabled].default_variant + assert decoded[enabled].state is InMemoryFlag.State.ENABLED + + variants = dict(decoded[disabled].variants) + served = variants.pop(decoded[disabled].default_variant) + assert len(variants) == 1, ( + f"{disabled} no longer offers exactly one other variant for a row's " + f"caller default to come from" + ) + [other] = variants.values() + # Compared the way the Then step compares, because that is the comparison a + # vacuous row would slip past. + assert not values_equal(served, other), ( + f"{disabled} would resolve {describe(served)} either way, so a provider " + f"that ignores DISABLED would pass its row" + ) + + +def test_the_hand_built_changing_flag_matches_the_file() -> None: + """``change_flag`` rebuilds ``changing-flag`` at its other variant. + + That one flag is therefore built by hand rather than decoded, and it names + both variants itself. The file has to define exactly those two, or flipping + between them either changes nothing or invents a variant the backend under + test does not have. + """ + from_file = canonical_flag_set()[CHANGING_FLAG_KEY] + hand_built = changing_flag(from_file.default_variant) + + assert hand_built.variants == from_file.variants + assert from_file.default_variant in hand_built.variants + + +# A document exercising every level a $comment can appear at, including the one +# level it must not be stripped from. +_COMMENTED_DOCUMENT = json.dumps( + { + "$comment": "prose about the document", + "flags": { + "structured-flag": { + "$comment": "prose about the flag", + "state": "ENABLED", + "variants": { + "$comment": "prose about the variants", + "on": {"$comment": "a member of the value, not prose"}, + }, + "defaultVariant": "on", + } + }, + } +) + + +def test_a_comment_is_prose_at_the_document_flag_and_variant_levels() -> None: + """``$comment`` is how the specification's assets carry prose. + + A loader that took one for a flag, or for a variant, would serve a flag + nothing asked for and offer a variant no scenario can resolve. + """ + flags = _decode_canonical_flags(_COMMENTED_DOCUMENT) + + assert set(flags) == {"structured-flag"} + assert set(flags["structured-flag"].variants) == {"on"} + + +def test_a_comment_inside_a_variant_value_is_part_of_the_value() -> None: + """The line the other languages drew deliberately. + + A variant's value is opaque data. An object flag with a ``$comment`` member + is a perfectly good object flag, and a loader that reached into the value to + strip it would serve an object no scenario expects -- silently, because the + rest of the object still matches. + """ + value = _decode_canonical_flags(_COMMENTED_DOCUMENT)["structured-flag"].variants[ + "on" + ] + + assert value == {"$comment": "a member of the value, not prose"} + + +@pytest.mark.parametrize( + ("document", "message"), + [ + ("[]", "not a JSON object"), + ('{"flags": {}}', "defines no flags"), + ('{"flags": {"a": []}}', "expected an object"), + ('{"flags": {"a": {"state": "ENABLED"}}}', "variants is not an object"), + ( + '{"flags": {"a": {"state": "ENABLED", "variants": {"on": 1}, ' + '"defaultVariant": "off"}}}', + "is not one of its variants", + ), + ( + '{"flags": {"a": {"state": "PARTLY", "variants": {"on": 1}, ' + '"defaultVariant": "on"}}}', + "is none of", + ), + ], +) +def test_a_flag_file_this_decoder_does_not_understand_is_refused( + document: str, message: str +) -> None: + """Unreachable for a pinned spec revision, and it says which flag if it happens. + + The assets are copied in from the submodule at build time, so a failure here + means the pinned assets and this decoder disagree about the file's shape -- + which moving the pin should have surfaced. Refusing beats seeding a flag set + that is quietly missing a flag. + """ + with pytest.raises(ValueError, match=message): + _decode_canonical_flags(document) + + +def test_update_flags_names_the_union_of_old_and_new_keys() -> None: + """Appendix A asks for the union, not just the new keys. + + A consumer caching evaluations needs to know everything that might have + changed, and a key that disappeared has changed as much as one that arrived. + """ + provider = ControllableInMemoryProvider(canonical_flag_set()) + + seen: list[list[str] | None] = [] + provider.attach(lambda _p, _e, details: seen.append(details.flags_changed)) + + provider.update_flags({}) + + assert seen, "no event was emitted" + assert seen[-1] is not None + assert set(seen[-1]) == set(canonical_flag_set()), ( + "the event did not name every flag that disappeared" + ) diff --git a/tools/openfeature-tck/tests/test_lifecycle_steps.py b/tools/openfeature-tck/tests/test_lifecycle_steps.py new file mode 100644 index 00000000..f556fedb --- /dev/null +++ b/tools/openfeature-tck/tests/test_lifecycle_steps.py @@ -0,0 +1,283 @@ +"""The steps that talk to the provider directly, pinned outside the Gherkin. + +The shutdown scenarios live in ``lifecycle.feature``, which is gated on +``@lifecycle``, and neither in-memory self-test declares that -- there is no +backend to reach, so the readiness scenario would pass without testing anything. +That leaves the shutdown, re-initialise and shutdown-bound steps with no +canonical scenario running them here, and a step that first runs in a +containerised adopter's suite fails there looking like a provider defect. + +So they are driven directly, with a provider that records what was called of it +and can be told to misbehave. What is pinned is the contract the feature file +relies on: that the calls reach the provider's *own* methods rather than the +SDK's, that a raise is recorded and surfaces through the one "no exception" +step rather than through a second mechanism, that the client still reaches the +instance after it was brought back, and that the scenario's teardown copes with +a provider that was shut down underneath it. +""" + +from __future__ import annotations + +import typing +from collections.abc import Iterator + +import pytest + +from openfeature import api +from openfeature.contrib.tools.tck import ( + Capability, + ControlApi, + TckConfig, + TckState, + canonical_flag_set, +) +from openfeature.contrib.tools.tck.steps.flag_steps import ( + a_flag_with_key_and_default, + no_exception_should_have_been_thrown, + the_flag_was_evaluated_with_details, + the_resolved_value_should_be, +) +from openfeature.contrib.tools.tck.steps.provider_steps import ( + a_stable_provider, + the_provider_is_initialized_again, + the_provider_is_shut_down, + the_provider_metadata_name_should_not_be_empty, + the_shutdown_should_have_completed_within, +) +from openfeature.evaluation_context import EvaluationContext +from openfeature.provider import Metadata +from openfeature.provider.in_memory_provider import InMemoryProvider + + +class RecordingProvider(InMemoryProvider): + """The SDK's in-memory provider, remembering its lifecycle calls. + + ``fail_shutdown`` and ``fail_initialize`` make the corresponding call raise, + which is how the recording half of the steps is checked; ``metadata_name`` + is what the metadata step is checked against. + """ + + def __init__(self) -> None: + super().__init__(canonical_flag_set()) + self.calls: list[str] = [] + self.fail_shutdown = False + self.fail_initialize = False + self.metadata_name: typing.Any = "recording" + + def initialize(self, evaluation_context: EvaluationContext) -> None: + self.calls.append("initialize") + if self.fail_initialize: + msg = "initialize refused" + raise RuntimeError(msg) + + def shutdown(self) -> None: + self.calls.append("shutdown") + if self.fail_shutdown: + msg = "already closed" + raise RuntimeError(msg) + + def get_metadata(self) -> Metadata: + return Metadata(name=self.metadata_name) + + +class _NoControl: + @property + def description(self) -> str: + return "nothing" + + @property + def control_api(self) -> ControlApi: + return "in-process" + + def prepare_scenario(self) -> None: ... + + def change_flag(self) -> None: ... + + +@pytest.fixture +def provider() -> RecordingProvider: + return RecordingProvider() + + +@pytest.fixture +def state(provider: RecordingProvider) -> Iterator[TckState]: + """A scenario's state, with the recording provider registered by the real step. + + Through ``a_stable_provider`` rather than by hand, so what is tested is the + hand-off the feature files rely on: the step that registers the provider is + the one that makes it available to the steps that call it directly. + """ + config = TckConfig( + name="lifecycle-steps", + control=_NoControl(), + new_provider=lambda: provider, + capabilities={Capability.EVENTS}, + ) + state = TckState(config=config) + a_stable_provider(state) + yield state + state.teardown() + api.shutdown() + api.clear_providers() + + +# -- the calls reach the provider itself ------------------------------------- + + +def test_shutdown_calls_the_providers_own_shutdown_each_time( + state: TckState, provider: RecordingProvider +) -> None: + """Twice asked, twice called -- which the SDK would never do on its own. + + The registry shuts a provider down once per registration. The double-close + scenario needs two calls on one instance, and gets them only because the + step bypasses the registry. + """ + before = list(provider.calls) + the_provider_is_shut_down(state) + the_provider_is_shut_down(state) + assert provider.calls[len(before) :] == ["shutdown", "shutdown"] + assert [record.operation for record in state.lifecycle] == ["shutdown", "shutdown"] + no_exception_should_have_been_thrown(state) + + +def test_initialize_again_reaches_the_same_instance_the_client_uses( + state: TckState, provider: RecordingProvider +) -> None: + """The scenario's whole point: after the round trip, the client serves flags + from the very object that was shut down and brought back. + """ + the_provider_is_shut_down(state) + the_provider_is_initialized_again(state) + assert provider.calls[-2:] == ["shutdown", "initialize"] + + a_flag_with_key_and_default(state, "Boolean", "boolean-flag", "false") + the_flag_was_evaluated_with_details(state) + the_resolved_value_should_be(state, "true") + no_exception_should_have_been_thrown(state) + assert state.client is not None + assert state.client.get_provider_status().value == "READY" + + +def test_initialize_again_passes_an_empty_context( + state: TckState, provider: RecordingProvider +) -> None: + seen: list[EvaluationContext] = [] + original = provider.initialize + + def spy(evaluation_context: EvaluationContext) -> None: + seen.append(evaluation_context) + original(evaluation_context) + + provider.initialize = spy # type: ignore[method-assign] + the_provider_is_initialized_again(state) + assert len(seen) == 1 + assert seen[0].attributes == {} + assert seen[0].targeting_key is None + + +# -- a raise is recorded, and surfaces through the one step ------------------ + + +def test_a_raising_shutdown_fails_the_no_exception_step( + state: TckState, provider: RecordingProvider +) -> None: + """Recorded, not propagated: the step returns and the assertion is elsewhere.""" + provider.fail_shutdown = True + the_provider_is_shut_down(state) + + assert state.lifecycle[-1].raised is not None + with pytest.raises(AssertionError, match="shutdown raised RuntimeError"): + no_exception_should_have_been_thrown(state) + + +def test_a_raising_initialize_fails_the_no_exception_step( + state: TckState, provider: RecordingProvider +) -> None: + provider.fail_initialize = True + the_provider_is_shut_down(state) + the_provider_is_initialized_again(state) + + with pytest.raises(AssertionError, match="initialize raised RuntimeError"): + no_exception_should_have_been_thrown(state) + + +def test_a_lifecycle_raise_is_reported_even_after_a_clean_evaluation( + state: TckState, provider: RecordingProvider +) -> None: + """One mechanism for both kinds of call. + + The re-initialise scenario ends with an evaluation and then the no-exception + step. A raise from the shutdown before it must not be hidden behind the + evaluation that went fine. + """ + provider.fail_shutdown = True + the_provider_is_shut_down(state) + provider.fail_initialize = False + the_provider_is_initialized_again(state) + a_flag_with_key_and_default(state, "Boolean", "boolean-flag", "false") + the_flag_was_evaluated_with_details(state) + + assert state.last is not None and state.last.raised is None + with pytest.raises(AssertionError, match="shutdown raised"): + no_exception_should_have_been_thrown(state) + + +def test_the_no_exception_step_needs_something_to_have_been_called( + state: TckState, +) -> None: + """Before anything was asked of the provider the step has nothing to assert, + and says so rather than passing on an empty record.""" + with pytest.raises(AssertionError, match="nothing has been asked of the provider"): + no_exception_should_have_been_thrown(state) + + +# -- the shutdown bound ------------------------------------------------------ + + +def test_the_shutdown_bound_reads_the_most_recent_shutdown(state: TckState) -> None: + the_provider_is_shut_down(state) + the_shutdown_should_have_completed_within(state, "10000") + + state.lifecycle[-1].duration = 11.0 + with pytest.raises(AssertionError, match="shutdown took 11000ms"): + the_shutdown_should_have_completed_within(state, "10000") + + +def test_the_shutdown_bound_needs_a_shutdown(state: TckState) -> None: + with pytest.raises(AssertionError, match="has not been shut down"): + the_shutdown_should_have_completed_within(state, "10000") + + +# -- metadata ---------------------------------------------------------------- + + +def test_the_metadata_step_accepts_a_name_and_refuses_an_empty_one( + state: TckState, provider: RecordingProvider +) -> None: + the_provider_metadata_name_should_not_be_empty(state) + + for empty in ("", " ", None): + provider.metadata_name = empty + with pytest.raises(AssertionError, match="expected a non-empty string"): + the_provider_metadata_name_should_not_be_empty(state) + + +# -- the scenario after this one --------------------------------------------- + + +def test_a_shut_down_provider_does_not_break_the_next_registration( + state: TckState, provider: RecordingProvider +) -> None: + """What the fixture teardown and the next "Given a stable provider" do. + + Both shut the provider down again through the SDK. Requirement 2.5.3 makes + the second call harmless, and the suite relies on that: a provider that was + shut down directly is still the registered one when the scenario ends. + """ + the_provider_is_shut_down(state) + + replacement = RecordingProvider() + api.set_provider(replacement, state.config.domain) + assert state.client is not None + assert state.client.get_boolean_details("boolean-flag", False).value is True diff --git a/tools/openfeature-tck/tests/test_spec_assets.py b/tools/openfeature-tck/tests/test_spec_assets.py new file mode 100644 index 00000000..a36ef918 --- /dev/null +++ b/tools/openfeature-tck/tests/test_spec_assets.py @@ -0,0 +1,206 @@ +"""Where the conformance assets came from, and that it is the revision claimed. + +The assets are not this repository's. They arrive through a submodule, the pin is +the only record of which revision a run asked its questions at, and the copies in +the package are gitignored -- so the one thing that can go wrong silently is the +copies being from a different revision than the pin names. + +It has gone wrong. A rebase moves the gitlink and not the submodule's working +tree, so a sync after a rebase copied the *previous* pin's Gherkin over the +capability the suite had just been given, and only a self-test comparing the enum +against the assets noticed. That guard fires for one symptom -- a declarable +capability no scenario carries. A pin that changes nothing but the content of a +scenario would pass every guard in this package and still run the wrong suite, +which is what happened in another language: an entire adoption suite ran against +stale assets and reported byte-identical numbers to the previous run. + +So the checkout is wired into the copy, the copy is a dependency of the test +task, and this is what holds both ends of that to their promise. +""" + +from __future__ import annotations + +import warnings +from collections.abc import Sequence +from pathlib import Path + +import hatch_build_sync +import pytest +from hatch_build_sync import ( + SPEC_ROOT, + UNPINNED_ENV, + checkout_pinned_spec, + pinned_revision, +) + +PIN = "1111111111111111111111111111111111111111" +OTHER = "2222222222222222222222222222222222222222" + + +class _FakeGit: + """A git that answers from a script and remembers what it was asked. + + Enough to pin the decision without a fixture repository: the three questions + the checkout asks are the gitlink in the index, ``HEAD`` in the submodule, + and the update itself, and what matters is which of them are asked and in + what order. + """ + + def __init__(self, *, pin: str | None, heads: Sequence[str | None]) -> None: + self.pin = pin + self.heads = list(heads) + self.calls: list[list[str]] = [] + + def __call__(self, args: Sequence[str], cwd: Path) -> str | None: + self.calls.append(list(args)) + if args[0] == "ls-files": + return f"160000 {self.pin} 0\tspec" if self.pin else None + if args[0] == "rev-parse": + return self.heads.pop(0) if self.heads else None + return "" + + @property + def updated(self) -> bool: + return any(call[0] == "submodule" for call in self.calls) + + +# -- against the real checkout ------------------------------------------------ + + +def test_the_assets_on_disk_are_the_revision_the_pin_names() -> None: + """The invariant itself, measured against this very checkout. + + ``poe test`` runs the sync first, so by the time this runs the checkout has + already been brought to the pin -- which makes this a check that it really + was, rather than a check of something the sync would have had to do anyway. + It is also the assertion that would have failed in the pass where the stale + Gherkin got through. + + Skipped, loudly, where the pin cannot be read at all. That is an unpacked + sdist, which has no repository and no pin, and a linked git worktree whose + ``.git`` file names a path this process's filesystem namespace cannot follow + -- git inside the submodule answers there and git in the superproject does + not. The guarantee is genuinely not in force in those environments, and a + skip that says so is the honest report; a pass would not be. + """ + pinned = pinned_revision() + if pinned is None: + pytest.skip( + "no submodule pin is readable from here, so which revision these " + "assets came from cannot be established -- see checkout_pinned_spec" + ) + + head = hatch_build_sync._run_git(["rev-parse", "HEAD"], SPEC_ROOT) + assert head == pinned, ( + f"the spec submodule is checked out at {head} and the pin names " + f"{pinned}, so the assets this run tested against are not the ones it " + f"claims. Run `poe sync-spec-assets`" + ) + + +def test_only_a_gitlink_counts_as_a_pin() -> None: + """A plain directory checked in under that name is not a revision. + + Reading the second field of whatever ``ls-files -s`` printed would turn a + blob's hash into a commit id and compare it against the submodule's HEAD + forever after, which fails in a way that explains nothing. + """ + assert pinned_revision(lambda args, cwd: "100644 abc123 0\tspec") is None + assert pinned_revision(lambda args, cwd: "") is None + assert pinned_revision(lambda args, cwd: None) is None + assert pinned_revision(lambda args, cwd: f"160000 {PIN} 0\tspec") == PIN + + +# -- the decision ------------------------------------------------------------- + + +def test_a_working_tree_behind_the_pin_is_checked_out() -> None: + """The case this exists for: a rebase moved the gitlink and nothing else.""" + git = _FakeGit(pin=PIN, heads=[OTHER, PIN]) + + assert checkout_pinned_spec(git) == PIN + + assert git.updated, "the stale working tree was left where it was" + assert git.calls[-1][0] == "rev-parse", ( + "the checkout was not confirmed after being asked for" + ) + + +def test_a_working_tree_already_at_the_pin_is_left_alone() -> None: + """No git writes on the ordinary path. + + Every ``poe test`` runs this, including in a checkout somebody is midway + through something in. Moving a submodule that is already where it should be + is a write nobody asked for. + """ + git = _FakeGit(pin=PIN, heads=[PIN]) + + assert checkout_pinned_spec(git) == PIN + + assert not git.updated + assert [call[0] for call in git.calls] == ["ls-files", "rev-parse"] + + +def test_a_checkout_that_does_not_reach_the_pin_stops_the_build() -> None: + """Refused rather than warned, because here the answer is known and wrong. + + Unlike the unreadable-pin case, nothing is in doubt: the pin says one + revision, the working tree is at another, and the update did not close the + gap -- the commit is probably not in the local object store. Copying now + would produce assets from a revision the build is about to claim it did not + use. + """ + git = _FakeGit(pin=PIN, heads=[OTHER, OTHER]) + + with pytest.raises(RuntimeError) as raised: + checkout_pinned_spec(git) + + message = str(raised.value) + assert PIN in message and OTHER in message + assert "nothing is copied" in message + assert "fetch" in message, "the likely cause is named" + assert UNPINNED_ENV in message, "so is the deliberate way round it" + + +def test_an_unreadable_pin_warns_and_lets_the_build_continue( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Two environments reach this and only one of them is a problem. + + An unpacked sdist has no repository, no pin and no submodule, and the assets + are already in the tree: nothing to say. A checkout whose superproject this + process cannot reach has all three and no way to check them, which is worth + a warning every time -- it is the only signal that the guarantee is off. + """ + git = _FakeGit(pin=None, heads=[]) + + monkeypatch.setattr(hatch_build_sync, "SPEC_ROOT", Path("/no/such/submodule")) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + assert checkout_pinned_spec(git) is None + assert not caught, f"an sdist build has nothing to warn about: {caught}" + + monkeypatch.setattr(hatch_build_sync, "SPEC_ROOT", SPEC_ROOT) + with pytest.warns(UserWarning, match="copied from the submodule working tree"): + assert checkout_pinned_spec(git) is None + assert not git.updated + + +def test_the_escape_hatch_says_which_revision_it_used( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Drafting a spec change locally is legitimate; doing it quietly is not. + + The revision is named because the point of the pin is that a report can say + what it ran against, and a run in this mode cannot. + """ + monkeypatch.setenv(UNPINNED_ENV, "1") + git = _FakeGit(pin=PIN, heads=[OTHER]) + + with pytest.warns(UserWarning, match=OTHER): + assert checkout_pinned_spec(git) is None + + assert not git.updated + assert [call[0] for call in git.calls] == ["rev-parse"], ( + "the pin is not consulted, which is the whole of what the flag does" + ) diff --git a/uv.lock b/uv.lock index 3168e251..ea025adb 100644 --- a/uv.lock +++ b/uv.lock @@ -19,6 +19,7 @@ members = [ "openfeature-provider-ofrep", "openfeature-provider-unleash", "openfeature-python-contrib", + "openfeature-tck", ] [[package]] @@ -1868,7 +1869,7 @@ dev = [ [[package]] name = "openfeature-provider-flagd" -version = "0.5.1" +version = "0.5.2" source = { editable = "providers/openfeature-provider-flagd" } dependencies = [ { name = "cachebox" }, @@ -2043,6 +2044,7 @@ dependencies = [ { name = "openfeature-provider-flipt" }, { name = "openfeature-provider-ofrep" }, { name = "openfeature-provider-unleash" }, + { name = "openfeature-tck" }, ] [package.dev-dependencies] @@ -2064,6 +2066,7 @@ requires-dist = [ { name = "openfeature-provider-flipt", editable = "providers/openfeature-provider-flipt" }, { name = "openfeature-provider-ofrep", editable = "providers/openfeature-provider-ofrep" }, { name = "openfeature-provider-unleash", editable = "providers/openfeature-provider-unleash" }, + { name = "openfeature-tck", editable = "tools/openfeature-tck" }, ] [package.metadata.requires-dev] @@ -2075,11 +2078,51 @@ dev = [ [[package]] name = "openfeature-sdk" -version = "0.8.4" +version = "0.10.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3b/08/f6698d0614b8703170117b786bd77b7b0a04f3ee00f19fbe9b360d2dee69/openfeature_sdk-0.8.4.tar.gz", hash = "sha256:66abf71f928ec8c0db1111072bb0ef2635dfbd09510f77f4b548e5d0ea0e6c1a", size = 29676, upload-time = "2025-12-09T07:31:13.137Z" } +sdist = { url = "https://files.pythonhosted.org/packages/24/30/cfc684b7d8314398d476ae8ed515c10db99c4d7f950989db464b4ded12ce/openfeature_sdk-0.10.0.tar.gz", hash = "sha256:938c2540bdea4da3b01ef507517ee636f223a35abaaca845c5587e594151b052", size = 33516, upload-time = "2026-06-01T19:45:35.136Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9c/80/f6532778188c573cc83790b11abccde717d4c1442514e722d6bb6140e55c/openfeature_sdk-0.8.4-py3-none-any.whl", hash = "sha256:805ba090669798fc343ca9fdcbc56ff0f4b57bf6757533f0854d2021192e620a", size = 35986, upload-time = "2025-12-09T07:31:12.092Z" }, + { url = "https://files.pythonhosted.org/packages/da/44/8a4f5225e930ff0d999fd43f5d743a4babeb6c7e76dddc00f0e118878ef3/openfeature_sdk-0.10.0-py3-none-any.whl", hash = "sha256:75497ea75d73f684eef509a25f79ad6386368862e050af80ab70a44ae49b33e4", size = 38941, upload-time = "2026-06-01T19:45:33.011Z" }, +] + +[[package]] +name = "openfeature-tck" +version = "0.1.0" +source = { editable = "tools/openfeature-tck" } +dependencies = [ + { name = "openfeature-sdk" }, + { name = "pytest" }, + { name = "pytest-bdd" }, +] + +[package.optional-dependencies] +compose = [ + { name = "testcontainers" }, +] + +[package.dev-dependencies] +dev = [ + { name = "coverage", extra = ["toml"] }, + { name = "mypy" }, + { name = "poethepoet" }, + { name = "testcontainers" }, +] + +[package.metadata] +requires-dist = [ + { name = "openfeature-sdk", specifier = ">=0.10.0" }, + { name = "pytest", specifier = ">=8.4.0" }, + { name = "pytest-bdd", specifier = ">=8.1.0,<9.0.0" }, + { name = "testcontainers", marker = "extra == 'compose'", specifier = ">=4.12.0,<5.0.0" }, +] +provides-extras = ["compose"] + +[package.metadata.requires-dev] +dev = [ + { name = "coverage", extras = ["toml"], specifier = ">=7.10.0,<8.0.0" }, + { name = "mypy", specifier = ">=1.18.0,<2.0.0" }, + { name = "poethepoet", specifier = ">=0.37.0" }, + { name = "testcontainers", specifier = ">=4.12.0,<5.0.0" }, ] [[package]]