test(flagd): run the provider conformance suite against both resolvers

providers/openfeature-provider-flagd/README.md

@@ -182,6 +182,46 @@ api.set_provider(FlagdProvider(
 ))
 ```
 
+## Provider conformance suite
+
+This provider runs the [OpenFeature Provider Conformance Suite][tck] against a flagd-testbed stack,
+once per resolver, in `tests/tck`. The suite owns the container stack: `tests/tck/conftest.py`
+declares a Compose file and the two ports the resolvers connect to, and nothing else.
+
+`tests/tck/docker-compose.yaml` is one definition of the backend for the whole repository, and the
+OFREP adoption carries a byte-identical copy — each provider package publishes its own distribution
+and must not read the other's files, so the two are kept in step by `diff` rather than by sharing a
+path. Change one, copy it to the other.
+
+**It is excluded from the default build, and a maintainer runs it by hand before merging a change to
+it.**
+
+```
+poe test-tck        # both resolvers, needs Docker
+poe test            # everything else, which is what CI runs
+```
+
+The exclusion lives in `pyproject.toml`: `--ignore=tests/tck` on the two tasks `build.yml` reaches,
+with the reason in a comment above them. Why a conformance suite is not a required gate is
+[Appendix F, "Running the suite in CI"][appendix-f], and is not restated here.
+
+Two things that are this provider's rather than the policy's:
+
+- **Docker is not what decides it.** `tests/e2e` needs Docker too and does run in the default build.
+  What decides it is the run: **8 failed, 119 passed, 3 skipped**, being three canonical flags that
+  no released flagd-testbed seeds yet, failing on each resolver, plus two genuine provider-side
+  gaps — one per resolver, and only one of them carries a known-deviation entry.
+  `tests/tck/conftest.py` accounts for all eight individually, with the flag or requirement each one
+  turns on, so a reviewer running the suite can tell a new failure from a known one and the number
+  above is what to expect.
+- **The default build still collects the suite** — `poe test` and `poe test-cov` end in
+  `pytest tests/tck --collect-only`, which imports every module and starts no container. An excluded
+  suite that has quietly stopped importing against the harness is worse than one that runs and
+  fails, and `mypy` here is configured over `src` alone, so nothing else would notice.
+
+[tck]: ../../tools/openfeature-tck/README.md
+[appendix-f]: https://github.com/open-feature/spec/blob/main/specification/appendix-f-provider-conformance.md
+
 ## License
 
 Apache 2.0 - See [LICENSE](./LICENSE) for more information.

providers/openfeature-provider-flagd/pyproject.toml

@@ -35,6 +35,12 @@ dev = [
   "coverage[toml]>=7.10.0,<8.0.0",
   "grpcio-health-checking>=1.82.1,<2.0.0",
   "mypy>=1.18.0,<2.0.0",
+  # The OpenFeature conformance suite. Ships the feature files, the flag set,
+  # the control-API client and the Compose harness that owns the container
+  # stack, and registers its step definitions through a pytest11 entry point --
+  # so tests/tck declares a Compose file and nothing else. The `compose` extra
+  # is what pulls testcontainers in for the harness.
+  "openfeature-tck[compose]",
   "poethepoet>=0.37.0",
   "pytest>=9.0.0,<10.0.0",
   "pytest-bdd>=8.1.0,<9.0.0",
@@ -112,13 +118,58 @@ warn_unused_ignores = false
 
 [tool.uv.sources]
 openfeature-flagd-core = { workspace = true }
+openfeature-tck = { workspace = true }
 
 [tool.pytest]
 strict = true
 
 [tool.poe.tasks]
-test = "pytest tests"
-test-cov = "coverage run -m pytest tests"
+# `tests/tck` is excluded from the default build on purpose, and a maintainer
+# runs `poe test-tck` by hand before merging a change to it. Why a conformance
+# suite is not a required gate is Appendix F, "Running the suite in CI" --
+# linked from tools/openfeature-tck/README.md -- and is deliberately not
+# restated here.
+#
+# What is local to this package: a full run is 8 failed, 119 passed, 3 skipped.
+# Six of those failures are canonical flags flagd-testbed does not seed yet, on
+# each of the two resolvers; the other two are real provider-side gaps, one per
+# resolver. tests/tck/conftest.py accounts for all eight and for the three
+# skips. Docker is not what decides the exclusion -- `tests/e2e` needs Docker
+# too and does run here.
+#
+# The suite is still *collected* on every default build, so it cannot quietly
+# stop importing against the harness while nobody runs it. `--ignore` would
+# otherwise leave nothing checking that, since mypy here is configured over
+# `src` alone.
+#
+# `ignore_fail = "return_non_zero"` is what makes that true rather than
+# aspirational: poe aborts a sequence at its first failing subtask, and this
+# package's default suite is red on this branch -- the TCK requires
+# openfeature-sdk >= 0.10 and this provider is not updated for it yet -- so the
+# collect step sat behind a step that never succeeds and never ran. Every
+# subtask now runs and a non-zero exit still propagates.
+test = { sequence = ["test-default", "test-tck-collect"], ignore_fail = "return_non_zero" }
+test-cov = { sequence = ["test-cov-default", "test-tck-collect"], ignore_fail = "return_non_zero" }
+test-default = "pytest tests --ignore=tests/tck"
+test-cov-default = "coverage run -m pytest tests --ignore=tests/tck"
+# The sync is part of the run rather than something a maintainer remembers,
+# because Appendix F requires the revision check to be in force **where the
+# scenarios execute** and not only in the TCK implementation's own tests. This
+# is a uv workspace, so `openfeature-tck` is installed editable and the
+# canonical assets this suite collects are the gitignored copies under
+# `tools/openfeature-tck/src`: a rebase that moves the submodule pin leaves them
+# from the previous revision, and a suite run from here would neither notice nor
+# say so. That is exactly how an adoption in another language published a full
+# pass against stale scenarios. `hatch_build_sync.py` checks the submodule out
+# at the pin, refuses to copy if it cannot, and needs nothing but the standard
+# library.
+#
+# Not on `test-tck-collect`, deliberately: that runs on every default build, it
+# executes no scenario, and it publishes no result, so the rule does not reach
+# it and the default build stays free of a submodule operation.
+sync-spec-assets = "python ../../tools/openfeature-tck/hatch_build_sync.py"
+test-tck = ["sync-spec-assets", {cmd = "pytest tests/tck"}]
+test-tck-collect = "pytest tests/tck --collect-only -q"
 cov-report = "coverage xml"
 cov = [
     "test-cov",

providers/openfeature-provider-flagd/tests/tck/conftest.py

@@ -0,0 +1,106 @@
+"""One testbed stack, declared rather than wired, and shared by both suites.
+
+The container lifecycle belongs to the TCK -- see its README for what
+``tck_backend`` does with the declaration below, and Appendix F, "The control
+API", for why the stack is started once and never restarted. What is left here
+is the declaration and one free port for the scenarios that need a backend that
+is not there.
+
+One stack and one ``HttpControl`` serve both suites, because one flagd process
+serves both resolver ports: 8013 for RPC and 8015 for sync. The launchpad
+registers no ``/reset`` and no ``/healthz``, so every ``prepare_scenario`` takes
+the harness's documented ``/start`` fallback and one 404 is logged per session.
+
+**What a full run reports, and what each failure is.** ``8 failed, 119 passed,
+3 skipped`` over the two resolvers. Read it here rather than counting: three
+distinct causes account for all eight, and only two of them are the provider's.
+
+*Six failures are the backend's flag set.* no released flagd-testbed seeds
+``large-integer-flag`` nor ``integral-float-flag``, so the untagged precision
+scenario, the ``max-int32`` row of the ``@variants`` outline and the lossless
+``@numeric-coercion`` scenario fail ``FLAG_NOT_FOUND`` on each resolver alike.
+open-feature/flagd-testbed#392 seeds all three of the flags the canonical set is
+missing and says what each catches; bump the tag in ``docker-compose.yaml``
+beside this file when it lands. Left as failures rather than xfailed, because
+they are true about the stack under test, and carrying no ``KnownDeviation`` in
+either suite: the provider was never given the flag to get wrong.
+
+*One failure is ``openfeature-flagd-core``'s*, on in-process alone:
+``boolean-flag`` requested as a Float resolves to ``1.0`` with reason ``STATIC``
+and no error code, where the mandatory wrong-type scenario asks for the caller's
+default. ``bool`` is a subclass of ``int`` in Python and the int-to-float
+widening does not exclude it. Filed as open-feature/python-sdk-contrib#417. RPC
+passes the row, because the server type-checks it. No ``KnownDeviation``: the
+scenario is mandatory and ungated, so it fails visibly on every run and an entry
+would add nothing a reader cannot see.
+
+*One failure is flagd's*, on RPC alone, and it is the one failure here carrying a
+``KnownDeviation``: ``float-flag`` (0.5) requested as an Integer comes back as
+``0`` with no error code. **The in-process resolver passes that scenario**, which
+is why the deviation is recorded against RPC only; ``test_rpc.py`` and
+``test_in_process.py`` carry the measurement on each side.
+
+*The three skips are two scenarios.* ``@large-integers`` gates one and is
+withheld on both resolvers, so it skips twice; ``@reinitialization`` gates one,
+which in-process declares and passes and RPC withholds, so it skips once. Each
+suite gives its own reason beside its declaration.
+
+**One finding came out of a scenario that passes**, so neither the results nor
+the report has anywhere to put it: the in-process *Shutting down a provider that
+cannot reach its backend completes promptly* returns well inside its bound and
+leaves a ``PytestUnhandledThreadExceptionWarning`` behind it -- gRPC's
+connectivity polling thread raising ``ValueError: Cannot invoke RPC: Channel
+closed!`` after ``shutdown`` closed the channel underneath it. Reproducible on
+every run, and filed as open-feature/python-sdk-contrib#419. Not a deviation:
+nothing required is unmet. It is noted here because a reader who sees the warning
+should know it is a recorded finding rather than noise.
+
+The canonical set's ``targeting-key-flag``, four ``disabled-*`` flags and three
+falsy flags need no testbed change: they are flagd-testbed's own flags, which is
+why the canonical set adopted their names and variants, and the launchpad's
+``default`` configuration serves them as they stand.
+"""
+
+from __future__ import annotations
+
+import socket
+from pathlib import Path
+
+import pytest
+
+from openfeature.contrib.tools.tck import ComposeBackend, RunningBackend
+from tests.tck.suite import IN_PROCESS_PORT, RPC_PORT
+
+
+@pytest.fixture(scope="session")
+def compose_backend() -> ComposeBackend:
+    """The stack under test, as the TCK's ``tck_backend`` fixture wants it.
+
+    Both resolver ports are declared even though each suite uses one of them,
+    because both suites share this stack. The Compose file publishes flagd's
+    OFREP port as well, which nothing here asks for: it is the same file the
+    OFREP adoption uses, and a port nobody declares is neither waited on nor
+    looked up.
+
+    The path is absolute so that pytest run from the repository root works too;
+    a relative one resolves against the working directory.
+    """
+    return ComposeBackend(
+        compose_file=Path(__file__).parent / "docker-compose.yaml",
+        backend_ports=[RPC_PORT, IN_PROCESS_PORT],
+    )
+
+
+@pytest.fixture(scope="session")
+def closed_port(tck_backend: RunningBackend) -> int:
+    """A port on localhost with nothing listening, for the ``@unavailable`` scenarios.
+
+    Discovered by binding and releasing rather than hard-coded, because the
+    stack's own host ports are mapped dynamically and a fixed number could
+    collide with one; depending on ``tck_backend`` orders this after the stack
+    has taken its ports. Deliberately not a port on the stack, which has to stay
+    up -- simulated outages belong to the control API.
+    """
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
+        probe.bind(("127.0.0.1", 0))
+        return int(probe.getsockname()[1])

providers/openfeature-provider-flagd/tests/tck/docker-compose.yaml

@@ -0,0 +1,20 @@
+# The backend every conformance suite in this repository runs against: the unmodified
+# flagd testbed image, serving flagd and its launchpad control API.
+#
+# Not flagd-testbed's own compose file, which adds an envoy sidecar for its
+# forbidden-endpoint scenarios, names its service `flagd`, and bind-mounts a flags
+# directory the launchpad writes into. None of that is needed to drive the launchpad.
+#
+# This file exists twice, once per provider package, and the two are byte-identical:
+#   providers/openfeature-provider-flagd/tests/tck/docker-compose.yaml
+#   providers/openfeature-provider-ofrep/tests/tck/docker-compose.yaml
+# Each package publishes its own distribution and must not read the other's files, so
+# `diff` the two paths after changing either -- that diff is what catches drift.
+services:
+  backend:
+    image: ghcr.io/open-feature/flagd-testbed:v3.10.1
+    ports:
+      - 8013   # flagd RPC evaluation (gRPC)
+      - 8015   # flagd in-process sync (gRPC)
+      - 8016   # flagd's OFREP HTTP API
+      - 8080   # launchpad control API