feat(provider-tck): let an adopter add their own scenarios to the suite

tools/openfeature-provider-tck/.gitignore

@@ -5,3 +5,6 @@
 src/openfeature/contrib/tools/provider_tck/features/
 src/openfeature/contrib/tools/provider_tck/flag_data/
 src/openfeature/contrib/tools/provider_tck/control-api.yaml
+# Generated alongside them, from the submodule pin, so a conformance report can
+# name the spec revision it ran against.
+src/openfeature/contrib/tools/provider_tck/spec_revision.json

tools/openfeature-provider-tck/hatch_build.py

@@ -18,15 +18,28 @@
 # the single definition of what gets copied where -- would not be importable.
 sys.path.insert(0, str(Path(__file__).parent))
 
-from hatch_build_sync import FILES, PACKAGE_REL, SPEC_ASSETS, TREES, sync
+from hatch_build_sync import (
+    FILES,
+    PACKAGE_REL,
+    REVISION_FILE,
+    SPEC_ASSETS,
+    TREES,
+    sync,
+)
 
 
 class SpecAssetsCopyHook(BuildHookInterface):
     PLUGIN_NAME = "spec-assets-copy"
 
     def initialize(self, version: str, build_data: dict) -> None:
         root = Path(self.root)
-        copies = [root / PACKAGE_REL / dest for _, dest in TREES + FILES]
+        # The generated revision file travels with the assets it describes. It
+        # has to be built here rather than read at run time, because the
+        # submodule that knows the answer is not in the wheel and a conformance
+        # report has to name the revision it ran against.
+        copies = [root / PACKAGE_REL / dest for _, dest in TREES + FILES] + [
+            root / PACKAGE_REL / REVISION_FILE
+        ]
 
         # Building from a checkout: refresh from the submodule, so what ships is
         # always the revision the pin names. Building from an sdist: there is no

tools/openfeature-provider-tck/hatch_build_sync.py

@@ -10,11 +10,16 @@
 needs no submodule: the copies are inside the distribution.
 """
 
+import json
 import shutil
+import subprocess
+import warnings
 from pathlib import Path
 
 ROOT = Path(__file__).parent
-SPEC_ASSETS = (ROOT / "spec/specification/assets/provider-tck").resolve()
+SPEC_ROOT = (ROOT / "spec").resolve()
+ASSETS_PATH_IN_SPEC = "specification/assets/provider-tck"
+SPEC_ASSETS = (SPEC_ROOT / ASSETS_PATH_IN_SPEC).resolve()
 PACKAGE_REL = Path("src/openfeature/contrib/tools/provider_tck")
 DEST_BASE = ROOT / PACKAGE_REL
 
@@ -28,6 +33,27 @@
 TREES = [("gherkin", "features"), ("flags", "flag_data")]
 FILES = [("openapi/control-api.yaml", "control-api.yaml")]
 
+REVISION_FILE = "spec_revision.json"
+"""Which revision of the specification the copied assets came from.
+
+Recorded at build time because the answer is only available at build time: the
+submodule that holds it is not in the wheel, and a conformance report that cannot
+name the revision it ran against cannot be compared with another. It is generated
+by the same command that copies the assets, which is what keeps the two from
+disagreeing.
+
+Not committed, for the same reason the assets are not: the submodule pin is the
+single record of which revision this package targets.
+"""
+
+UNKNOWN_REVISION = "unknown"
+"""Seven characters, the minimum the report schema accepts.
+
+A build that cannot reach git says it does not know rather than inventing a
+commit, and still produces a document that validates. Which happens for real:
+building from a source tarball has no ``.git`` to ask.
+"""
+
 
 def sync() -> None:
     if not SPEC_ASSETS.exists():
@@ -51,6 +77,48 @@ def sync() -> None:
             dest.unlink()
         shutil.copy2(SPEC_ASSETS / src_name, dest)
 
+    write_revision()
+
+
+def write_revision() -> None:
+    """Record the spec commit these copies came from.
+
+    The asset tree hash that used to accompany it is gone. It was carried so a
+    consumer could tell whether two runs executed the same questions; the
+    conformance report's results are now a Cucumber Messages stream, which
+    carries the executed feature source itself and answers that directly rather
+    than by proxy.
+    """
+    commit = _git("rev-parse", "HEAD") or UNKNOWN_REVISION
+    (DEST_BASE / REVISION_FILE).write_text(
+        json.dumps({"specRevision": commit}, indent=2) + "\n",
+        encoding="utf-8",
+    )
+
+
+def _git(*args: str) -> str:
+    """Run git inside the submodule, returning its output or an empty string.
+
+    A build must not hard-fail because git is absent or the checkout is not a
+    repository -- both are ordinary when building from an unpacked sdist. The
+    failure is reported as a warning and the identity degrades to ``unknown``,
+    which is legible in the resulting report rather than silently wrong.
+    """
+    command = ["git", "-C", str(SPEC_ROOT), *args]
+    try:
+        completed = subprocess.run(  # noqa: S603
+            command, capture_output=True, check=True, text=True
+        )
+    except (OSError, subprocess.CalledProcessError) as error:
+        warnings.warn(
+            f"could not determine the spec revision ({' '.join(command)}: {error}); "
+            f"conformance reports from this build will not name the revision they "
+            f"ran against",
+            stacklevel=2,
+        )
+        return ""
+    return completed.stdout.strip()
+
 
 if __name__ == "__main__":
     sync()

tools/openfeature-provider-tck/pyproject.toml

@@ -22,6 +22,21 @@ dependencies = [
     # Same runner the flagd provider and the flagd testkit already use, so an
     # adopting module gains no new test framework.
     "pytest-bdd>=8.1.0,<9.0.0",
+    # The conformance report's results are a Cucumber Messages stream rather
+    # than a format this package defines. These two are the reference
+    # implementations of the halves of that protocol: cucumber-messages is the
+    # official Python types, published from the same repository as the protocol
+    # itself, and gherkin-official is the parser that produces the
+    # gherkinDocument and pickle messages. pytest-bdd already depends on
+    # gherkin-official, so only the first is genuinely new -- and it has no
+    # dependencies of its own.
+    #
+    # pytest-bdd ships no Messages emitter (its cucumber_json.py is the legacy
+    # JSON format), so the stream is assembled here; assembling it from typed
+    # messages rather than hand-written dicts is what keeps it from drifting
+    # away from the protocol.
+    "cucumber-messages>=34.0.0,<35.0.0",
+    "gherkin-official>=29.0.0",
 ]
 requires-python = ">=3.10"
 
@@ -58,6 +73,10 @@ artifacts = [
     "src/openfeature/contrib/tools/provider_tck/features/",
     "src/openfeature/contrib/tools/provider_tck/flag_data/",
     "src/openfeature/contrib/tools/provider_tck/control-api.yaml",
+    # Which spec revision those assets came from, generated beside them. The
+    # submodule is not in the wheel, so a conformance report emitted by an
+    # installed copy has no other way to name the revision it ran against.
+    "src/openfeature/contrib/tools/provider_tck/spec_revision.json",
 ]
 
 [tool.hatch.build.hooks.custom]
@@ -74,6 +93,21 @@ fixed_format_cache = true
 pretty = true
 strict = true
 disallow_any_generics = false
+# cucumber-messages and gherkin-official ship no py.typed. Both are annotated
+# internally, so following them gives real types for the messages this package
+# builds rather than the Any a plain `ignore_missing_imports` would hand back --
+# which is the point of using the typed library at all.
+follow_untyped_imports = true
+
+[[tool.mypy.overrides]]
+# gherkin-official has no annotations at all, so following it turns every call
+# into a `no-untyped-call` error rather than into a type. pytest-bdd silences
+# the same import the same way. cucumber-messages is the opposite case -- fully
+# annotated, only missing py.typed -- and is followed, which is where the value
+# of using it rather than hand-written dicts actually lands.
+module = ["gherkin.*"]
+follow_untyped_imports = false
+ignore_missing_imports = true
 
 [tool.coverage.run]
 omit = ["tests/**"]

tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py

@@ -16,7 +16,7 @@
         Capability,
         InProcessControl,
         TckConfig,
-        features_path,
+        feature_paths,
     )
 
     @pytest.fixture(scope="session")
@@ -29,11 +29,14 @@ def tck_config():
             capabilities={Capability.EVENTS, Capability.OBJECT},
         )
 
-    scenarios(features_path())
+    scenarios(*feature_paths())
 
 ``scenarios()`` is pytest-bdd's own, called directly rather than wrapped: it
 injects the generated tests into the *calling module* by walking the stack, so a
 convenience wrapper around it would deposit them inside this package instead.
+:func:`~.extensions.feature_paths` is the canonical assets plus a
+``tck-extensions`` directory beside the calling module, if there is one -- see
+:mod:`~.extensions`.
 
 The step definitions arrive through this package's pytest plugin, so there is
 nothing to import for them and no ``conftest.py`` to write. Everything else --
@@ -49,33 +52,51 @@ def tck_config():
 
 import importlib.resources
 
-from .capability import ALL_CAPABILITIES, Capability
-from .config import TckConfig
+from .canonical import PARTIAL_ENV
+from .capability import DECLARABLE_CAPABILITIES, RESERVED_CAPABILITIES, Capability
+from .config import KnownDeviation, TckConfig
 from .control import (
     BackendControl,
     ConnectionControl,
     UnsupportedControlError,
 )
+from .extensions import (
+    EXTENSIONS_DIRECTORY,
+    feature_paths,
+    features_path,
+)
 from .inprocess import InProcessControl
+from .messages import MESSAGES_FORMAT
 from .provider import (
     CHANGING_FLAG_KEY,
     ControllableInMemoryProvider,
     canonical_flag_set,
 )
+from .report import REPORT_DIR_ENV, SCHEMA_VERSION
+from .state import TckState
 
 __all__ = [
-    "ALL_CAPABILITIES",
     "CHANGING_FLAG_KEY",
+    "DECLARABLE_CAPABILITIES",
+    "EXTENSIONS_DIRECTORY",
+    "MESSAGES_FORMAT",
+    "PARTIAL_ENV",
+    "REPORT_DIR_ENV",
+    "RESERVED_CAPABILITIES",
+    "SCHEMA_VERSION",
     "BackendControl",
     "Capability",
     "ConnectionControl",
     "ControllableInMemoryProvider",
     "InProcessControl",
+    "KnownDeviation",
     "TckConfig",
+    "TckState",
     "UnsupportedControlError",
     "canonical_flag_set",
     "canonical_flags_json",
     "control_api_spec",
+    "feature_paths",
     "features_path",
 ]
 
@@ -101,21 +122,6 @@ def tck_config():
 _PACKAGE = "openfeature.contrib.tools.provider_tck"
 
 
-def features_path() -> str:
-    """Return the directory holding the canonical feature files.
-
-    Packaged with this distribution, so a consumer needs no submodule and no
-    particular directory layout. Hand it to pytest-bdd's ``scenarios()``, which
-    accepts an absolute path::
-
-        scenarios(features_path())
-
-    pytest-bdd generates one test per scenario -- and one per row of a Scenario
-    Outline -- so failures name a scenario and ``-k`` selects one as usual.
-    """
-    return str(importlib.resources.files(_PACKAGE) / "features")
-
-
 def canonical_flags_json() -> str:
     """Return the canonical flag set as raw JSON, in the flagd flag-definition format.