Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ bazel_dep(name = "rules_python", version = "2.3.2")
bazel_dep(name = "rules_proto", version = "7.1.0")
bazel_dep(name = "googletest", version = "1.18.0.bcr.1")
bazel_dep(name = "abseil-cpp", version = "20250814.2")
bazel_dep(name = "platforms", version = "0.0.10")
bazel_dep(name = "openfeature_cpp_sdk")
git_override(
module_name = "openfeature_cpp_sdk",
Expand Down Expand Up @@ -42,3 +43,26 @@ git_repository(
remote = "https://github.com/pboettch/json-schema-validator.git",
tag = "2.4.0",
)

git_repository(
name = "cwt_cucumber",
build_file = "//providers/flagd:cwt_cucumber.BUILD",
remote = "https://github.com/ThoSe1990/cwt-cucumber.git",
tag = "2.9",
Comment thread
m-olko marked this conversation as resolved.
)

git_repository(
name = "flagd_testbed",
build_file = "//providers/flagd:flagd_testbed.BUILD",
remote = "https://github.com/open-feature/flagd-testbed.git",
tag = "v3.8.0",
)

http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

http_archive(
name = "flagd_binary",
build_file_content = 'exports_files(["flagd_linux_x86_64"]) \nalias(name = "flagd", actual = "flagd_linux_x86_64", visibility = ["//visibility:public"])',
integrity = "sha256-mvJrkOJgbRKLphEoL4trq1x3K2lY8QULF+AZP91Vusk=",
urls = ["https://github.com/open-feature/flagd/releases/download/flagd%2Fv0.15.5/flagd_0.15.5_Linux_x86_64.tar.gz"],
Comment thread
coderabbitai[bot] marked this conversation as resolved.
)
38 changes: 38 additions & 0 deletions providers/flagd/cwt_cucumber.BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
load("@rules_cc//cc:defs.bzl", "cc_library")

# Generate the version file from the template by extracting the version from CMakeLists.txt
genrule(
name = "generate_version_file",
srcs = [
"CMakeLists.txt",
"src/version.template",
],
outs = ["src/version.hpp"],
cmd = """
VERSION=$$(grep 'project(cwt-cucumber VERSION' $(location CMakeLists.txt) | sed 's/.*VERSION \\([0-9.]*\\).*/\\1/');
MAJOR=$${VERSION%%.*};
MINOR=$${VERSION#*.};
sed -e "s/@PROJECT_VERSION@/$$VERSION/g" \
-e "s/\\$${PROJECT_VERSION_MAJOR}/$$MAJOR/g" \
-e "s/\\$${PROJECT_VERSION_MINOR}/$$MINOR/g" \
-e "s/\\$${PROJECT_VERSION}/$$VERSION/g" \
$(location src/version.template) > $@
""",
)

cc_library(
name = "cwt-cucumber",
srcs = glob(
["src/**/*.cpp"],
exclude = ["src/main.cpp"],
),
hdrs = glob(
["src/**/*.hpp"],
exclude = ["src/version.hpp"],
) + [
"src/version.hpp",
],
copts = ["-std=c++20"],
strip_include_prefix = "src",
visibility = ["//visibility:public"],
)
13 changes: 13 additions & 0 deletions providers/flagd/flagd_testbed.BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
exports_files(glob(["gherkin/**/*.feature"]))

filegroup(
name = "features",
srcs = glob(["gherkin/**/*.feature"]),
visibility = ["//visibility:public"],
)

filegroup(
name = "flags",
srcs = glob(["flags/**/*.json"]),
visibility = ["//visibility:public"],
)
1 change: 1 addition & 0 deletions providers/flagd/tests/gherkin/.clang-tidy
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Checks: "-*"
2 changes: 2 additions & 0 deletions providers/flagd/tests/gherkin/.clangd
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
CompileFlags:
Add: [-std=c++20]
148 changes: 148 additions & 0 deletions providers/flagd/tests/gherkin/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test")

# cwt-cucumber requires C++20. The provider itself targets C++17, so this must
# stay scoped to these targets rather than moving into .bazelrc. copts are not
# propagated across targets in either direction, which is why @cwt_cucumber
# carries its own copy for its own sources.
GHERKIN_COPTS = ["-std=c++20"]

GHERKIN_PLATFORM = [
"@platforms//os:linux",
"@platforms//cpu:x86_64",
]

# Feature files the provider can currently satisfy.
#
# Deliberately excluded:
# connection.feature, events.feature - need event-handler steps, and the
# provider exposes no event API yet.
# contextEnrichment.feature - needs sync-metadata enrichment.
# sync-payload.feature - needs sync-metadata enrichment.
# rpc-caching.feature - @rpc only; there is no RPC resolver.
#
# Listing them would add roughly 170 permanently-undefined steps, which buries
# real regressions in noise. Add a file here as soon as its steps exist.
SUPPORTED_FEATURES = [
"@flagd_testbed//:gherkin/config.feature",
"@flagd_testbed//:gherkin/disabled.feature",
"@flagd_testbed//:gherkin/evaluation.feature",
"@flagd_testbed//:gherkin/metadata.feature",
"@flagd_testbed//:gherkin/selector.feature",
"@flagd_testbed//:gherkin/targeting.feature",
]

# flagd binary and fixture flags. Needed in the runfiles of anything that runs
# the suite, and needed as a direct prerequisite of any rule whose `env` uses
# $(rootpaths) on them.
GHERKIN_DATA = [
"@flagd_binary//:flagd",
"@flagd_testbed//:flags",
]

# Deliberately NOT built with GHERKIN_COPTS. absl::SourceLocation aliases to
# std::source_location only under C++20, which changes the mangled name of
# every absl error factory. Abseil itself is built at the repo's default
# standard, so a C++20 translation unit calling absl::NotFoundError fails to
# link. This layer pulls in no cucumber headers, so it stays on the default
# standard and can use absl::Status; the C++20 step definitions only ever
# consume the returned Status, never construct one.
cc_library(
name = "test_env",
testonly = True,
srcs = ["test_env.cpp"],
hdrs = ["test_env.h"],
data = GHERKIN_DATA,
tags = ["manual"],
deps = [
"@abseil-cpp//absl/status",
"@abseil-cpp//absl/status:statusor",
"@abseil-cpp//absl/strings",
"@bazel_tools//tools/cpp/runfiles",
"@com_github_grpc_grpc//:grpc++",
"@nlohmann_json//:json",
],
)

# Step definitions register from static initializers that nothing in main()
# references, so without alwayslink the linker drops every object file here and
# every step reports as UNDEFINED.
cc_library(
name = "gherkin_steps",
testonly = True,
srcs = [
"steps/config_steps.cpp",
"steps/context_steps.cpp",
"steps/evaluation_steps.cpp",
"steps/flag_steps.cpp",
"steps/lifecycle_steps.cpp",
"steps/provider_steps.cpp",
"steps/step_utils.cpp",
"test_context.cpp",
],
hdrs = [
"steps/step_utils.h",
"test_context.h",
],
copts = GHERKIN_COPTS,
data = GHERKIN_DATA,
# `bazel build //...` expands to every target in the package, so leaving
# the library untagged would still make CI fetch the external repos.
tags = ["manual"],
deps = [
":test_env",
"//providers/flagd/src:flagd_provider",
"@abseil-cpp//absl/status",
"@abseil-cpp//absl/strings",
"@bazel_tools//tools/cpp/runfiles",
"@com_github_grpc_grpc//:grpc++",
"@cwt_cucumber//:cwt-cucumber",
"@nlohmann_json//:json",
"@openfeature_cpp_sdk//openfeature",
"@openfeature_cpp_sdk//openfeature:openfeature_api",
],
alwayslink = True,
)

cc_test(
name = "gherkin_test",
size = "large",
srcs = ["test_runner.cpp"],
args = ["$(rootpath %s)" % feature for feature in SUPPORTED_FEATURES],
copts = GHERKIN_COPTS,
data = SUPPORTED_FEATURES + GHERKIN_DATA,
env = {
"FLAGD_TEST_FLAGS": "$(rootpaths @flagd_testbed//:flags)",
# The provider is in-process only; @rpc-tagged scenarios would
# otherwise run against the sync port and fail for the wrong reason.
"GHERKIN_TAGS": "@in-process",
},
# TODO(#135): drop once the suite is green, so CI can enforce it.
tags = ["manual"],
target_compatible_with = GHERKIN_PLATFORM,
deps = [
":gherkin_steps",
"@cwt_cucumber//:cwt-cucumber",
],
)

# Same binary without the Bazel test harness, for running a subset by hand:
# bazel run //providers/flagd/tests/gherkin:gherkin_bin -- \
# --tags "@in-process and @targeting" <path/to/feature>
#
# Manual for the same reason as above.
cc_binary(
name = "gherkin_bin",
testonly = True,
srcs = ["test_runner.cpp"],
copts = GHERKIN_COPTS,
data = SUPPORTED_FEATURES + GHERKIN_DATA,
env = {
"FLAGD_TEST_FLAGS": "$(rootpaths @flagd_testbed//:flags)",
},
tags = ["manual"],
target_compatible_with = GHERKIN_PLATFORM,
deps = [
":gherkin_steps",
"@cwt_cucumber//:cwt-cucumber",
],
)
114 changes: 114 additions & 0 deletions providers/flagd/tests/gherkin/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
# Gherkin integration tests

Runs the [flagd-testbed](https://github.com/open-feature/flagd-testbed) Gherkin
suite against the C++ flagd provider, using
[cwt-cucumber](https://github.com/ThoSe1990/cwt-cucumber) as the runner.

A real `flagd` binary is downloaded by Bazel and started as a subprocess for
the duration of the run; the tests talk to it over gRPC exactly as a real
application would.

## Running

The target is tagged `manual`, so `bazel test //providers/...` skips it. Run it
explicitly:

```sh
bazel test //providers/flagd/tests/gherkin:gherkin_test --test_output=all
```

To run a subset, use the binary directly:

```sh
# Everything tagged @targeting
bazel run //providers/flagd/tests/gherkin:gherkin_bin -- \
--tags "@in-process and @targeting" \
$PWD/bazel-cpp-sdk-contrib/external/+_repo_rules+flagd_testbed/gherkin/targeting.feature

# A single scenario by name
bazel run //providers/flagd/tests/gherkin:gherkin_bin -- \
--name "Returns metadata" \
$PWD/bazel-cpp-sdk-contrib/external/+_repo_rules+flagd_testbed/gherkin/metadata.feature
```

`--tags` and `--name` can also be supplied as `GHERKIN_TAGS` and
`GHERKIN_NAME`; command-line flags win over the environment.

> [!NOTE]
> Linux x86_64 only. The `flagd` release archive pinned in `MODULE.bazel` has
> no other platform, so `target_compatible_with` makes these targets *skip*
> silently elsewhere rather than fail.

## Layout

| File | Purpose |
|---|---|
| `test_runner.cpp` | `main()`; normalises arguments and calls `cuke::entry_point` |
| `test_env.{h,cpp}` | Starts/stops flagd, merges the fixture files, resolves runfiles. **Separate target, not built with C++20** — see below |
| `test_context.{h,cpp}` | All mutable test state, plus environment save/restore |
| `steps/flag_steps.cpp` | `a <Type>-flag with key ...` |
| `steps/context_steps.cpp` | `a context containing ...` |
| `steps/provider_steps.cpp` | `a stable flagd provider`, option collection |
| `steps/evaluation_steps.cpp` | `the flag was evaluated with details` and its assertions |
| `steps/config_steps.cpp` | `a config was initialized` and option assertions |
| `steps/lifecycle_steps.cpp` | `BEFORE_ALL` / `AFTER_ALL` / per-scenario reset |
| `steps/step_utils.{h,cpp}` | Conversions, parsing, assertion helpers |

## Three things that will confuse you

**Empty Scenario-Outline cells arrive as four quote characters.**
cwt-cucumber substitutes an empty Examples cell with the literal `""`, which
combines with the quotes already in the step text. `{string}` (`"([^"]*)"`)
cannot match that, so every step whose value may be blank is registered twice —
once normally and once with `GHERKIN_EMPTY_ARG`. Both registrations delegate to
one function. See the comment on `GHERKIN_EMPTY_ARG` in `steps/step_utils.h`.

**Step definitions need `alwayslink`.**
Steps register themselves from static initializers. In a plain `cc_library` the
linker discards every object file that `main()` does not reference, taking the
registrations with it — the binary links cleanly and reports *every* step as
undefined. The `gherkin_steps` target sets `alwayslink = True`.

**C++20 changes Abseil's ABI, so `test_env` is a separate target.**
cwt-cucumber requires C++20, but the provider and all its dependencies — including
Abseil — build at the repository default. `absl::SourceLocation` aliases to
`std::source_location` only under C++20, which changes the mangled name of every
Abseil error factory. A C++20 translation unit calling `absl::NotFoundError` therefore
fails to link:

```
undefined reference to absl::status_internal::MakeErrorImpl<5>(
string_view, std::source_location)
```

`test_env.{h,cpp}` pulls in no cucumber headers, so it lives in its own `cc_library`
*without* `GHERKIN_COPTS` and can use `absl::Status` normally. Do not add
`copts = GHERKIN_COPTS` to that target.

> [!WARNING]
> The step definitions may only **consume** an `absl::Status` — `.ok()`,
> `.message()`, `operator<<`. Constructing one from a C++20 translation unit will
> not link. `absl::Status` is a single `uintptr_t`, so passing it across the
> boundary is layout-safe, but any Abseil API whose *signature* depends on a C++20
> feature is not usable from `gherkin_steps`.
>
> If a step ever needs to build a `Status`, add a factory to `test_env` and call
> that instead — or move the whole repository to C++20.

## Known gaps

These are real provider gaps, not harness bugs. Scenarios covering them fail or
are excluded on purpose; see the `SUPPORTED_FEATURES` list in `BUILD`.

| Gap | Effect |
|---|---|
| No RPC resolver | `rpc-caching.feature` excluded; `GHERKIN_TAGS` pins the run to `@in-process` |
| No file/offline resolver (TODO #20) | `FlagdProvider` calls `LOG(FATAL)` when `offlineFlagSourcePath` is set, which would abort the whole run |
| No provider events | `connection.feature`, `events.feature` excluded |
| No sync-metadata enrichment | `contextEnrichment.feature`, `sync-payload.feature` excluded |
| No `resolver` / `cache` / `maxCacheSize` in `FlagdProviderConfig` | Those `config.feature` scenarios report as not-implemented |
| `edge-case-flags.json`, `custom-ops.json` rejected by FlagSync (TODO #129) | Fixtures skipped; dependent scenarios fail |

Steps deliberately fail rather than pass when they cannot verify something. A
step that recognises none of its inputs runs zero assertions, and reporting
that as success is how a suite ends up certifying unimplemented behaviour.
Loading