diff --git a/.agent/plugins/marketplace.json b/.agent/plugins/marketplace.json new file mode 100644 index 0000000..0b4793c --- /dev/null +++ b/.agent/plugins/marketplace.json @@ -0,0 +1,20 @@ +{ + "name": "confit", + "interface": { + "displayName": "ConfIT" + }, + "plugins": [ + { + "name": "confit", + "source": { + "source": "local", + "path": "./" + }, + "policy": { + "installation": "AVAILABLE", + "authentication": "ON_INSTALL" + }, + "category": "testing" + } + ] +} diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 0000000..fee68ff --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json.schemastore.org/claude-code-marketplace.json", + "name": "confit", + "description": "ConfIT \u2014 declarative API integration testing for .NET. Skills for setting up a test suite and authoring test definitions from an API contract.", + "owner": { + "name": "Rahul Garg", + "url": "https://github.com/techygarg" + }, + "plugins": [ + { + "name": "confit", + "description": "Set up a ConfIT suite, write component tests from your controller and mocks, and write API tests from a spec.", + "source": "./", + "category": "testing" + } + ] +} diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..b6fb267 --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,20 @@ +{ + "name": "confit", + "version": "1.0.0", + "description": "Agent skills for ConfIT \u2014 set up a declarative API test suite, write component tests from your controller and its mocks, and write black-box API tests from an OpenAPI spec.", + "author": { + "name": "Rahul Garg", + "url": "https://github.com/techygarg" + }, + "homepage": "https://github.com/techygarg/ConfIT", + "repository": "https://github.com/techygarg/ConfIT", + "license": "MIT", + "keywords": [ + "testing", + "api-testing", + "integration-testing", + "component-testing", + "dotnet", + "confit" + ] +} diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json new file mode 100644 index 0000000..53d174f --- /dev/null +++ b/.codex-plugin/plugin.json @@ -0,0 +1,54 @@ +{ + "name": "confit", + "version": "1.0.0", + "description": "Agent skills for ConfIT — set up a declarative API test suite, write component tests from your controller and its mocks, and write black-box API tests from an OpenAPI spec.", + "author": { + "name": "Rahul Garg" + }, + "homepage": "https://github.com/techygarg/ConfIT", + "repository": "https://github.com/techygarg/ConfIT", + "license": "MIT", + "keywords": [ + "testing", + "api-testing", + "integration-testing", + "component-testing", + "contract-testing", + "test-automation", + "openapi", + "swagger", + "graphql", + "wiremock", + "xunit", + "dotnet", + "confit" + ], + "skills": "./skills/", + "interface": { + "displayName": "ConfIT", + "shortDescription": "Set up a declarative API test suite, write component tests from your code, and API tests from your spec.", + "longDescription": "ConfIT is a .NET library for declarative API integration testing — tests are YAML or JSON files, not C# code. This plugin gives Codex three skills. Suite Setup wires ConfIT into a project: startup mode (in-process, command/AppLauncher, or integration), suite.config.yaml, the fixture and test class, auth profiles and tag filters. Component Tests is for a developer mid-implementation: it works from the controller plus the mocks behind it, tracing the call path to discover what the service calls, and driving error paths by changing what a mocked dependency returns. Integration Tests is for testing an already-deployed service black box: it works from an OpenAPI spec, a Postman collection or a live endpoint, assumes no access to the service source, and handles environment selection, auth profiles and the shared-persistent-state problem. All three read ConfIT's own CI-verified example suites rather than carrying templates that go stale.", + "developerName": "Rahul Garg", + "category": "Testing", + "capabilities": [ + "Interactive", + "Read", + "Write" + ], + "defaultPrompt": [ + "Set up a ConfIT component test suite for my API.", + "I just implemented this controller — write component tests for it.", + "Discover what outbound calls this endpoint makes and mock them.", + "My component test gets a 404 from the mock — fix the interaction.", + "Here's our OpenAPI spec — write API tests against staging.", + "Convert this Postman collection into ConfIT tests.", + "Point the suite at the QA environment with a bearer token.", + "My ConfIT suite passes once then fails on the second run." + ], + "websiteURL": "https://github.com/techygarg/ConfIT", + "privacyPolicyURL": "https://docs.github.com/en/site-policy/privacy-policies/github-general-privacy-statement", + "termsOfServiceURL": "https://docs.github.com/en/site-policy/github-terms/github-terms-of-service", + "brandColor": "#2F6FEB", + "screenshots": [] + } +} diff --git a/CHANGELOG.md b/CHANGELOG.md index 356ed8d..bbf4b62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,34 @@ ConfIT uses [Semantic Versioning](https://semver.org/). --- +## [3.2.0] + +### Added + +- **`mock.enableLogs`** — WireMock request logging is now switchable from `suite.config.yaml` instead of only from fixture code. Beyond debugging a stub that will not match, it turns an unknown dependency surface into a listing: run a component test with no `mock:` block and every outbound call is logged as an unmatched request, which is enough to write the interactions from. Works whatever language the service under test is written in, since it observes HTTP rather than code. See [Mock Interactions](doc/mock-interactions.md#discovering-what-a-service-calls). + +- **Agent skills** — three [Agent Skills](https://docs.claude.com/en/docs/claude-code/skills) in [`skills/`](skills), distributed as agent plugins: `confit-suite-setup` (wire a suite), `confit-component-tests` (developer, mid-implementation — works from the controller plus the mocks behind it) and `confit-integration-tests` (QA, post-deployment — black box, works from a spec, collection or live endpoint and assumes no access to the service source). See [AI Agent Skills](doc/ai-skills.md). + +- **Plugin manifests** — the repository is its own Claude Code marketplace (`/plugin marketplace add techygarg/ConfIT`, then `/plugin install confit@confit`), and carries a Codex manifest. Further agents are onboarded by adding one manifest directory each. + +- **`example/README.md`** — maps the three startup modes to their example projects, states what every suite structurally needs, and lists what is demo-specific so none of it is copied into a consuming project. + +- **Validation tooling in [`tools/`](tools)** — `check-testcases.py` statically validates test definitions (missing expected bodies, invalid `depends:`, unresolvable `{{variables}}`, `mock:` blocks in an integration suite, unregistered files, and matcher problems such as an unknown name, a missing closing parenthesis, or a wildcard in a `semantic` path). `verify-suite.sh` checks project wiring. Both exit non-zero on error and run in CI via the new `make skills` target, and both read ground truth from `src/ConfIT/` — supported frameworks, built-in matcher names — rather than hardcoding it. + +### Changed + +- **Skills read ConfIT's own `example/` and `doc/` instead of shipping templates.** Embedded copies of the fixture, `suite.config.yaml` and `.csproj` drifted from the real projects, so they were removed; each skill now resolves the repository it ships inside and reads the suites CI verifies. + +### Fixed + +- **`example/User.IntegrationTests`** — `TestReader.GetTestsForAFile` was called with its arguments reversed; removed `AuthTokenProvider.cs`, which implemented `IAuthTokenProvider` but was never referenced, since no `SuiteBootstrapper` overload accepts a custom provider (auth is configured declaratively). The `qa` environment now reads its URL from `${QA_API_URL}` instead of a hardcoded host. +- **`example/User.ComponentTests`** — `appsettings.Tests.json` carried an `environmentVariables` key that nothing reads; `UserDbInitializer.Seed()` had a commented-out body and now seeds real reference data, demonstrating the `onStarted` hook. +- **`example/User.ComponentTests.AppLauncher`** — configuration comments referred to a `Startup.SeedDatabase` method that does not exist, and now describe the actual `IsLocalComponentTests` / `UserDbContext` mechanism; the namespace was aligned with the project folder. + +Other than `mock.enableLogs`, the library is unchanged — the skills are distributed as agent plugins, not as package content. + +--- + ## [3.1.0] ### Added diff --git a/CLAUDE.md b/CLAUDE.md index acbdaa5..07a2c24 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -54,7 +54,8 @@ make test # unit + component tests only make unit # unit tests only make component # component tests only make integration # wipe DB, start services, run integration tests, stop services -make ci # full pipeline: build + unit + component + integration +make skills # validate the agent skills' scripts against every example suite +make ci # full pipeline: build + unit + component + skills + integration make clean # stop services, remove build artefacts and SQLite DB make services-stop# kill running User.Api / JustAnotherService processes make help # list all targets with descriptions @@ -337,6 +338,42 @@ The `example/` projects are not just demos — they run in CI as regression gate --- +## Agent Skills + +Three agent skills live in `skills/` at the repository root and are documented in +[doc/ai-skills.md](doc/ai-skills.md): + +- **`confit-suite-setup`** — wiring a suite: startup mode, `suite.config.yaml`, fixture, test class, auth, filters +- **`confit-component-tests`** — developer, mid-implementation: works from the controller plus the mocks behind it +- **`confit-integration-tests`** — QA, post-deployment: black box, works from a spec/collection, assumes no source + +Authoring is split by **persona**, not by feature. The two produce nearly the same artifact but are +different jobs — different input, state model, matcher instinct and achievable test matrix. Do not +merge them back. + +The skills are **not** shipped in the NuGet package — they are distributed as agent plugins +only. The repository root doubles as a plugin root (`.claude-plugin/`, `.codex-plugin/`; further +agents get one manifest directory each). Installing a plugin clones the whole repository, which is +what lets the skills read `example/` and `doc/` directly. + +`skills/` must stay at the repository root: the manifests name `./skills/`, and +`skills/confit-suite-setup/scripts/reference-path.sh` resolves the reference by finding a +directory holding `example/`, `doc/` and `skills/` as siblings. Work on the skills locally with +`claude --plugin-dir .` rather than copying them anywhere. + +**`confit-suite-setup` must not gain template files again.** It was rebuilt to read the live +`example/` suites precisely because embedded copies of the fixture, config and `.csproj` went +stale — `example/` took 21 commits in twelve months. New setup patterns go into `example/`, where +CI runs them, and `example/README.md` records what is structural versus demo-specific. + +Shared executables live in `tools/`, not inside a skill: `tools/check-testcases.py` (test +definitions) and `tools/verify-suite.sh` (project wiring). Both must report zero errors against +every suite in `example/` — `make skills` enforces this and runs as part of `make ci`. Each reads +ground truth from `src/ConfIT/` (supported frameworks, built-in matcher names) rather than +hardcoding it, so they do not drift. + +--- + ## Curated References - [.NET API docs](https://learn.microsoft.com/en-us/dotnet/api/) — official Microsoft reference diff --git a/Makefile b/Makefile index e3f7acb..132ae9a 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ API_PORT := 5170 SVC_PORT := 9999 DB := example/User.Api/User.db -.PHONY: default help build unit component component.applauncher test integration services-start services-stop ci clean +.PHONY: default help build unit component component.applauncher test integration services-start services-stop skills ci clean default: build test @@ -62,8 +62,20 @@ services-stop: ## Kill any running User.Api / JustAnotherService processes @lsof -ti :$(API_PORT) 2>/dev/null | xargs kill -9 2>/dev/null || true @lsof -ti :$(SVC_PORT) 2>/dev/null | xargs kill -9 2>/dev/null || true +# ── Agent skills ──────────────────────────────────────────────────────────────── +skills: ## Validate the agent-skill tooling against every example suite + @for p in example/User.ComponentTests example/User.IntegrationTests example/User.ComponentTests.AppLauncher; do \ + printf " %-42s" "$$p"; \ + python3 tools/check-testcases.py "$$p" > /tmp/confit-skills.log 2>&1 \ + || { echo "FAILED"; cat /tmp/confit-skills.log; exit 1; }; \ + bash tools/verify-suite.sh "$$p" >> /tmp/confit-skills.log 2>&1 \ + || { echo "FAILED"; cat /tmp/confit-skills.log; exit 1; }; \ + echo "ok"; \ + done + @echo " ✓ Skill scripts clean on all example suites" + # ── Pipelines ─────────────────────────────────────────────────────────────────── -ci: build test component.applauncher integration ## Full pipeline: build + unit + component + integration +ci: build test component.applauncher skills integration ## Full pipeline: build + unit + component + skills + integration clean: services-stop ## Stop services, remove build artefacts and SQLite DB @rm -f $(DB) diff --git a/README.md b/README.md index ca796c5..ec363c1 100755 --- a/README.md +++ b/README.md @@ -32,6 +32,15 @@ Component and integration tests share a large common surface — how tests are d → **[Test Execution Flow](doc/test-execution-flow.md)** — ASCII flow diagrams showing what happens at runtime across all three suite types. +→ **[AI Agent Skills](doc/ai-skills.md)** — three skills that let a coding agent set up a suite, write component tests from your controller, and write API tests from your spec. Install as a plugin: + +``` +/plugin marketplace add techygarg/ConfIT +/plugin install confit@confit +``` + +Codex has its own manifest; more agents are added over time. The skills read the live [`example/`](example) suites rather than carrying templates, so what they show you is what CI proves works. + --- ## Documentation @@ -71,12 +80,13 @@ Component and integration tests share a large common surface — how tests are d |---|---| | [Reading Failure Output](doc/failure-output.md) | Per-field failure messages, path notation, suite summary table, debugging tips | | [Extending ConfIT](doc/extending-confit.md) | `ITestOutputLogger`, `ITestProcessor` / `ITestProcessorFactory` hooks, custom semantic matchers, `IAuthTokenProvider` | +| [AI Agent Skills](doc/ai-skills.md) | `confit-suite-setup`, `confit-component-tests`, `confit-integration-tests` — plugin install, why authoring splits by persona, plus two standalone validation scripts | --- ## Example Projects -The `example/` directory contains a working reference implementation: +The `example/` directory contains working reference implementations of all three startup modes — see [`example/README.md`](example/README.md) for the mode-to-project map and what is demo-specific: | Project | Role | |---|---| diff --git a/doc/Package.Readme.md b/doc/Package.Readme.md index c5f4ea6..a6fff19 100644 --- a/doc/Package.Readme.md +++ b/doc/Package.Readme.md @@ -82,6 +82,14 @@ await Execute(testName, test, sourceFile); --- +## Agent Skills + +Two agent skills live in the repository — one for wiring up a suite, one for turning an OpenAPI spec, ASP.NET controller, or GraphQL schema into test definitions. They install as an agent plugin (Claude Code, Codex, more over time) and are not part of this package. + +→ [Agent Skills documentation](https://github.com/techygarg/ConfIT/blob/main/doc/ai-skills.md) + +--- + ## Full Documentation → [github.com/techygarg/ConfIT](https://github.com/techygarg/ConfIT) diff --git a/doc/ai-skills.md b/doc/ai-skills.md new file mode 100644 index 0000000..4c76454 --- /dev/null +++ b/doc/ai-skills.md @@ -0,0 +1,178 @@ +# AI Agent Skills + +ConfIT ships three [Agent Skills](https://docs.claude.com/en/docs/claude-code/skills) that teach a +coding agent how to use this library. They live in [`skills/`](../skills) at the repository root. + +| Skill | Who, and when | Works from | +|---|---|---| +| [`confit-suite-setup`](../skills/confit-suite-setup) | anyone adding ConfIT to a project | the startup mode you need — in-process, command/AppLauncher, or integration | +| [`confit-component-tests`](../skills/confit-component-tests) | a developer, while implementing | the controller, plus the mocks behind it | +| [`confit-integration-tests`](../skills/confit-integration-tests) | QA or a peer, after deployment | an API spec, a collection, or a live endpoint — black box | + +They are **not** part of the `ConfIT` NuGet package. `dotnet add package ConfIT` gives you the +test runner; an agent plugin gives you the know-how to drive it. + +--- + +## Why authoring is two skills + +The two authoring skills produce nearly the same artifact — the same lifecycle scenario is 89 +lines as a component test and 64 as an integration test, differing only by a `mock:` block. But +they are different jobs: + +| | Component | Integration | +|---|---|---| +| Primary input | the controller in code | the API spec | +| Reads deeper? | yes — far enough to find the mocks | no; stays at the contract | +| Repo access | full | often none; may be a separate repository | +| Dependencies | mocked; discovering them is half the work | real | +| State | fresh per process | shared, persistent | +| Matcher instinct | exact values are safe | lean on `semantic` / `ignore` | +| Can force a dependency failure? | yes — that is the point | no | + +Splitting on the persona rather than the feature means each skill can be direct rather than +hedged. The integration skill contains no advice that assumes repo access; the component skill +contains no advice about shared-environment data hygiene. + +What they share — the DSL field table and matcher decision order — is roughly 500 words, and each +carries its own copy tailored to its persona. Depth routes to `doc/` for both. + +--- + +## Install + +**Claude Code** + +``` +/plugin marketplace add techygarg/ConfIT +/plugin install confit@confit +``` + +Update with `/plugin update confit`, remove with `/plugin uninstall confit`. This repository is +its own marketplace — [`.claude-plugin/marketplace.json`](../.claude-plugin/marketplace.json) +declares it, and [`.claude-plugin/plugin.json`](../.claude-plugin/plugin.json) declares the plugin +whose `skills/` folder is the one documented here. + +**Codex** — [`.codex-plugin/plugin.json`](../.codex-plugin/plugin.json) carries the equivalent +manifest, pointing at the same `skills/` folder. + +**Other agents** are onboarded by adding one more manifest directory at the repository root. That +is the whole integration: a manifest names `./skills/`, and nothing about the skills changes. + +**Working inside this repository**, load it as a local plugin rather than copying anything: + +```bash +claude --plugin-dir . +``` + +--- + +## Why plugins, and not an install script + +A plugin install clones the **whole repository** into the agent's plugin cache — not just +`skills/`. That single fact removes an entire category of problem. + +These skills need to show a consumer *correct, current* configuration, fixtures and test +definitions. Anything they carried as a template would be a copy that drifts: `example/` took 21 +commits in twelve months, and the files such templates mirror churned hardest of all — one +`.csproj` alone changed 13 times, and the fixture's shape changed materially when +`SuiteBootstrapper` was introduced. A stale sample is worse than none, because an agent trusts it +over the real repository. + +Because the plugin brings the repository, each skill reads [`example/`](../example) — suites +verified by `make ci` — and [`doc/`](../doc) directly. Nothing to sync, nothing to version. + +### Locating the reference + +Each skill resolves the repository root by walking up from its own physical location, looking for +a directory that holds `example/`, `doc/` and `skills/`: + +```bash +bash skills//scripts/reference-path.sh # prints the root +bash skills//scripts/reference-path.sh --check # and what it found +``` + +Symlink-safe, and works from a plugin cache, a clone, or a local `--plugin-dir` load. When it +cannot find the reference it exits non-zero with reinstall instructions and the skill stops — it +will not fall back to generating tests from memory. + +Consequence worth stating plainly: copying a skill folder on its own, away from the repository, +gives you a skill that refuses to run. That is the intended failure. + +--- + +## The bundled scripts + +Both live in [`../tools/`](../tools) rather than inside a skill, since two skills share the first +one. Neither needs a build, both exit non-zero on error, and `make skills` runs them against every +suite in `example/` as part of `make ci`. + +**Validate test definitions before running them:** + +```bash +python3 tools/check-testcases.py path/to/TestProject +``` + +Reports what ConfIT would otherwise raise at load time or on the first failing assertion: a +response with no expected body, `depends:` naming an unknown or later test, `{{variables}}` that +are never extracted or are ambiguous, `bodyFromFile` pointing at a missing file, `mock:` blocks in +an integration suite, unregistered test files, and matcher problems — an unknown or misspelled +matcher name, a missing closing parenthesis, a wildcard or array index in a `semantic` path, or a +trailing `*`. The built-in matcher list is read from `src/ConfIT/Matching/SemanticMatcher.cs`, and +names registered as custom matchers in the project's own C# are accepted, so the check stays +correct as the library and the project evolve. + +**Check a suite's wiring:** + +```bash +bash tools/verify-suite.sh path/to/TestProject +``` + +Reports missing package references, unregistered config or test files, a fixture that does not +match the config section, a hardcoded filter env var, and framework mismatches — reading the +supported frameworks from `src/ConfIT/ConfIT.csproj` rather than a hardcoded list. + +--- + +## What is inside a skill + +``` +skills/ + confit-suite-setup/ + SKILL.md locate reference → decide mode → read → adapt → verify + references/troubleshooting.md + confit-component-tests/ + SKILL.md controller → mocks → matrix → write → verify + references/mock-discovery.md trace the call path, branch points, narrowest match, + the observation loop for when there is no source + references/writing-tests.md DSL table, matcher order, example index, diagnosing + confit-integration-tests/ + SKILL.md contract → environment/auth → matrix → write → verify + references/environments-and-auth.md environment precedence, the three auth profiles, + the OAuth2 no-refresh caveat, auth tests worth writing + references/state-and-data.md shared state, ${RUN_ID} uniqueness, trailing-DELETE + cleanup, what the library does not support + references/writing-tests.md the same skeleton, tailored to black-box instincts +``` + +Every skill also carries `scripts/reference-path.sh`. Only `SKILL.md` enters the agent's context +when a skill triggers; reference files load when needed. + +--- + +## Maintaining them + +**No skill may gain template files.** New setup or test patterns belong in `example/`, where CI +runs them. If a skill needs to teach something new, teach it by pointing at the example that +demonstrates it. + +`skills/` must stay at the repository root: the plugin manifests name `./skills/`, and the +reference resolver expects `example/`, `doc/` and `skills/` to be siblings. + +`example/README.md` is the curation layer — it names what is structural and what is +demo-specific. Keep it accurate when the example projects change; it is what stops an agent +copying `UserDbInitializer` or a port number into someone else's project. + +One caveat that is easy to forget: `example/User.IntegrationTests` is not a template for a +deployed environment. It uses hardcoded data and relies on `make integration` wiping the database +first. The integration skill says so explicitly — keep that warning in place. diff --git a/doc/doc-strategy.md b/doc/doc-strategy.md index 2b15ba5..20aa1da 100644 --- a/doc/doc-strategy.md +++ b/doc/doc-strategy.md @@ -21,6 +21,7 @@ This file tracks which documents exist, which are missing, and the conventions t | `graphql-support.md` | ✅ current | `graphql` request block, `queryFromFile`, mutations + `extract`, error-array matching | | `test-filtering.md` | ✅ current | TEST_TAGS, TEST_NAMES, CI patterns | | `failure-output.md` | ✅ current | Field-level failure messages, path notation, suite summary, debugging tips | +| `ai-skills.md` | ✅ current | The three agent skills, plugin install, why authoring splits by persona, the `tools/` validators | | `doc-strategy.md` | ✅ this file | Planning only | --- diff --git a/doc/mock-interactions.md b/doc/mock-interactions.md index 868e91e..05ce774 100644 --- a/doc/mock-interactions.md +++ b/doc/mock-interactions.md @@ -253,7 +253,29 @@ This is the primary reason to prefer YAML when a test has several interactions s If the service makes an outbound call that does not match any declared interaction, WireMock returns a `404` response with no body. The service will handle that response however it is coded — but in most cases the test will fail on the `api.response` assertion, reporting an unexpected status code or body. -This is the intended behaviour: an unmatched call surfaces immediately as a test failure rather than silently passing or hanging. To diagnose it, enable `EnableMockServerLogs` in `SuiteConfig` — WireMock will print each incoming request and whether it matched a stub. +This is the intended behaviour: an unmatched call surfaces immediately as a test failure rather than silently passing or hanging. To diagnose it, set `enableLogs` on the `mock:` block — WireMock will print each incoming request and whether it matched a stub. + +```yaml +component: + mock: + url: http://localhost:8888 + enableLogs: true +``` + +### Discovering what a service calls + +The same switch turns an unknown dependency surface into a listing. Write the test with **no** `mock` block, turn logging on, and run it — every outbound call comes back `404` and is logged: + +``` +[Warn] : HttpStatusCode set to 404 : No matching mapping found + "Path": "/api/demo/test@test.com", + "Url": "http://localhost:8888/api/demo/test@test.com", + "Status": "No matching mapping found" +``` + +Each unmatched entry is one outbound call, with its method, path, query and body — enough to write the interactions from. This works whatever language the service under test is written in, because it observes HTTP rather than code, which makes it the practical way to build mocks for an [AppLauncher](./app-launcher.md) suite. + +**Ignore the test's pass/fail during this loop and read the log.** A test can pass with every mock missing: the unmatched `404`s make the service take its dependency-failure path, which may be exactly the error the test expected. --- diff --git a/doc/suite-setup.md b/doc/suite-setup.md index 084f32b..041f13c 100644 --- a/doc/suite-setup.md +++ b/doc/suite-setup.md @@ -25,6 +25,7 @@ component: url: http://localhost:5170 mock: url: http://localhost:8888 + # enableLogs: true # print every request WireMock receives, and whether it matched folders: response: responses filter: @@ -333,7 +334,7 @@ public class TestSuiteFixture : IDisposable |---|---| | `ApiServerUrl` | Base URL of the service under test. Can be empty for in-process component tests — the `TestServer` handles routing. | | `MockServerUrl` | WireMock base URL. Omit or leave empty to disable mocking. | -| `EnableMockServerLogs` | Print WireMock request logs to the console. Useful during debugging. | +| `EnableMockServerLogs` | Print WireMock request logs to the console. Set from YAML with `mock.enableLogs`. Useful when debugging a stub that will not match, or to discover what a service calls — see [Mock Interactions](./mock-interactions.md#discovering-what-a-service-calls). | | `RequestBodyFolder` | Folder to resolve `bodyFromFile` paths in request definitions. | | `ResponseBodyFolder` | Folder to resolve `bodyFromFile` paths in expected response definitions. | | `ApiResponseFolder` | Folder where actual responses are written after each test. | diff --git a/example/README.md b/example/README.md new file mode 100644 index 0000000..836001a --- /dev/null +++ b/example/README.md @@ -0,0 +1,93 @@ +# ConfIT Examples + +Working reference implementations of all three ConfIT startup modes. Every suite here runs in +CI (`make ci`), so what you read is what currently passes. + +If you are setting up your own suite, read the project matching your mode below, then read +**[What is demo-specific](#what-is-demo-specific)** before copying anything. + +--- + +## The three modes + +| Mode | Project | Use it when | Fixture call | +|---|---|---|---| +| **In-process component** | [`User.ComponentTests`](User.ComponentTests) | The service is .NET and the test project can reference it. Fastest start, full DI access for seeding. | `SuiteBootstrapper.ForComponent` | +| **Command / AppLauncher** | [`User.ComponentTests.AppLauncher`](User.ComponentTests.AppLauncher) | Any language, or a .NET app you want to exercise as a real process. ConfIT runs a shell command and speaks HTTP — the test project never references the app. | `SuiteBootstrapper.ForCommand` | +| **Integration** | [`User.IntegrationTests`](User.IntegrationTests) | Services are already running — locally, in CI, or in a deployed environment. Nothing is mocked. | `SuiteBootstrapper.ForIntegration` | + +Both component modes can stub outbound dependencies with WireMock via a `mock:` block. Integration +suites cannot — a `mock:` block in an integration test definition is an error. + +### Supporting projects + +| Project | Role | +|---|---| +| [`User.Api`](User.Api) | The service under test — a small user API (REST + GraphQL) on port 5170 | +| [`JustAnotherService`](JustAnotherService) | An external dependency the API calls. Real on port 9999; stubbed by WireMock on 8888 during component tests | + +--- + +## What every suite needs + +These four things are structural — ConfIT does not work without them: + +1. **`suite.config.yaml`** in the test project root, registered in the `.csproj` with + `Always`. Its top-level key (`component:` or + `integration:`) selects the loader. +2. **A fixture** implementing `IDisposable` that calls one `SuiteBootstrapper.For*` method, + exposes `TestSuiteContext Context`, and disposes the suite — disposal prints the summary and + shuts down infrastructure. +3. **A test class** deriving from `BaseTest`, using `IClassFixture` so the suite + starts once per class, with a `[Theory]` fed by + `[MemberData(nameof(GetTestCasesForFolder), "TestCase")]` → `TestReader.GetTestsForAFolder`. + That is the only discovery mechanism ConfIT has. +4. **Every runtime file registered in the `.csproj`** — test definitions, `suite.config.yaml`, + body fixtures, app settings. `TestReader` reads from the build output directory, so a missing + `` entry is the single most common cause of "no tests discovered". + +Two things that look required but are not: + +- **`TestOutputLogger`** is optional. It is a three-line adapter onto xUnit's `ITestOutputHelper`; + `BaseTest`'s logger parameter is nullable. `User.ComponentTests.AppLauncher` passes `null` and + works fine. +- **`appsettings.Tests.json`** is needed only for `mode: in-process`, where `startup.settings` + names it. Neither other mode has one. + +--- + +## What is demo-specific + +Everything below exists because this example happens to be a user API. **Do not carry it into +your own project.** + +| Thing | Where | Why it is here | +|---|---|---| +| `User.*` namespaces, `UserComponentTests` / `UserIntegrationTests` class names | all suites | Naming for this demo | +| `UserDbInitializer` | `User.ComponentTests/SetUp/` | Shows *where* to seed (the `onStarted` hook). Your seeding will look nothing like this — and data a single test needs should be created by that test over HTTP instead | +| `JustAnotherService` | `appsettings.*.json`, mock interactions | This demo's one outbound dependency. Yours will have different ones, or none | +| Ports `5170`, `8888`, `9999`, `8887` | configs, launch profiles | Arbitrary. `5170` is the API, `8888` the WireMock stub, `9999` the real dependency, `8887` a hand-started WireMock serving OAuth2 tokens | +| `IsLocalComponentTests` flag, `Startup.AddDbContexts` branching | `User.Api` | How *this* app selects an in-memory database under test. The pattern (app owns its test environment) transfers; the flag name does not | +| Everything under `TestCase/` | all suites | Test definitions for this API's endpoints. Useful to read for DSL patterns, not to copy | +| `Microsoft.EntityFrameworkCore.InMemory` | `User.ComponentTests.csproj` | This app uses EF Core. Yours may not | +| `WireMock.Net` as a direct package | `User.ComponentTests.AppLauncher.csproj` | Only needed because that fixture hand-starts an extra WireMock server for the OAuth2 token endpoint. ConfIT's own `mock:` block needs no package reference | +| `folders.response: responses` vs `ApiResponses` | component vs integration configs | Two arbitrary names for the same thing. ConfIT imposes neither | + +The `protected virtual` hooks on `User.Api.Startup` (`AddDbContexts`, `AddMvcServices`) exist as +extension points but are **not overridden anywhere in this repo** — the in-memory/SQLite choice is +driven entirely by the `IsLocalComponentTests` config flag. Treat them as available, not as +demonstrated. + +--- + +## Running them + +```bash +make component # in-process component suite +make component.applauncher # command mode — starts User.Api as a real process on 5170 +make integration # wipes the DB, starts both services, runs, tears down +make ci # everything +``` + +Full documentation is in [`doc/`](../doc) — start with +[Suite Setup](../doc/suite-setup.md), then [AppLauncher](../doc/app-launcher.md) for command mode. diff --git a/example/User.ComponentTests.AppLauncher/SetUp/TestSuiteFixture.cs b/example/User.ComponentTests.AppLauncher/SetUp/TestSuiteFixture.cs index 5a84c43..38ba7a2 100644 --- a/example/User.ComponentTests.AppLauncher/SetUp/TestSuiteFixture.cs +++ b/example/User.ComponentTests.AppLauncher/SetUp/TestSuiteFixture.cs @@ -4,7 +4,7 @@ using WireMock.ResponseBuilders; using WireMock.Server; -namespace User.ComponentTests.Launcher.SetUp +namespace User.ComponentTests.AppLauncher.SetUp { /// /// Demonstrates the AppLauncher (command) mode with OAuth2 client credentials auth. diff --git a/example/User.ComponentTests.AppLauncher/UserComponentTests.cs b/example/User.ComponentTests.AppLauncher/UserComponentTests.cs index 1c46f66..4bd0d0d 100644 --- a/example/User.ComponentTests.AppLauncher/UserComponentTests.cs +++ b/example/User.ComponentTests.AppLauncher/UserComponentTests.cs @@ -4,11 +4,11 @@ using ConfIT.Extension; using ConfIT.Reader; using Newtonsoft.Json.Linq; -using User.ComponentTests.Launcher.SetUp; +using User.ComponentTests.AppLauncher.SetUp; using Xunit; using Xunit.Abstractions; -namespace User.ComponentTests.Launcher +namespace User.ComponentTests.AppLauncher { public class UserComponentTests : BaseTest, IClassFixture { diff --git a/example/User.ComponentTests.AppLauncher/suite.config.yaml b/example/User.ComponentTests.AppLauncher/suite.config.yaml index 78bc1b7..73bb212 100644 --- a/example/User.ComponentTests.AppLauncher/suite.config.yaml +++ b/example/User.ComponentTests.AppLauncher/suite.config.yaml @@ -1,10 +1,14 @@ # AppLauncher component test configuration. # # startup.mode: command — AppLauncher starts User.Api as an external process before -# tests run and stops it after. The app seeds its own database on startup (see -# User.Api/Startup.cs:SeedDatabase and appsettings.ComponentTest.json). +# tests run and stops it after. ConfIT never reaches into the app; the app configures its +# own test environment. Here the ComponentTest launch profile sets +# ASPNETCORE_ENVIRONMENT=ComponentTest, which loads appsettings.ComponentTest.json — +# that sets IsLocalComponentTests=true (Startup then selects the in-memory database) and +# points JustAnotherService at the mock on :8888. UserDbContext creates a fresh schema +# once per process. # -# The command path is relative to the test output directory (bin/Debug/net9.0/). +# The command path is relative to the test output directory (bin/Debug/net10.0/). # Adjust if your project layout differs. component: startup: @@ -13,7 +17,7 @@ component: # stopCommand: explicitly kills the process by port on dispose. # Without this, AppLauncher kills the process tree and waits for port release automatically. # Provide this when the default kill does not release the port reliably on your OS. - # Unix/macOS: lsof -ti :5170 | xargs kill -9 + # Unix/macOS: lsof -ti :5170 -sTCP:LISTEN | xargs kill -9 # Windows: taskkill /F /IM User.Api.exe stopCommand: lsof -ti :5170 -sTCP:LISTEN | xargs kill -9 readiness: diff --git a/example/User.ComponentTests/SetUp/UserDbInitializer.cs b/example/User.ComponentTests/SetUp/UserDbInitializer.cs index 830ced5..ae5f648 100755 --- a/example/User.ComponentTests/SetUp/UserDbInitializer.cs +++ b/example/User.ComponentTests/SetUp/UserDbInitializer.cs @@ -1,10 +1,16 @@ using System.Collections.Generic; +using System.Linq; using User.Api.Persistence; namespace User.ComponentTests.SetUp { /// - /// Add any initial data set-up required for testApi suite. It will runs only one during all tests runs + /// Reference data inserted once, before any test runs, from the fixture's onStarted hook. + /// + /// This is the place for data that must exist before the suite starts — lookup tables, + /// a tenant row, a fixed admin account. Data a single test needs should be created by + /// that test over HTTP instead, so the test stays readable and self-contained: the test + /// definitions in TestCase/ create every user they assert on. /// public class UserDbInitializer { @@ -15,18 +21,16 @@ public UserDbInitializer(UserDbContext context) => public void Seed() { - //SeedUsers(_context); - } + // UserDbContext creates a fresh schema once per process, so this runs against an + // empty database — but guard anyway, so seeding stays safe to call more than once. + if (_context.Users.Any()) return; - private static void SeedUsers(UserDbContext context) - { - var data = new List + _context.AddRange(new List { - new() { Name = "User1", Id = 1, Age = 10, Email = "testApi@testApi.com" }, - new() { Name = "User2", Id = 2, Age = 20, Email = "test1@testApi.com" }, - }; - context.AddRange(data); - context.SaveChanges(); + new() { Id = 1, Name = "Seed User One", Email = "seed-user-1@test.com", Age = 41 }, + new() { Id = 2, Name = "Seed User Two", Email = "seed-user-2@test.com", Age = 42 } + }); + _context.SaveChanges(); } } -} \ No newline at end of file +} diff --git a/example/User.ComponentTests/appsettings.Tests.json b/example/User.ComponentTests/appsettings.Tests.json index ca278f2..ca3d70f 100755 --- a/example/User.ComponentTests/appsettings.Tests.json +++ b/example/User.ComponentTests/appsettings.Tests.json @@ -1,7 +1,4 @@ { - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "component-tests" - }, "JustAnotherService": { "Url": "http://localhost:8888" }, diff --git a/example/User.ComponentTests/suite.config.yaml b/example/User.ComponentTests/suite.config.yaml index 011bbce..6a5fdd0 100644 --- a/example/User.ComponentTests/suite.config.yaml +++ b/example/User.ComponentTests/suite.config.yaml @@ -6,6 +6,8 @@ component: url: http://localhost:5170 mock: url: http://localhost:8888 + # enableLogs: true # log every request WireMock receives — also how you discover + # # what the service calls: run with no mock: block and read the log folders: response: responses requestBody: TestCase/Request diff --git a/example/User.IntegrationTests/AuthTokenProvider.cs b/example/User.IntegrationTests/AuthTokenProvider.cs deleted file mode 100755 index 1c37bd9..0000000 --- a/example/User.IntegrationTests/AuthTokenProvider.cs +++ /dev/null @@ -1,12 +0,0 @@ -using ConfIT.Contract; - -namespace User.IntegrationTests -{ - public class AuthTokenProvider : IAuthTokenProvider - { - public string Token() - { - return "Bearer Have Your Token Provider Implementation Here."; - } - } -} \ No newline at end of file diff --git a/example/User.IntegrationTests/UserIntegrationTests.cs b/example/User.IntegrationTests/UserIntegrationTests.cs index 212091b..01155ba 100755 --- a/example/User.IntegrationTests/UserIntegrationTests.cs +++ b/example/User.IntegrationTests/UserIntegrationTests.cs @@ -22,7 +22,7 @@ public async Task ExecuteTest(string testName, JToken test, string sourceFile) = await Execute(testName, test, sourceFile); public static IEnumerable GetTestCases(string fileName) => - TestReader.GetTestsForAFile(fileName, "TestCase"); + TestReader.GetTestsForAFile("TestCase", fileName); public static IEnumerable GetTestCasesForFolder(string folder) => TestReader.GetTestsForAFolder(folder); diff --git a/example/User.IntegrationTests/suite.config.yaml b/example/User.IntegrationTests/suite.config.yaml index 8d4f9bf..8a7c025 100644 --- a/example/User.IntegrationTests/suite.config.yaml +++ b/example/User.IntegrationTests/suite.config.yaml @@ -14,7 +14,7 @@ integration: qa: api: - url: "https://api.qa.example.com" + url: ${QA_API_URL} auth: type: bearer token: ${QA_API_TOKEN} diff --git a/skills/README.md b/skills/README.md new file mode 100644 index 0000000..9640725 --- /dev/null +++ b/skills/README.md @@ -0,0 +1,81 @@ +# ConfIT Agent Skills + +Three [Agent Skills](https://docs.claude.com/en/docs/claude-code/skills) that teach a coding agent +how to use ConfIT. They split by the job being done, not by the feature being used. + +| Skill | Who, and when | Works from | +|---|---|---| +| [`confit-suite-setup`](./confit-suite-setup) | anyone adding ConfIT to a project | the startup mode you need | +| [`confit-component-tests`](./confit-component-tests) | a developer, while implementing | **the controller, plus the mocks behind it** | +| [`confit-integration-tests`](./confit-integration-tests) | QA or a peer, after deployment | **the API spec**, black box, no source | + +The two authoring skills are genuinely different jobs. A component test is written from your own +code, mocks the dependencies, runs against fresh state, and can force a dependency to fail. An +integration test is written from a spec or a collection, hits a real deployed service, shares +persistent data with everyone else, and cannot make a dependency misbehave. + +--- + +## Install + +They ship as agent plugins. Installing a plugin brings the whole repository along, so each skill +sits next to the live [`example/`](../example) suites and [`doc/`](../doc) it reads — no templates +to go stale. + +**Claude Code** + +``` +/plugin marketplace add techygarg/ConfIT +/plugin install confit@confit +``` + +**Codex** — the repository carries a [`.codex-plugin/`](../.codex-plugin) manifest. + +Manifests for further agents are added over time; each is a directory at the repository root, so +adding one never touches the skills themselves. + +Working inside this repository, load it as a local plugin: + +```bash +claude --plugin-dir . +``` + +> Do not copy a skill folder on its own. Each one reads `example/` and `doc/` from the repository +> around it and refuses to run without them — by design, so it can never generate tests from a +> stale template. + +--- + +## Then just ask + +The agent picks the right skill from how you phrase it. + +> Set up a ConfIT component test suite for `MyService.Api`. +> +> I just implemented `CreateOrder` — write component tests for it. +> +> The mock isn't matching and my test gets a 404. +> +> Here's our OpenAPI spec — write tests against staging. +> +> Convert this Postman collection into ConfIT tests. + +--- + +## Two scripts you can run yourself + +Both live in [`../tools/`](../tools), need no build, and exit non-zero on error, so either can gate +CI. `make skills` runs them against every suite in `example/`. + +```bash +python3 ../tools/check-testcases.py path/to/TestProject +bash ../tools/verify-suite.sh path/to/TestProject +``` + +The first validates test definitions — missing expected bodies, bad `depends:`, unresolvable +`{{variables}}`, unknown or malformed matcher names, `mock:` blocks in an integration suite, +unregistered files. It reads the built-in matcher list from `src/ConfIT/Matching/SemanticMatcher.cs` +rather than a hardcoded copy, so it stays correct as the library gains matchers. + +The second checks project wiring — package references, config registration, fixture shape, +framework mismatch. diff --git a/skills/confit-component-tests/SKILL.md b/skills/confit-component-tests/SKILL.md new file mode 100644 index 0000000..8449c1c --- /dev/null +++ b/skills/confit-component-tests/SKILL.md @@ -0,0 +1,135 @@ +--- +name: confit-component-tests +description: This skill should be used when a developer asks to "write component tests", "add ConfIT tests for this controller", "I just implemented X, test it", "test this endpoint against mocks", "mock this dependency", "my component test is calling the real service", "the mock isn't matching", or when adding or editing test definitions in a ConfIT suite whose config has a `component:` section. Works from the developer's own source — the controller, its handlers and the outbound clients behind them — to produce test definitions plus the WireMock interactions they need. For black-box tests against a deployed environment, use confit-integration-tests instead. +--- + +# ConfIT Component Tests + +Written by the developer, from their own code, while the service still runs on their machine or +CI. **Two inputs: the controller, and the mocks behind it.** Nothing else is needed and nothing +else should be read. + +Setting up the suite itself is a different job — use `confit-suite-setup`. + +## Step 0 — Resolve the reference, then confirm the mode + +```bash +bash /scripts/reference-path.sh +``` + +It prints the ConfIT repository root, ``. Everything below is relative to it. If it exits +non-zero, stop: the install is broken. Do not write tests from memory — the DSL and matcher set +move between releases. + +Then open the target project's `suite.config.yaml`: + +- Top-level key is **`component:`** → continue here. +- Top-level key is **`integration:`** → wrong skill. Integration suites cannot mock; hand over to + `confit-integration-tests`. +- **No suite at all** → hand over to `confit-suite-setup` first. + +Note whether `component.mock.url` is set. Without it no mock server runs, and every outbound call +will hit the real dependency. + +Read one or two neighbouring files in `TestCase/` before writing anything — match the project's +own file numbering, tag vocabulary and error-body shape. The house style beats any example. + +## Step 1 — The controller is the anchor + +Identify the controller or endpoint under test: named by the developer, or located in the repo. +Read it for the contract — routes, methods, request DTOs, response types, status codes. Expand +route tokens; a class-level `[Route("api/[Controller]")]` on `OrderController` means `/api/order`. + +Do not derive scope from `git diff` by default. The change may already be pushed and the tests +written on a later branch. Use the diff only when the developer says "test what I just changed". + +## Step 2 — Go down the call path, for the mocks + +This is the half of the job that no spec could give you. Follow controller → handler → outbound +client, and extract for every outbound call: **method, path, query params, headers, body**. + +On the way, note the **branch points** — the conditions under which the handler throws or returns +an error. They are usually driven by what a dependency *returned*, which is what makes error +tests writable at all. In ConfIT's own example, `CreateUserCommandHandler` throws +`BadRequestException("Invalid Email.")` when any of three dependency checks comes back +`isValid: false`; the test for that 400 drives it by changing the **mock response**, not the +request body. + +Then confirm the dependency's base-URL config key actually resolves to `mock.url` in the test +settings. If it does not, every component test silently calls the real service and may pass for +the wrong reason — that is a suite-setup defect, so hand it over. + +Full procedure, including what to do when there is no source to read: +`references/mock-discovery.md`. + +## Step 3 — Propose the matrix, confirm once + +For the endpoints on that controller: + +- the happy path +- each error branch found in Step 2, driven by the mock or by the request +- each `4xx` the controller itself owns (missing resource, invalid input) +- **dependency failure** — a `5xx` or a timeout from a stubbed dependency. This is the matrix + entry that only component tests can have; integration cannot force it. + +Show the list and get one confirmation before writing files. Do not generate a wall of tests +unasked. + +## Step 4 — Write + +Field reference, matcher decision order and an index of which example file demonstrates which +feature: `references/writing-tests.md`. + +Component-specific instincts: + +- **State is usually fresh each run, but ConfIT itself does not guarantee it.** It only starts + the application; whether the store behind it resets is that application's own behavior — true + of ConfIT's own example (an in-memory EF Core database, recreated per process) but not + automatic elsewhere. Confirm the app's DB setup — or ask — before assuming it, especially in + command mode (`ForCommand`/AppLauncher), where the launched process may point at a real, + persistent store. +- **When freshness is confirmed, exact-value assertions are safe and preferred.** Assert the + real `name`, `email`, `age`; reach for `semantic`/`ignore` only for genuinely server-generated + fields such as an id or a timestamp. Fixed literal test data is fine under the same condition — + do not copy that habit into an integration suite, and do not assume it here without checking. + When freshness is not confirmed, use the same run-unique-data approach as integration tests + (`confit-integration-tests`' `references/state-and-data.md`). +- Mocks go in the `mock:` block of the same test. Reuse a repeated response body with a YAML + anchor rather than restating it. + +## Step 5 — Verify + +```bash +python3 /tools/check-testcases.py +``` + +Catches what ConfIT would otherwise raise at load time or on first failure: a response with no +expected body, `depends:` naming an unknown or later test, unresolvable `{{variables}}`, unknown +or malformed matcher names, unregistered test files. + +Then run the suite. Every new test file needs a `` entry with +`Always`, or it will not exist at runtime and the +theory will simply not see it. + +When a test fails, work the mock angle first — it is the most common cause in this mode. See +"Diagnosing" in `references/writing-tests.md`. + +## When the service is not .NET + +An AppLauncher suite runs the app as an external process, which may be Go, Node, Python or +anything else. The controller-reading step becomes "find the route declaration in whatever +framework this is" — the shape of the job does not change: find the route, find the handler, find +the outbound calls. + +When the source is unreadable or absent, skip straight to the traffic-observation loop in +`references/mock-discovery.md`. It watches HTTP rather than code, so it works in any language. + +## Additional resources + +- **`references/mock-discovery.md`** — trace the call path, branch points, the narrowest-match + rule, and the observation loop for when there is no source. +- **`references/writing-tests.md`** — DSL field table, matcher decision order, the example index, + and diagnosing failures. +- **`/example/README.md`** — what in the examples is structural and what is demo-specific. +- **`/doc/`** — `test-file-format.md`, `matchers-and-patterns.md`, `mock-interactions.md`, + `graphql-support.md` for depth. diff --git a/skills/confit-component-tests/references/mock-discovery.md b/skills/confit-component-tests/references/mock-discovery.md new file mode 100644 index 0000000..cf0fccb --- /dev/null +++ b/skills/confit-component-tests/references/mock-discovery.md @@ -0,0 +1,172 @@ +# Mock Discovery + +The half of a component test that a contract cannot give you. An OpenAPI document for +`POST /api/user` says "post a user, get an id or a 400". It cannot tell you that the handler makes +three outbound calls, or that the 400 is triggered by what one of them returned. + +Everything here is about the **outbound** contract: what the service under test calls, and how its +answers change the service's behaviour. + +--- + +## 1. Trace the call path + +Controller → handler → outbound client. The client is whatever the project calls it: a typed +`HttpClient`, a provider, a gateway, a repository, an SDK wrapper. + +```bash +# .NET +grep -rn "AddHttpClient\|HttpClient\|BaseAddress" --include='*.cs' + +# any language — find where a base URL is read from config, then find its callers +grep -rn "http://\|https://" +``` + +For each outbound call, record: + +| | | +|---|---| +| method | `GET`, `POST`, … | +| path | including path parameters as they will actually be sent | +| query params | every one the client attaches | +| headers | only those the client actually sets | +| body | the shape, for `POST`/`PUT`/`PATCH` | + +**Count matters.** One inbound request can produce several outbound calls, and each needs its own +interaction. In ConfIT's own example, `CreateUserCommandHandler` calls a single dependency three +different ways: + +``` +VerifyByGet(email) GET /api/demo/{email} +VerifyByPost(email) POST /api/demo {"email": "..."} +VerifyByGetQuery(email) GET /api/demo?email=... +``` + +Three calls, three interactions. Nothing in the inbound contract hints at that number. + +## 2. Find the branch points + +While reading the handler, note every condition that changes the outcome — each `throw`, each +guard, each early return — and what feeds it. This is what makes error tests writable. + +Same example: + +```csharp +if (!isValid1.IsValid || !isValid2.IsValid || !isValid3.IsValid) + throw new BadRequestException("Invalid Email."); +``` + +So the 400 is driven by a **mock response**, not by the request body. Flip any one of the three +stubs to `isValid: false` and the API returns 400. That is the entire error test — see +`/example/User.ComponentTests/TestCase/02-user-errors.yaml`. + +This is the component mode's unique power: you can make a dependency answer anything, including +things that never happen in a healthy environment. + +| To test | Make the stub return | +|---|---| +| a validation/business rejection | the value the guard rejects | +| the dependency being down | `statusCode: 503` | +| the dependency returning nothing | an empty body or empty collection | +| a partial/degraded response | the payload with a field missing | + +## 3. Check the wiring before writing anything + +The application, not ConfIT, decides where its dependencies point. The dependency's base-URL +config key must resolve to `component.mock.url` in whatever settings the test run uses — the +`startup.settings` file for in-process mode, or the app's own environment-selected config in +command mode. + +If it does not, every component test silently calls the real service. Tests may still pass, for +the wrong reason. That is a suite wiring defect: hand it to `confit-suite-setup`. + +## 4. Declare the narrowest stub that identifies the call + +WireMock matches on **every field the interaction declares**. An over-specified stub — a header +the client does not actually send, a query param it omits — simply never matches, and the request +falls through to a `404`. The test then fails on an unexpected status with nothing pointing at the +mock. + +So: declare the least that uniquely identifies the call, and widen only when two stubs collide. +The three `/api/demo` stubs above are distinguished by path shape, query param and body +respectively — nothing more. + +```yaml +mock: + interactions: + - request: + method: GET + path: /api/demo/test@test.com + response: + statusCode: 200 + body: &valid { isValid: true } # anchor the repeated body + - request: + method: GET + path: /api/demo + params: { email: test@test.com } + response: { statusCode: 200, body: *valid } + - request: + method: POST + path: /api/demo + body: { email: test@test.com } + response: { statusCode: 200, body: *valid } +``` + +Body matching is structural and order-insensitive, so a partial body is a legitimate narrowing +tool. `Content-Type: application/json` is added to every mock response automatically. + +--- + +## 5. When there is no source to read + +An AppLauncher suite against a Go, Node or Python service; a third-party dependency; a binary you +cannot see inside. **Let the service tell you what it calls.** + +1. Turn on mock logging in `suite.config.yaml`: + + ```yaml + component: + mock: + url: http://localhost:8888 + enableLogs: true + ``` + +2. Write the test with **no `mock:` block at all**, and run it. + +3. WireMock answers every unmatched call with `404` and logs it. Read the log: + + ``` + [Warn] : HttpStatusCode set to 404 : No matching mapping found + "Path": "/api/demo/test@test.com", + "Url": "http://localhost:8888/api/demo/test@test.com", + "Status": "No matching mapping found" + ``` + + Every unmatched entry is one outbound call, with its method, path, query and body. That listing + *is* the outbound contract. + +4. Write the interactions from what you observed, remove `enableLogs`, and re-run. + +**The test result during this loop is meaningless — read the log, not the verdict.** A test can +easily pass while every mock is missing: unmatched calls return `404`, the service treats that as +a failed dependency check, and returns the very error the test expected. Passing proves nothing +here. + +This works in any language, because it observes HTTP rather than code. It is also the fastest way +to *confirm* a hand-written stub set is complete: if any unmatched request appears in the log, an +interaction is missing or over-specified. + +--- + +## 6. When a mock does not match + +Symptoms: an unexpected `404` from the dependency, or the service returning its +dependency-failure behaviour when you expected the happy path. + +1. Set `enableLogs: true` and re-run. The log shows what actually arrived. +2. Compare it field by field against the interaction. Method, path, **every** declared query + param and header must match exactly. +3. Suspect over-specification first — remove declared fields until it matches, then add back only + what is needed to distinguish it from other stubs. +4. Check the call is even reaching WireMock: if the log shows nothing, the dependency's base URL + is not pointed at `mock.url` (see §3). diff --git a/skills/confit-component-tests/references/writing-tests.md b/skills/confit-component-tests/references/writing-tests.md new file mode 100644 index 0000000..434c0b5 --- /dev/null +++ b/skills/confit-component-tests/references/writing-tests.md @@ -0,0 +1,109 @@ +# Writing Component Test Definitions + +Compact reference. For depth, read the `doc/` page named in each section — they are on disk at +`/doc/`. + +--- + +## The rule that catches most people + +**The expected body must account for every field in the actual response.** Matchers remove the +fields they claim, then whatever is left is compared structurally. A response field that is +neither listed in `body` nor claimed by a matcher fails as `expected: `. + +Two consequences: + +- **Omitting `body` fails the test.** With no expected body the diff compares the response against + nothing and reports a difference. Use `body: {}` when every field is claimed by a matcher. +- **An empty response body cannot be asserted.** A `204`, or any zero-length body, fails while + being parsed as JSON. Verify such an endpoint through a follow-up request instead. + +## Field reference + +`doc/test-file-format.md` for the full version. + +| | | +|---|---| +| `tags` | strings; matched against the filter env var. With `strategy: tags` active, an **untagged test is skipped** | +| `depends` | prerequisite test names — same file, defined earlier. Skips instead of failing when one did not pass | +| `mock.interactions` | component only — see `mock-discovery.md` | +| `api.request` | `method`, `path`, `body`, `bodyFromFile`, `override`, `params`, `headers`, `graphql` | +| `api.response` | `statusCode`, `body`, `bodyFromFile`, `override`, `headers`, `matcher`, `extract` | +| `matcher` | `ignore` (list), `pattern` (field→regex), `semantic` (field→matcher) | +| `extract` | name → `$.body.…` / `$.headers['x-…']` / `$.statusCode`. Runs only when the test passes | +| `{{var}}` | injects an extracted value into path, body, headers, params, mock bodies | +| `${ENV}` | a different namespace — process environment, not extracted data | + +Nested paths use `__`: `address__city`. A bare key name in `ignore`/`pattern` matches at **every** +depth; a `__` path is anchored to the root. + +## Choosing a matcher + +Work down; stop at the first line that applies. + +1. **Is the value deterministic?** Put the literal in `body`. In component mode state is fresh + each run, so this covers far more than people expect — assert the real `name`, `email`, `age`. +2. **Does a built-in `semantic` matcher describe it?** Use it: + `isUuid`, `isIsoDate`, `isIsoDateTime`, `isEmail`, `isNull`, `isNotNull`, `isEmpty`, + `isNotEmpty`, `greaterThan(n)`, `lessThan(n)`, `hasLength(n)`, `hasLength(min,max)`. +3. **Is the format specific to this system?** `pattern` with an anchored regex. +4. **Does the same domain assertion repeat?** Register a custom matcher in the fixture and give it + a name. That is C# in the *test* project, which the developer owns. +5. **Nothing to assert at all?** Only then `ignore`. + +`ignore` is the weakest option — an ignored field is an untested field. Use it for values that +carry no meaning to the test, not merely for values that change. + +`semantic` paths do **not** support wildcards or array indexes. `ignore` and `pattern` accept a +`*` segment to reach across array elements (`errors__*__path`), but never as the final segment. +Depth: `doc/matchers-and-patterns.md`. + +## Chaining tests + +`extract` → `{{inject}}` → `depends` go together. `extract` runs only on a passing test, so +without `depends` a dependent test fails on a missing variable and hides the real cause. All three +must be in the same file, prerequisites first — files load alphabetically, and `depends` is +file-scoped. + +## Which example shows what + +All under `/example/`. These are real, CI-verified files — read one rather than working from +a paraphrase. + +| Need | File | +|---|---| +| create → read chain, mock anchors, extract/inject/depends | `User.ComponentTests/TestCase/01-user-lifecycle.yaml` | +| 404s, and a 400 driven by a mock response | `User.ComponentTests/TestCase/02-user-errors.yaml` | +| all three matcher types side by side | `User.ComponentTests/TestCase/03-response-matchers.yaml` | +| cascading skips | `User.ComponentTests/TestCase/04-depends.yaml` | +| GraphQL query, mutation, error arrays, wildcards | `User.ComponentTests/TestCase/05-graphql.yaml` | +| OAuth2 against a stubbed token endpoint | `User.ComponentTests.AppLauncher/TestCase/03-oauth2.yaml` | +| `bodyFromFile` + `override` | `User.IntegrationTests/TestCase/04-body-fixtures.yaml` | +| array responses | `User.IntegrationTests/TestCase/05-array-responses.yaml` | +| deeply nested structures | `User.IntegrationTests/TestCase/06-nested-structures.yaml` | + +Read `/example/README.md` first — it lists what in those files is demo-specific and must not +be copied. + +--- + +## Diagnosing a failure + +**Work the mock angle first.** In component mode it is the most common cause, and it does not +announce itself — an unmatched stub surfaces as an unexpected status code, not as a mock error. +See `mock-discovery.md` §6. + +Then the field-level output (`doc/failure-output.md`): + +| Output | Meaning | Fix | +|---|---|---| +| both values shown | field present on both sides, values differ | fix the expectation, or claim the field with a matcher if it is legitimately dynamic | +| `actual: ` | expected field absent from the response | the API does not return it — correct the expected body | +| `expected: ` | response carries a field the test does not account for | add it to `body`, or claim it with the right matcher. Blanket-`ignore` last | + +Paths use dots for objects and brackets for arrays: `orders[0].lines[2].sku`. + +**Zero tests discovered** is never a ConfIT problem — the file is missing its `` / +`CopyToOutputDirectory` entry, so it does not exist in the build output. + +**Every test skipped** means a tag filter is active and the tests carry no `tags:`. diff --git a/skills/confit-component-tests/scripts/reference-path.sh b/skills/confit-component-tests/scripts/reference-path.sh new file mode 100755 index 0000000..5c6e672 --- /dev/null +++ b/skills/confit-component-tests/scripts/reference-path.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# +# Prints the absolute path of the ConfIT checkout this skill ships inside — the reference the +# skill reads (example/, doc/, src/). +# +# Skills live at /skills//, so the reference sits alongside them. Rather than +# counting "../.." levels — which breaks when the skill is reached through a symlink — this +# resolves the script's own physical location and walks up for the repository markers. +# +# bash reference-path.sh print the root, or exit 1 with guidance +# bash reference-path.sh --check also list what was found +# +# Exit 1 means the reference is missing: the plugin install is broken or partial. Do not write +# tests from memory in that case — report it. + +set -u + +if [ -t 1 ]; then RED=$'\033[31m'; OFF=$'\033[0m'; else RED=""; OFF=""; fi + +# A directory is the ConfIT root when it holds all three. Requiring skills/ too keeps this from +# matching some unrelated repository that happens to have example/ and doc/. +is_root() { + [ -d "$1/example" ] && [ -d "$1/doc" ] && [ -d "$1/skills" ] +} + +ROOT="" + +# The plugin host may name the root outright. +if [ -n "${CLAUDE_PLUGIN_ROOT:-}" ] && is_root "$CLAUDE_PLUGIN_ROOT"; then + ROOT="$(cd "$CLAUDE_PLUGIN_ROOT" && pwd -P)" +fi + +# Otherwise walk up from this script's real location. pwd -P resolves any symlink in the path. +if [ -z "$ROOT" ]; then + dir="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd -P)" + while [ "$dir" != "/" ]; do + if is_root "$dir"; then ROOT="$dir"; break; fi + dir="$(dirname "$dir")" + done +fi + +if [ -z "$ROOT" ]; then + printf '%serror%s ConfIT reference not found.\n' "$RED" "$OFF" >&2 + cat >&2 <<'GUIDANCE' + +This skill reads ConfIT's own example suites and documentation, which ship in the same +repository. Neither example/ nor doc/ could be located from the skill's install path, so the +plugin install is broken or partial. + +Reinstall the plugin, then retry: + + Claude Code /plugin marketplace add techygarg/ConfIT + /plugin install confit@confit + + Working in the ConfIT repository itself: + claude --plugin-dir . + +Do not fall back to writing tests from memory — ConfIT's DSL, matcher set and config schema +move between releases. +GUIDANCE + exit 1 +fi + +echo "$ROOT" + +if [ "${1:-}" = "--check" ]; then + { + echo + echo "reference contents:" + for d in example doc src/ConfIT tools; do + if [ -e "$ROOT/$d" ]; then echo " ok $d"; else echo " missing $d"; fi + done + echo + echo "suites available:" + find "$ROOT/example" -maxdepth 2 -name suite.config.yaml -not -path '*/bin/*' 2>/dev/null \ + | sed "s|$ROOT/| |;s|/suite.config.yaml||" + } >&2 +fi diff --git a/skills/confit-integration-tests/SKILL.md b/skills/confit-integration-tests/SKILL.md new file mode 100644 index 0000000..62cc583 --- /dev/null +++ b/skills/confit-integration-tests/SKILL.md @@ -0,0 +1,121 @@ +--- +name: confit-integration-tests +description: This skill should be used when someone asks to "write API tests for our deployed service", "here's our OpenAPI/Swagger spec, write tests", "test the staging/QA environment", "write integration tests", "convert this Postman collection to ConfIT", "add end-to-end API tests", "point the suite at a different environment", or when adding or editing test definitions in a ConfIT suite whose config has an `integration:` section. Black box: works from a spec, a collection or a live endpoint plus a token, assuming no access to the service's source code. For tests written against your own code with mocked dependencies, use confit-component-tests instead. +--- + +# ConfIT Integration Tests + +Black-box API tests against a service that is already running — a deployed environment, or one +brought up by CI. **Assume no access to the service's source, and that this test project may live +in a different repository entirely.** Nothing is mocked. + +Setting up the suite itself is a different job — use `confit-suite-setup`. + +## Step 0 — Resolve the reference, then confirm the mode + +```bash +bash /scripts/reference-path.sh +``` + +It prints the ConfIT repository root, `` — this skill's own reference material, not the +service under test. If it exits non-zero, stop: the install is broken. + +Then open the target project's `suite.config.yaml`: + +- Top-level key is **`integration:`** → continue here. Note which environments exist and which is + active (`default:`, or `TEST_ENVIRONMENT`). +- Top-level key is **`component:`** → wrong skill. That suite mocks its dependencies; hand over to + `confit-component-tests`. +- **No suite at all** → hand over to `confit-suite-setup`. + +Read one or two neighbouring files in the test-case folder to match the house style. + +## Step 1 — Establish the contract + +Any of these works. Later rows are often *better* than a spec, because they carry values someone +already proved against the real service. + +| Source | What to take from it | +|---|---| +| OpenAPI / Swagger | operations, parameters, request and response schemas, documented status codes. Schemas are shapes — invent realistic values, and prefer any `example` the document supplies | +| Postman collection | real request bodies, headers and auth that are known to work; often the fastest path to a correct test | +| `.http` / `.rest` files | the same, and usually already in a repository you can read | +| curl examples | a captured response is the most reliable expected body there is | +| live probe | when nothing else exists, call the endpoint and build the expectation from what actually comes back | + +Converting an existing Postman or `.http` suite is a first-class path: same target format, same +matchers, same grouping rules. Take the requests as given, then decide the assertions. + +**Never invent endpoints, fields or status codes the source does not show.** Ask, or probe. + +## Step 2 — Settle the environment and auth + +Before any test: which environment is this suite pointed at, and how does it authenticate? +Environments are named blocks under `integration:`, selected by the `ForIntegration` argument, +then `TEST_ENVIRONMENT`, then `default:`. Auth is a declarative `auth:` block per environment — +bearer, OAuth2 client credentials, or API key — and every secret comes from `${ENV_VAR}`. + +Two things that bite in this mode specifically: auth is **suite-level**, applied to every request +with no per-test override; and an OAuth2 token is fetched **once at startup and never refreshed**. +Details, and the auth-focused tests worth writing: `references/environments-and-auth.md`. + +## Step 3 — Propose the matrix, confirm once + +Default depth, when the user does not say otherwise: **the happy path for each operation, plus +every non-2xx the source documents.** Roughly one test per operation plus its documented errors. + +Offer deeper coverage — pagination, array shapes, boundary values, auth edge cases — as a +follow-up rather than generating it unasked. + +**Say what is not achievable here.** A dependency returning `503`, a timeout, a partial outage — +none of these can be forced black box. If the user wants them, that is a component test; hand over. + +Show the list and get one confirmation before writing files. + +## Step 4 — Write + +Field reference, matcher decision order and the example index: +`references/writing-tests.md`. + +Two instincts that are the opposite of the component mode's: + +- **You do not control the data**, so lean on `semantic` and `ignore`. Assert what the contract + guarantees — types, formats, ranges, presence — not values a shared environment happens to hold + today. Exact counts (`hasLength(3)`) are usually wrong against live data. +- **State persists between runs.** Anything the suite creates is still there next time. Make + created data unique per run and identifiable, or the second run collides with the first. + `references/state-and-data.md` has the recipe, which needs no C#. + +## Step 5 — Verify + +```bash +python3 /tools/check-testcases.py +``` + +Catches a response with no expected body, `depends:` naming an unknown or later test, +unresolvable `{{variables}}`, unknown or malformed matcher names, `mock:` blocks that do not +belong in an integration suite, and unregistered test files. + +Then run it against the environment — twice. **A suite that passes once and fails the second time +is the characteristic integration-suite bug**, and it means data is not unique per run. + +Every test file needs a `` entry with +`Always`, or it will not exist at runtime. + +## What ConfIT does not do for you + +Do not design tests around features that are not there. There are no cleanup hooks, no retry or +polling for eventual consistency, no per-request timeout configuration, no unique-data helpers, +no read-only guard, and no latency assertions. The workarounds that do work — and the ones that +do not exist at all — are in `references/state-and-data.md`. + +## Additional resources + +- **`references/environments-and-auth.md`** — environment selection and precedence, the three auth + profiles, the OAuth2 refresh caveat, `${ENV_VAR}` secrets, auth tests worth writing. +- **`references/state-and-data.md`** — shared persistent state, per-run unique data, cleanup, + the verified list of what the library does not support. +- **`references/writing-tests.md`** — DSL field table, matcher decision order, example index, + diagnosing failures. +- **`/doc/`** — `test-file-format.md`, `matchers-and-patterns.md`, `auth-profiles.md`, + `test-filtering.md` for depth. diff --git a/skills/confit-integration-tests/references/environments-and-auth.md b/skills/confit-integration-tests/references/environments-and-auth.md new file mode 100644 index 0000000..adefbd1 --- /dev/null +++ b/skills/confit-integration-tests/references/environments-and-auth.md @@ -0,0 +1,127 @@ +# Environments and Auth + +The two things a black-box suite must settle before any test is written. Depth: +`/doc/auth-profiles.md` and `/doc/suite-setup.md`. + +--- + +## Environments + +Named blocks under `integration:`, each carrying its own `api`, `auth`, `folders` and `filter`: + +```yaml +integration: + default: local + local: + api: { url: http://localhost: } + filter: { strategy: tags, envVariable: TEST_TAGS } + qa: + api: { url: ${QA_API_URL} } + auth: { type: bearer, token: ${QA_API_TOKEN} } + filter: { strategy: tags, envVariable: TEST_TAGS } +``` + +**Selection order**, highest first: + +1. the `environment` argument to `SuiteBootstrapper.ForIntegration(configFile, environment)` +2. the `TEST_ENVIRONMENT` process variable +3. the `default:` key +4. none of the above → the suite fails to load + +```bash +TEST_ENVIRONMENT=qa TEST_TAGS=smoke dotnet test +``` + +An unknown name fails loudly (`No environment 'staging' found in the integration section`), and a +hardcoded `environment:` argument in the fixture beats `TEST_ENVIRONMENT`, which makes CI unable +to switch — prefer leaving it unset. + +Only the **active** environment's block is parsed. A `${VAR}` in an inactive block is never +resolved, so unused environments cannot break a run. + +## Auth profiles + +| `type` | Required | Optional | +|---|---|---| +| `bearer` | `token` | `headerKey` (default `Authorization`) | +| `oauth2-client-credentials` | `tokenUrl`, `clientId`, `clientSecret` | `scope`, `headerKey` | +| `api-key` | `headerKey`, `value` | — | + +ConfIT adds the `Bearer ` prefix for bearer and OAuth2; do not include it in the value. + +### Two constraints that shape what you can test + +**Auth is suite-level.** The provider is asked for a token before every request, and there is no +per-test override in the DSL. + +**A per-test `Authorization` header does not replace it — both are sent.** Headers declared on a +test are added first, then the provider's header is appended to the same header's value list: + +``` +Authorization: Bearer per-test-bad-token, Bearer suite-level-token +``` + +So **you cannot write a 401 test by putting a bad token on one test** while the suite has auth +configured. The server sees two credentials and will behave unpredictably. To test rejection +paths, use an environment block with **no `auth:`**, and run those tests against it: + +```yaml + qa-noauth: + api: { url: ${QA_API_URL} } + filter: { strategy: tags, envVariable: TEST_TAGS } +``` + +```bash +TEST_ENVIRONMENT=qa-noauth TEST_TAGS=auth dotnet test +``` + +Now a test in that run genuinely sends no credential, or exactly the one it declares. + +### OAuth2: fetched once, never refreshed + +The token request is a single blocking `POST` in the **provider's constructor** — at suite +startup, before the first test. The `access_token` is cached for the entire run. + +- **There is no refresh and no expiry handling.** A suite that runs longer than the token's TTL + starts returning 401s partway through, and ConfIT will not recover. For a long QA suite, either + keep the run short, or supply a custom `IAuthTokenProvider` that refreshes. +- **A token failure fails the whole suite, not one test.** An unreachable endpoint, a non-2xx + response, or a missing `access_token` field throws before any test runs, naming the URL and + status. + +### Anything else + +Request signing, rotating credentials, per-tenant tokens: implement `IAuthTokenProvider` +(`HeaderKey()` + `Token()`) and wire it through the adapter-chain setup. `Token()` is called per +request, so it can vary. See `/doc/extending-confit.md`. + +## Auth tests worth writing + +Given the constraint above, split them by run: + +| Test | Where | +|---|---| +| happy path carries a valid credential | the normal authenticated run — implicit in every test | +| no credential → `401` | an environment block with no `auth:` | +| malformed or expired credential → `401` | same, with the bad token declared on the test | +| valid credential, insufficient scope or role → `403` | an environment block whose `auth:` uses the lesser-privileged credential | +| wrong tenant or another user's resource → `403`/`404` | the normal run, requesting a resource the credential should not reach | + +The last one is the highest-value and most often missed: it needs no special environment, only a +resource identifier the token should not be able to see. + +## Secrets + +Every credential comes from `${ENV_VAR}` — never a literal in a committed file. + +Two interpolation mechanisms exist, and they behave differently: + +| | In `suite.config.yaml` | In test definitions | +|---|---|---| +| Applies to | every string in the active section | `path`, `params`, `headers`, `body`, expected bodies | +| Name pattern | uppercase and underscore only | permissive | +| Fails | at suite load, naming the field | when that test runs | + +Both fail loudly; neither substitutes an empty string. `${ENV}` and `{{extracted}}` are separate +namespaces — the first is the process environment at load, the second is data captured from an +earlier response. diff --git a/skills/confit-integration-tests/references/state-and-data.md b/skills/confit-integration-tests/references/state-and-data.md new file mode 100644 index 0000000..04388ed --- /dev/null +++ b/skills/confit-integration-tests/references/state-and-data.md @@ -0,0 +1,141 @@ +# State and Test Data + +The thing that breaks black-box suites. A component suite gets a fresh process and an empty +database every run; a deployed environment remembers everything the last run did. + +**The characteristic bug: the suite passes the first time and fails the second.** A `CreateUser` +with a hardcoded email succeeds on run 1 and collides on a uniqueness constraint on run 2 — and +the failure looks like a bug in the service, not in the test. + +--- + +## Make created data unique per run — no C# required + +`${ENV_VAR}` interpolation resolves inside test definitions, not just in `suite.config.yaml`. Put +a run-scoped value into any created identifier: + +```yaml +CreateOrder: + tags: [orders, smoke] + api: + request: + method: POST + path: /api/order + body: + reference: "qa-${RUN_ID}-001" + email: "qa-${RUN_ID}@example.com" + response: + statusCode: 201 + extract: + orderId: $.body.id + matcher: + semantic: + id: isNotNull +``` + +Set `RUN_ID` from whatever the runner already has: + +```bash +RUN_ID=$GITHUB_RUN_ID dotnet test # CI +RUN_ID=$(date +%s) dotnet test # locally +``` + +An unset variable fails loudly at the point the test runs, so a forgotten export never silently +degrades into a collision. + +This also makes the data **identifiable**: everything the suite created is greppable by run, which +matters when someone has to clean up a shared environment by hand later. + +## Cleanup — a trailing test + +There are no teardown hooks. What works instead: tests in a file run in definition order, and a +test with **no `depends:`** runs regardless of whether earlier tests failed. So a final delete is +a legitimate declarative teardown. + +```yaml +# ... create / read / update tests above ... + +DeleteOrder_Cleanup: + tags: [orders, cleanup] + depends: CreateOrder + api: + request: + method: DELETE + path: "/api/order/{{orderId}}" + response: + statusCode: 200 + body: { deleted: true } +``` + +Caveats worth stating to the user rather than hiding: + +- It needs `{{orderId}}`, which only exists if the create passed — hence `depends: CreateOrder`. + Without that dependency, a failed create leaves `{{orderId}}` unresolved and the cleanup test + fails on variable injection instead of skipping, adding a second failure that obscures the + first. With it, a failed create means nothing was created, so cleanup skips and nothing leaks. +- If the create passed but an assertion in the middle failed, this still runs. That is the point. +- If the run is killed outright, it does not run. Unique data is what limits the damage. + +Where the environment offers a bulk cleanup endpoint, a single trailing call scoped to +`${RUN_ID}` beats one delete per entity. + +## Read-only and shared environments + +ConfIT has no read-only mode: `POST` and `DELETE` behave identically in every environment. If a +suite must be safe against a production-like target, that is a discipline in the test definitions +— tag write tests separately and run only reads there: + +```bash +TEST_TAGS=readonly dotnet test +``` + +Make that split explicit when proposing the matrix; do not assume a QA URL is safe to write to. + +## Assertions against data you do not control + +Prefer what the contract guarantees over what the environment happens to hold: + +| Instead of | Use | +|---|---| +| `hasLength(3)` on a collection | `isNotEmpty`, or a `greaterThan` bound | +| an exact `id` | `isUuid` / `isNotNull` / `greaterThan(0)` | +| an exact `createdAt` | `isIsoDateTime` | +| an exact total or count | `greaterThan(0)` | +| a full list body | assert the shape of one known item you created this run | + +The best expected body in this mode is one built from a **response you actually captured**, then +loosened where values are volatile. + +--- + +## What ConfIT does not support + +Verified against the library. Do not design a suite around any of these. + +| Capability | Supported | What to do instead | +|---|---|---| +| cleanup / teardown hooks | **no** | trailing delete test, above | +| retry / polling for eventual consistency | **no** | no honest workaround — say so; do not add sleeps to the DSL, there is nowhere to put them | +| per-request timeout config | **no** | inherits `HttpClient`'s 100-second default | +| unique / random data generation | **no** | `${RUN_ID}`, above | +| read-only guard | **no** | tag-based separation, above | +| latency assertions | **no** | durations appear in the summary but cannot be asserted | +| parallel execution controls | **no** | only "sequential within a file" is guaranteed | + +Two traps: + +- **`timeoutSeconds` is not a request timeout.** It exists only under `startup.readiness`, for + AppLauncher process startup — a component-mode concern that does not apply here at all. +- **`ITestProcessor.After` is not a `finally`.** It runs before assertions, and is skipped + entirely when the request throws or the body fails to parse, so it cannot be repurposed as + reliable cleanup. The interface is documented as legacy and may be removed. + +## The shipped example is not a template for this + +`/example/User.IntegrationTests` is worth reading for DSL mechanics — matchers, `depends`, +`bodyFromFile`, arrays, GraphQL. **Do not copy its data strategy.** It uses hardcoded emails +(`test@test.com`) and relies on cross-file ordering, which only works because `make integration` +runs `rm -f` on the database before every run. It is a freshly-provisioned-environment suite +wearing the "integration" label. + +Against a deployed environment, that same file fails on its second run. diff --git a/skills/confit-integration-tests/references/writing-tests.md b/skills/confit-integration-tests/references/writing-tests.md new file mode 100644 index 0000000..000a7dd --- /dev/null +++ b/skills/confit-integration-tests/references/writing-tests.md @@ -0,0 +1,124 @@ +# Writing Integration Test Definitions + +Compact reference. For depth, read the `doc/` page named in each section — they are on disk at +`/doc/`. + +--- + +## The rule that catches most people + +**The expected body must account for every field in the actual response.** Matchers remove the +fields they claim, then whatever is left is compared structurally. A response field that is +neither listed in `body` nor claimed by a matcher fails as `expected: `. + +This bites harder in black-box mode than anywhere else: a deployed service often returns fields +the spec never mentioned. When the spec and the response disagree, **the response wins** — correct +the test against reality, and tell the user the spec is out of date. + +Two consequences: + +- **Omitting `body` fails the test.** Use `body: {}` when every field is claimed by a matcher. +- **An empty response body cannot be asserted.** A `204`, or any zero-length body, fails while + being parsed as JSON. Verify such an endpoint through a follow-up request instead. + +## Field reference + +`doc/test-file-format.md` for the full version. + +| | | +|---|---| +| `tags` | strings; matched against the filter env var. With `strategy: tags` active, an **untagged test is skipped**. Also how you separate read-only from write tests | +| `depends` | prerequisite test names — same file, defined earlier. Skips instead of failing when one did not pass | +| `mock` | **not valid here** — an integration suite has no mock server | +| `api.request` | `method`, `path`, `body`, `bodyFromFile`, `override`, `params`, `headers`, `graphql` | +| `api.response` | `statusCode`, `body`, `bodyFromFile`, `override`, `headers`, `matcher`, `extract` | +| `matcher` | `ignore` (list), `pattern` (field→regex), `semantic` (field→matcher) | +| `extract` | name → `$.body.…` / `$.headers['x-…']` / `$.statusCode`. Runs only when the test passes | +| `{{var}}` | injects an extracted value into path, body, headers, params | +| `${ENV}` | process environment — secrets, and run-scoped values like `${RUN_ID}` | + +Nested paths use `__`: `address__city`. A bare key name in `ignore`/`pattern` matches at **every** +depth; a `__` path is anchored to the root. + +`bodyFromFile` + `override` is worth reaching for here more than in component mode: response +bodies from a real service are large, and one fixture with per-test overrides beats repeating it. + +## Choosing a matcher + +The instinct is the opposite of a component suite's. **You do not control the data**, so assert +what the contract guarantees rather than what the environment holds today. + +1. **Did this suite create the value, this run?** Then it is deterministic and you may assert the + literal — that is what `${RUN_ID}`-scoped data buys you. +2. **Otherwise, does a built-in `semantic` matcher describe it?** This is the default here: + `isUuid`, `isIsoDate`, `isIsoDateTime`, `isEmail`, `isNull`, `isNotNull`, `isEmpty`, + `isNotEmpty`, `greaterThan(n)`, `lessThan(n)`, `hasLength(n)`, `hasLength(min,max)`. +3. **A system-specific format?** `pattern` with an anchored regex. +4. **A repeated domain assertion?** Register a custom matcher in the fixture. That is C# in the + *test* project — which you own even with no access to the service's source. +5. **Nothing to assert?** Only then `ignore`. + +Avoid exact counts on collections you did not create — `hasLength(3)` is a promise about someone +else's data. Use `isNotEmpty`, or assert the shape of one item you created this run. + +`semantic` paths do **not** support wildcards or array indexes. `ignore` and `pattern` accept a +`*` segment to reach across array elements (`errors__*__path`), but never as the final segment. +Depth: `doc/matchers-and-patterns.md`. + +## Chaining tests + +`extract` → `{{inject}}` → `depends` go together. `extract` runs only on a passing test, so +without `depends` a dependent test fails on a missing variable and hides the real cause. All three +must be in the same file, prerequisites first — files load alphabetically, and `depends` is +file-scoped. + +In this mode the chain usually starts with a create that owns its own `${RUN_ID}`-scoped data, so +the rest of the file operates on something the run definitely owns. + +## Which example shows what + +All under `/example/`. Real, CI-verified files — read one rather than working from a +paraphrase. Read `/example/README.md` first for what is demo-specific, and see +`state-and-data.md` for why the data strategy in these files does not transfer to a deployed +environment. + +| Need | File | +|---|---| +| create → read chain with `extract` / `{{inject}}` / `depends` | `User.IntegrationTests/TestCase/01-user-lifecycle.yaml` | +| error responses | `User.IntegrationTests/TestCase/02-user-errors.yaml` | +| all three matcher types side by side | `User.IntegrationTests/TestCase/03-response-matchers.yaml` | +| `bodyFromFile` + `override` | `User.IntegrationTests/TestCase/04-body-fixtures.yaml` | +| array responses | `User.IntegrationTests/TestCase/05-array-responses.yaml` | +| deeply nested structures | `User.IntegrationTests/TestCase/06-nested-structures.yaml` | +| cascading skips | `User.IntegrationTests/TestCase/07-depends.yaml` | +| GraphQL query, mutation, error arrays | `User.IntegrationTests/TestCase/08-graphql.yaml` | +| multi-environment config with per-environment auth | `User.IntegrationTests/suite.config.yaml` | + +--- + +## Diagnosing a failure + +**Work the data and environment angle first** — in this mode that is the usual cause, not the +assertions. + +| Symptom | Likely cause | +|---|---| +| passes once, fails on the next run | created data is not unique per run — see `state-and-data.md` | +| `409`/`400` on a create that used to work | leftover data from an earlier run | +| everything returns `401` partway through a long run | the OAuth2 token expired; it is fetched once and never refreshed | +| everything returns `401` from the first test | auth block is on an inactive environment, or the secret is unset | +| passes locally, fails in CI | a different `TEST_ENVIRONMENT`, or an unexported `${VAR}` | +| a field the spec documents is missing | the spec is out of date — trust the response | + +Then the field-level output (`doc/failure-output.md`): + +| Output | Meaning | Fix | +|---|---|---| +| both values shown | values differ | fix the expectation, or claim the field with a matcher if it is environment-dependent | +| `actual: ` | expected field absent from the response | the service does not return it here — correct the expected body | +| `expected: ` | response carries a field the test does not account for | add it, or claim it with a matcher | + +Paths use dots for objects and brackets for arrays: `orders[0].lines[2].sku`. + +**Zero tests discovered** is never a ConfIT problem — the file is missing its `` / +`CopyToOutputDirectory` entry, so it does not exist in the build output. diff --git a/skills/confit-integration-tests/scripts/reference-path.sh b/skills/confit-integration-tests/scripts/reference-path.sh new file mode 100755 index 0000000..5c6e672 --- /dev/null +++ b/skills/confit-integration-tests/scripts/reference-path.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# +# Prints the absolute path of the ConfIT checkout this skill ships inside — the reference the +# skill reads (example/, doc/, src/). +# +# Skills live at /skills//, so the reference sits alongside them. Rather than +# counting "../.." levels — which breaks when the skill is reached through a symlink — this +# resolves the script's own physical location and walks up for the repository markers. +# +# bash reference-path.sh print the root, or exit 1 with guidance +# bash reference-path.sh --check also list what was found +# +# Exit 1 means the reference is missing: the plugin install is broken or partial. Do not write +# tests from memory in that case — report it. + +set -u + +if [ -t 1 ]; then RED=$'\033[31m'; OFF=$'\033[0m'; else RED=""; OFF=""; fi + +# A directory is the ConfIT root when it holds all three. Requiring skills/ too keeps this from +# matching some unrelated repository that happens to have example/ and doc/. +is_root() { + [ -d "$1/example" ] && [ -d "$1/doc" ] && [ -d "$1/skills" ] +} + +ROOT="" + +# The plugin host may name the root outright. +if [ -n "${CLAUDE_PLUGIN_ROOT:-}" ] && is_root "$CLAUDE_PLUGIN_ROOT"; then + ROOT="$(cd "$CLAUDE_PLUGIN_ROOT" && pwd -P)" +fi + +# Otherwise walk up from this script's real location. pwd -P resolves any symlink in the path. +if [ -z "$ROOT" ]; then + dir="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd -P)" + while [ "$dir" != "/" ]; do + if is_root "$dir"; then ROOT="$dir"; break; fi + dir="$(dirname "$dir")" + done +fi + +if [ -z "$ROOT" ]; then + printf '%serror%s ConfIT reference not found.\n' "$RED" "$OFF" >&2 + cat >&2 <<'GUIDANCE' + +This skill reads ConfIT's own example suites and documentation, which ship in the same +repository. Neither example/ nor doc/ could be located from the skill's install path, so the +plugin install is broken or partial. + +Reinstall the plugin, then retry: + + Claude Code /plugin marketplace add techygarg/ConfIT + /plugin install confit@confit + + Working in the ConfIT repository itself: + claude --plugin-dir . + +Do not fall back to writing tests from memory — ConfIT's DSL, matcher set and config schema +move between releases. +GUIDANCE + exit 1 +fi + +echo "$ROOT" + +if [ "${1:-}" = "--check" ]; then + { + echo + echo "reference contents:" + for d in example doc src/ConfIT tools; do + if [ -e "$ROOT/$d" ]; then echo " ok $d"; else echo " missing $d"; fi + done + echo + echo "suites available:" + find "$ROOT/example" -maxdepth 2 -name suite.config.yaml -not -path '*/bin/*' 2>/dev/null \ + | sed "s|$ROOT/| |;s|/suite.config.yaml||" + } >&2 +fi diff --git a/skills/confit-suite-setup/SKILL.md b/skills/confit-suite-setup/SKILL.md new file mode 100644 index 0000000..122951e --- /dev/null +++ b/skills/confit-suite-setup/SKILL.md @@ -0,0 +1,168 @@ +--- +name: confit-suite-setup +description: This skill should be used when the user asks to "set up ConfIT", "add ConfIT to my project", "create a component test suite", "scaffold integration tests", "wire up suite.config.yaml", "add a ConfIT test project", "configure the ConfIT fixture", "run my API tests against QA", or when a ConfIT suite fails to start — no tests discovered, config not found, mock server not reachable, auth not applied. Routes to ConfIT's own working reference suites for the chosen startup mode (in-process, command/AppLauncher, integration), then adapts them to the target project. +--- + +# ConfIT Suite Setup + +ConfIT ships three working test suites — one per startup mode — that run in its own CI. This +skill reads the right one and adapts it, rather than carrying templates that drift out of date. + +Adding *tests* to an existing suite is a different job: `confit-component-tests` when the +developer works from their own controller and mocks, `confit-integration-tests` when someone +writes black-box tests against a deployed service. + +## Step 0 — Locate the reference, always, before anything else + +This skill ships inside the ConfIT repository, so its example suites and documentation are on +disk next to it. Resolve the root first — do not guess the path or count `../..` levels: + +```bash +bash /scripts/reference-path.sh +``` + +It prints the repository root. Everything this skill refers to is relative to it: + +``` +/example/ three working suites, one per mode, all verified by CI +/doc/ the prose documentation +``` + +Then read **`/example/README.md`** — it maps modes to projects and lists what is +demo-specific. + +**If the script exits non-zero, stop and say so.** It means the plugin install is broken or +partial. Ask the user to reinstall it. Do not reconstruct a suite from memory: the fixture shape, +config schema and package versions all move between releases, and a plausible invention fails in +ways that are slow to debug. No reference, no setup. + +## Step 1 — Choose the mode + +Two questions settle it: + +1. **Is the service already running** — locally, in CI, or in a deployed environment — and should + the tests hit it as-is? → **integration** (`SuiteBootstrapper.ForIntegration`). Nothing is + mocked; environments are selected from config at runtime. +2. Otherwise the tests start the service. **Is it .NET, and may the test project reference it?** + - Yes → **in-process** (`SuiteBootstrapper.ForComponent`). Fastest start, and the + DI container is reachable for schema setup and seeding. + - No — another language, or a .NET app you want exercised as a real process → **command / + AppLauncher** (`SuiteBootstrapper.ForCommand`). ConfIT runs a shell command, waits for a + readiness probe, and speaks only HTTP. + +Both component modes can stub outbound dependencies with WireMock. Integration suites cannot — +a `mock:` block there is an error. + +When the user wants both a fast inner loop and a real-environment check, that is two projects, +not one suite. The test definitions are largely portable between them minus the `mock:` blocks. + +## Step 2 — Read the reference for that mode + +| Mode | Read this project | And this doc | +|---|---|---| +| in-process | `/example/User.ComponentTests/` | `/doc/suite-setup.md` | +| command | `/example/User.ComponentTests.AppLauncher/` | `/doc/app-launcher.md` | +| integration | `/example/User.IntegrationTests/` | `/doc/suite-setup.md` | + +Four files carry the wiring: `suite.config.yaml`, the fixture, the test class, and the `.csproj`. +Read those. Skip `TestCase/` unless also writing tests. + +Add `/doc/auth-profiles.md` when the API needs auth, and `/doc/test-filtering.md` when +the suite needs tag or name filtering. + +## Step 3 — Create the project + +```bash +dotnet new xunit -n .ComponentTests +cd .ComponentTests +dotnet add package ConfIT +dotnet add reference ../.Api/.Api.csproj # in-process mode only +``` + +**Take versions from the CLI, never from the reference.** The example projects build ConfIT from +source via `ProjectReference`, so they pin nothing a consumer should copy; `dotnet add package` +resolves what is current. The same goes for the target framework — match what `dotnet new` +produced, as long as ConfIT supports it (`/src/ConfIT/ConfIT.csproj` lists its +`TargetFrameworks`). + +## Step 4 — Author the files + +Write fresh files modelled on the reference. Do not copy-and-rename: the examples carry demo +content that will not make sense in another project. + +**Structural — every suite needs these:** + +- `suite.config.yaml` in the project root, its top-level key (`component:` / `integration:`) + matching the fixture's bootstrapper call. +- A fixture implementing `IDisposable` that calls one `SuiteBootstrapper.For*`, exposes + `TestSuiteContext Context`, and disposes the suite — disposal prints the summary and shuts + infrastructure down. +- A test class deriving from `BaseTest` with `IClassFixture`, and a `[Theory]` + fed by `[MemberData]` → `TestReader.GetTestsForAFolder`. That is ConfIT's only discovery + mechanism. +- A `` entry with `Always` for every + file read at runtime — test definitions, `suite.config.yaml`, body fixtures, app settings. + `TestReader` reads from the build output directory; a missing entry is the usual cause of + "no tests discovered". +- A `TestCase/` folder with at least one test definition. + +**Optional, despite appearing in the reference:** + +- `TestOutputLogger` — a three-line adapter onto xUnit's `ITestOutputHelper`. `BaseTest`'s logger + parameter is nullable; the AppLauncher example passes `null`. +- `appsettings.Tests.json` — needed only for `mode: in-process`, where `startup.settings` names + it. Neither other mode has one. +- The `onStarted` and `configureServices` hooks on `ForComponent` — for seeding and DI overrides. + Omit them when the app configures itself. + +**Demo-specific — leave behind:** the `User.*` namespaces, `UserDbInitializer`, +`JustAnotherService`, every port number, the `IsLocalComponentTests` flag, the EF Core InMemory +package, and everything under `TestCase/`. `/example/README.md` has the full list with +reasons. + +Two things in the reference that are available but *not* demonstrated, so do not present them as +worked examples: the `protected virtual` hooks on `User.Api.Startup`, which nothing overrides, and +`IAuthTokenProvider`, which no `SuiteBootstrapper` overload accepts — declarative auth goes in +`suite.config.yaml`. + +## Step 5 — Verify + +```bash +bash /tools/verify-suite.sh +``` + +It checks the wiring faults that produce confusing runtime symptoms: missing package reference, +unregistered config or test files, a fixture that does not match the config section, a hardcoded +filter env var. Fix what it reports before interpreting a test failure. + +Then add one trivial test definition hitting a health or list endpoint and run the suite. It is +wired correctly when the run prints ConfIT's summary table with one passing test. Zero tests +discovered means files are missing from the output directory — a `.csproj` registration problem, +not a ConfIT problem. + +In a repository that wraps builds behind `make` or a CI target, use that target rather than +calling `dotnet test` directly; the wrapper usually handles service lifecycle and database +resets. + +## Non-.NET services + +Command mode is the reason ConfIT can test a Go, Node, Python or Java service. The test project +still needs to be a .NET xUnit project — it references ConfIT only, never the application — and +`startup.command` is any shell command that starts the service. `/doc/app-launcher.md` is the +guide, and `/example/User.ComponentTests.AppLauncher/` is the working proof: it has **no** +project reference to the app under test. + +The application owns its own test environment in this mode: selecting a test database, seeding +itself, and pointing its dependencies at the mock URL, driven by whatever environment the command +sets. + +## Additional resources + +- **`references/troubleshooting.md`** — symptom → cause → fix for suites that will not start, + mocks that do not match, auth that is not applied, and tests that pass alone but fail together. +- **`scripts/reference-path.sh`** — resolves and validates the reference root; `--check` also + lists the suites it found. +- **`/tools/verify-suite.sh`** — wiring diagnostics for an existing suite. +- **`/example/README.md`** — mode-to-project map and the demo-specific list. +- **`/doc/`** — the full documentation set, including `extending-confit.md` for custom + matchers and processor hooks. diff --git a/skills/confit-suite-setup/references/troubleshooting.md b/skills/confit-suite-setup/references/troubleshooting.md new file mode 100644 index 0000000..e4bbed2 --- /dev/null +++ b/skills/confit-suite-setup/references/troubleshooting.md @@ -0,0 +1,167 @@ +# Suite Troubleshooting + +Symptom → cause → fix, for failures that happen *around* the tests rather than inside them. For a +test that runs and fails its assertions, read the field-level output instead. + +Working configurations for every case below live in `/example/`; the prose reference is +`/doc/`. + +--- + +## Discovery and startup + +### The run reports zero tests + +`TestReader` reads from the **build output directory**, not the source tree. A test file with no +`.csproj` entry does not exist at runtime. + +```xml + + Always + +``` + +Confirm what actually shipped by listing the `TestCase` folder under the project's build output. +The same applies to `suite.config.yaml`, the in-process settings file, and every `bodyFromFile` +fixture. `PreserveNewest` is fine until an edit lands with an older timestamp — prefer `Always` +for test data. + +### `Suite config file not found` + +Same cause, for the config file. It is read by filename relative to the working directory, which +is the output directory during a test run. + +### Every test is skipped + +A tag filter is active and the tests carry no `tags:`. With `strategy: tags`, an untagged test is +skipped whenever the filter's env var is set. Either tag the tests, or unset the variable. + +Check that no fixture calls `Environment.SetEnvironmentVariable` for the filter variable — a +hardcoded filter in fixture code hides tests from CI while the suite still reports green. + +### The app never becomes ready (command mode) + +`AppLauncher` throws with the elapsed timeout, the command it ran, and the readiness target it +was polling. Read those three values first, then work down: + +1. The command fails immediately — run it by hand **from the test output directory**, since + relative paths in `startup.command` resolve from there. +2. The app listens on a different port or path than the probe checks. +3. First run includes a build. Raise `timeoutSeconds`, or pre-build and pass a no-build flag. +4. The app is waiting on a dependency that is not up yet. + +The probe checks on each interval whether the process already exited, and reports the exit code +and captured output when it has — read that before raising the timeout. + +### Port already in use + +A previous run's process survived. Add a `stopCommand` that kills by port; `/doc/app-launcher.md` +has platform-specific forms, and `/example/User.ComponentTests.AppLauncher/suite.config.yaml` +shows one in place. `AppLauncher` verifies the port is free before starting, so this surfaces at +startup rather than mid-run. + +### The suite summary never prints + +The fixture is not disposing `BootstrappedSuite`. `Dispose()` prints the summary and then shuts +infrastructure down — a fixture that swallows it loses both. + +--- + +## Mocking + +### Outbound calls reach the real dependency + +The application, not ConfIT, decides where its dependencies point. Its test configuration must +aim the dependency's base URL at the `mock.url` from `suite.config.yaml`. In in-process mode that +is the settings file named by `startup.settings`; in command mode the launched app reads its own +configuration, usually selected by an environment variable the command sets. + +### A test fails with an unexpected status and the mock looks correct + +WireMock returns `404` for any outbound request matching no stub, and the service then fails in +its own way. Set `EnableMockServerLogs` on `SuiteConfig` to see what actually arrived, then +compare against the interaction: method, path, and **every** declared query parameter and header +must match exactly; body matching is structural. An interaction that over-specifies — a header +the client does not send — never matches. + +### Mock interactions are ignored entirely + +`mock.url` is missing from `suite.config.yaml`, so no mock server was created. In an integration +suite this is by design — remove the `mock:` blocks from the test definitions. + +--- + +## Auth + +### The `Authorization` header is missing + +The `auth:` block must sit inside the section actually in use — `component:`, or the *active* +integration environment. An `auth:` block under an inactive environment does nothing. + +### OAuth2 fails at startup + +The token request happens once, eagerly, when the provider is constructed. When the token +endpoint is itself a stub started by the fixture, that server must be running **before** +`SuiteBootstrapper` is called. `/example/User.ComponentTests.AppLauncher/SetUp/TestSuiteFixture.cs` +shows the ordering, and `/doc/auth-profiles.md` explains it. + +### `Unresolved env var` + +Export it before the run. The failure is deliberate — a missing secret fails loudly at load +rather than producing 401s test by test. + +--- + +## State and ordering + +### Tests pass alone and fail in a full run + +Shared state. Files load alphabetically and share one variable store, so a test that depends on +data created earlier is coupled to that ordering. Two fixes, in order of preference: + +1. Make the test self-sufficient — create what it needs in its own file, chained with + `extract` / `{{inject}}` / `depends`. +2. Move it into the file that creates the state, positioned after its prerequisite. + +`depends:` cannot cross files, so ordering alone gives no protection when a prerequisite fails. + +### `UndefinedVariableException` for a variable that clearly exists + +The producing test did not pass — `extract` runs only on success — or it runs later in the +alphabetical file order. Add `depends:` on the producer so the dependent skips with a readable +reason instead of failing on the missing variable. + +### `AmbiguousVariableException` + +Two tests extracted the same short name. Reference it with the full prefix, +`{{TestName.varName}}`, or rename one of them. + +--- + +## Environments + +### The wrong environment is used + +Resolution order: the `environment:` argument to `ForIntegration`, then `TEST_ENVIRONMENT`, then +the `default:` key in the YAML. An argument hardcoded in the fixture wins over the environment +variable and makes CI unable to switch — prefer leaving it unset. + +--- + +## Building and running + +### The suite builds locally but not in CI + +Check the target framework against the `TargetFrameworks` in `/src/ConfIT/ConfIT.csproj`. +Also confirm every file the suite reads is committed *and* registered — a locally present file +that was never added to git produces "no tests discovered" only on the CI machine. + +### Where to look first + +```bash +bash /tools/verify-suite.sh +``` + +It reports package references, config presence and registration, fixture and test-class shape, +hardcoded filters, and framework mismatch — before any of them turn into a confusing test +failure. diff --git a/skills/confit-suite-setup/scripts/reference-path.sh b/skills/confit-suite-setup/scripts/reference-path.sh new file mode 100755 index 0000000..5c6e672 --- /dev/null +++ b/skills/confit-suite-setup/scripts/reference-path.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# +# Prints the absolute path of the ConfIT checkout this skill ships inside — the reference the +# skill reads (example/, doc/, src/). +# +# Skills live at /skills//, so the reference sits alongside them. Rather than +# counting "../.." levels — which breaks when the skill is reached through a symlink — this +# resolves the script's own physical location and walks up for the repository markers. +# +# bash reference-path.sh print the root, or exit 1 with guidance +# bash reference-path.sh --check also list what was found +# +# Exit 1 means the reference is missing: the plugin install is broken or partial. Do not write +# tests from memory in that case — report it. + +set -u + +if [ -t 1 ]; then RED=$'\033[31m'; OFF=$'\033[0m'; else RED=""; OFF=""; fi + +# A directory is the ConfIT root when it holds all three. Requiring skills/ too keeps this from +# matching some unrelated repository that happens to have example/ and doc/. +is_root() { + [ -d "$1/example" ] && [ -d "$1/doc" ] && [ -d "$1/skills" ] +} + +ROOT="" + +# The plugin host may name the root outright. +if [ -n "${CLAUDE_PLUGIN_ROOT:-}" ] && is_root "$CLAUDE_PLUGIN_ROOT"; then + ROOT="$(cd "$CLAUDE_PLUGIN_ROOT" && pwd -P)" +fi + +# Otherwise walk up from this script's real location. pwd -P resolves any symlink in the path. +if [ -z "$ROOT" ]; then + dir="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd -P)" + while [ "$dir" != "/" ]; do + if is_root "$dir"; then ROOT="$dir"; break; fi + dir="$(dirname "$dir")" + done +fi + +if [ -z "$ROOT" ]; then + printf '%serror%s ConfIT reference not found.\n' "$RED" "$OFF" >&2 + cat >&2 <<'GUIDANCE' + +This skill reads ConfIT's own example suites and documentation, which ship in the same +repository. Neither example/ nor doc/ could be located from the skill's install path, so the +plugin install is broken or partial. + +Reinstall the plugin, then retry: + + Claude Code /plugin marketplace add techygarg/ConfIT + /plugin install confit@confit + + Working in the ConfIT repository itself: + claude --plugin-dir . + +Do not fall back to writing tests from memory — ConfIT's DSL, matcher set and config schema +move between releases. +GUIDANCE + exit 1 +fi + +echo "$ROOT" + +if [ "${1:-}" = "--check" ]; then + { + echo + echo "reference contents:" + for d in example doc src/ConfIT tools; do + if [ -e "$ROOT/$d" ]; then echo " ok $d"; else echo " missing $d"; fi + done + echo + echo "suites available:" + find "$ROOT/example" -maxdepth 2 -name suite.config.yaml -not -path '*/bin/*' 2>/dev/null \ + | sed "s|$ROOT/| |;s|/suite.config.yaml||" + } >&2 +fi diff --git a/src/ConfIT/Config/MockConfig.cs b/src/ConfIT/Config/MockConfig.cs index 903233d..d09f3f5 100644 --- a/src/ConfIT/Config/MockConfig.cs +++ b/src/ConfIT/Config/MockConfig.cs @@ -3,4 +3,11 @@ namespace ConfIT.Config; public sealed class MockConfig { public string? Url { get; set; } + + /// + /// Print every request WireMock receives, and whether it matched a stub. + /// Switch on to discover what the service under test actually calls: run a test with no + /// mock: block and read the unmatched requests. + /// + public bool EnableLogs { get; set; } } diff --git a/src/ConfIT/Extension/SuiteConfigurationExtensions.cs b/src/ConfIT/Extension/SuiteConfigurationExtensions.cs index 63f4fb4..1962035 100644 --- a/src/ConfIT/Extension/SuiteConfigurationExtensions.cs +++ b/src/ConfIT/Extension/SuiteConfigurationExtensions.cs @@ -13,6 +13,7 @@ public static SuiteConfig ToSuiteConfig(this ComponentConfig config) { ApiServerUrl = config.Api.Url ?? string.Empty, MockServerUrl = config.Mock?.Url ?? string.Empty, + EnableMockServerLogs = config.Mock?.EnableLogs ?? false, ApiResponseFolder = config.Folders?.Response ?? string.Empty, RequestBodyFolder = config.Folders?.RequestBody ?? string.Empty, ResponseBodyFolder = config.Folders?.ResponseBody ?? string.Empty diff --git a/src/ConfIT/Runner/Http/TestSuiteInitializer.cs b/src/ConfIT/Runner/Http/TestSuiteInitializer.cs index 77a3b93..3651f0a 100644 --- a/src/ConfIT/Runner/Http/TestSuiteInitializer.cs +++ b/src/ConfIT/Runner/Http/TestSuiteInitializer.cs @@ -1,4 +1,5 @@ using System.IO; +using ConfIT.Contract; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.AspNetCore.TestHost; @@ -12,13 +13,16 @@ public class TestSuiteInitializer : IDisposable where TProgram : class { private readonly InternalFactory _factory; - public TestSuiteInitializer(string settingsFile, Action? configureServices = null) + public TestSuiteInitializer( + string settingsFile, + Action? configureServices = null, + IAuthTokenProvider? authTokenProvider = null) { if (string.IsNullOrWhiteSpace(settingsFile)) throw new ArgumentException("Please provide app settings file name", nameof(settingsFile)); _factory = new InternalFactory(settingsFile, configureServices); - TestHttpClient = new TestHttpClient(_factory.CreateClient()); + TestHttpClient = new TestHttpClient(_factory.CreateClient(), authTokenProvider); } public TestHttpClient TestHttpClient { get; } diff --git a/src/ConfIT/SuiteBootstrapper.cs b/src/ConfIT/SuiteBootstrapper.cs index ec41170..8f5eaec 100644 --- a/src/ConfIT/SuiteBootstrapper.cs +++ b/src/ConfIT/SuiteBootstrapper.cs @@ -21,6 +21,12 @@ public static class SuiteBootstrapper /// Bootstraps a component test suite where the application under test runs in-process /// via . /// + /// + /// If the YAML auth block uses oauth2-client-credentials, the token endpoint + /// must be reachable before this method is called (i.e. any stub WireMock server + /// must already be running — the caller owns that lifecycle). The token request happens + /// once, eagerly, before the in-process host starts. + /// /// The application's entry-point class (Program or Startup). /// Path to the suite YAML config file (e.g. "suite.config.yaml"). /// Optional DI overrides applied to the in-process host. @@ -36,8 +42,9 @@ public static BootstrappedSuite ForComponent( Dictionary? customMatchers = null) where TStartup : class { - var cfg = SuiteConfiguration.LoadComponent(configFile); - var initializer = new TestSuiteInitializer(cfg.Startup.Settings!, configureServices); + var cfg = SuiteConfiguration.LoadComponent(configFile); + var authProvider = cfg.ToAuthTokenProvider(); + var initializer = new TestSuiteInitializer(cfg.Startup.Settings!, configureServices, authProvider); onStarted?.Invoke(initializer.Services); diff --git a/test/ConfIT.UnitTest/Config/SuiteConfigurationTests.cs b/test/ConfIT.UnitTest/Config/SuiteConfigurationTests.cs index 62eac86..4b48907 100644 --- a/test/ConfIT.UnitTest/Config/SuiteConfigurationTests.cs +++ b/test/ConfIT.UnitTest/Config/SuiteConfigurationTests.cs @@ -47,6 +47,56 @@ public void LoadComponent_InProcessMode_ReturnsValidConfig() } } + [Fact] + public void LoadComponent_MockEnableLogsTrue_ParsesFlag() + { + var path = Write(""" + component: + startup: + mode: in-process + settings: appsettings.Tests.json + api: + url: http://localhost:5170 + mock: + url: http://localhost:8888 + enableLogs: true + """); + try + { + var cfg = SuiteConfiguration.LoadComponent(path); + + Assert.True(cfg.Mock?.EnableLogs); + Assert.True(cfg.ToSuiteConfig().EnableMockServerLogs); + } + finally + { + Cleanup(path); + } + } + + [Fact] + public void LoadComponent_MockWithoutEnableLogs_DefaultsToFalse() + { + var path = Write(""" + component: + startup: + mode: in-process + settings: appsettings.Tests.json + api: + url: http://localhost:5170 + mock: + url: http://localhost:8888 + """); + try + { + Assert.False(SuiteConfiguration.LoadComponent(path).Mock?.EnableLogs); + } + finally + { + Cleanup(path); + } + } + [Fact] public void LoadComponent_CommandMode_ReturnsValidConfig() { @@ -379,6 +429,27 @@ public void ToSuiteConfig_Component_MapsAllFields() Assert.Equal("resp", sc.ApiResponseFolder); Assert.Equal("req", sc.RequestBodyFolder); Assert.Equal("expResp", sc.ResponseBodyFolder); + Assert.False(sc.EnableMockServerLogs); + } + + [Fact] + public void ToSuiteConfig_MockEnableLogsTrue_EnablesMockServerLogs() + { + var cfg = new ComponentConfig + { + Api = new ApiConfig { Url = "http://api:5170" }, + Mock = new MockConfig { Url = "http://mock:8888", EnableLogs = true } + }; + + Assert.True(cfg.ToSuiteConfig().EnableMockServerLogs); + } + + [Fact] + public void ToSuiteConfig_NoMockSection_EnableMockServerLogsIsFalse() + { + var cfg = new ComponentConfig { Api = new ApiConfig { Url = "http://api:5170" } }; + + Assert.False(cfg.ToSuiteConfig().EnableMockServerLogs); } [Fact] diff --git a/tools/check-testcases.py b/tools/check-testcases.py new file mode 100755 index 0000000..b292854 --- /dev/null +++ b/tools/check-testcases.py @@ -0,0 +1,707 @@ +#!/usr/bin/env python3 +""" +check-testcases.py — static validation of ConfIT test definition files. + +Catches, without a build or a test run, the errors ConfIT raises at load time or on the +first failing assertion: + + * a response with no expected body (the structural diff always reports a difference) + * depends: naming an unknown test, a later test, or a test in another file + * {{variables}} that are never extracted, extracted later, or ambiguous + * graphql blocks with neither query nor queryFromFile + * bodyFromFile / queryFromFile pointing at a file that does not exist + * mock: blocks in an integration suite, or with no mock server configured + * test files and fixtures missing their csproj registration + * untagged tests in a suite that filters by tag + +Usage: + python3 check-testcases.py [TEST_PROJECT_DIR] # defaults to the current directory + +Exit code 1 when any ERROR is reported. PyYAML is used when installed; otherwise a bundled +subset parser handles the YAML dialect the ConfIT DSL uses. +""" + +import fnmatch +import json +import os +import re +import sys + +# --------------------------------------------------------------------------- YAML subset + + +class YamlError(Exception): + pass + + +def _strip_comment(line): + out, quote = [], None + for i, ch in enumerate(line): + if quote: + out.append(ch) + if ch == quote and line[i - 1: i] != "\\": + quote = None + elif ch in "\"'": + quote = ch + out.append(ch) + elif ch == "#" and (i == 0 or line[i - 1] in " \t"): + break + else: + out.append(ch) + return "".join(out).rstrip() + + +def _scalar(text): + text = text.strip() + if len(text) >= 2 and text[0] == text[-1] and text[0] in "\"'": + return text[1:-1] + if text in ("", "~", "null", "Null", "NULL"): + return None + if text in ("true", "True", "TRUE"): + return True + if text in ("false", "False", "FALSE"): + return False + try: + return int(text) + except ValueError: + pass + try: + return float(text) + except ValueError: + pass + return text + + +def _split_key(text): + """Split 'key: value' at the first structural colon. Returns (key, rest) or None.""" + quote, depth = None, 0 + for i, ch in enumerate(text): + if quote: + if ch == quote: + quote = None + elif ch in "\"'": + quote = ch + elif ch in "{[": + depth += 1 + elif ch in "}]": + depth -= 1 + elif ch == ":" and depth == 0 and (i + 1 == len(text) or text[i + 1] in " \t"): + key = text[:i].strip() + if len(key) >= 2 and key[0] == key[-1] and key[0] in "\"'": + key = key[1:-1] + return key, text[i + 1:].strip() + return None + + +def _parse_flow(text, pos=0, top=False): + """Parse a single-line flow collection or scalar. Returns (value, next_pos). + + `top` marks a value that is not inside a flow collection, where a plain scalar runs to end of + line — `hasLength(1, 100)` must not be truncated at the comma. + """ + while pos < len(text) and text[pos] in " \t": + pos += 1 + if pos >= len(text): + return None, pos + if text[pos] == "{": + out, pos = {}, pos + 1 + while True: + while pos < len(text) and text[pos] in " \t,": + pos += 1 + if pos < len(text) and text[pos] == "}": + return out, pos + 1 + if pos >= len(text): + raise YamlError("unterminated flow mapping") + key, pos = _parse_flow_scalar(text, pos, stop=":") + while pos < len(text) and text[pos] in " \t:": + pos += 1 + val, pos = _parse_flow(text, pos) + out[str(key)] = val + if text[pos] == "[": + out, pos = [], pos + 1 + while True: + while pos < len(text) and text[pos] in " \t,": + pos += 1 + if pos < len(text) and text[pos] == "]": + return out, pos + 1 + if pos >= len(text): + raise YamlError("unterminated flow sequence") + val, pos = _parse_flow(text, pos) + out.append(val) + return _parse_flow_scalar(text, pos, stop="" if top else ",}]") + + +def _parse_flow_scalar(text, pos, stop): + if text[pos] in "\"'": + quote, pos, buf = text[pos], pos + 1, [] + while pos < len(text) and text[pos] != quote: + buf.append(text[pos]) + pos += 1 + return "".join(buf), pos + 1 + start = pos + while pos < len(text) and text[pos] not in stop: + pos += 1 + return _scalar(text[start:pos]), pos + + +class _Block: + """Indentation-driven parser for the block YAML the ConfIT DSL uses.""" + + def __init__(self, text): + self.lines = [] + for n, raw in enumerate(text.splitlines(), start=1): + body = _strip_comment(raw) + if not body.strip(): + continue + self.lines.append({"n": n, "indent": len(body) - len(body.lstrip()), "text": body.strip()}) + self.anchors = {} + self.lineno = {} + + def parse(self): + if not self.lines: + return {} + value, idx = self.block(0, self.lines[0]["indent"]) + if idx != len(self.lines): + raise YamlError("line %d: unexpected indentation" % self.lines[idx]["n"]) + return value + + def block(self, idx, indent): + if self.lines[idx]["text"] == "-" or self.lines[idx]["text"].startswith("- "): + return self.sequence(idx, indent) + return self.mapping(idx, indent) + + def mapping(self, idx, indent): + out = {} + while idx < len(self.lines) and self.lines[idx]["indent"] == indent: + line = self.lines[idx] + if line["text"].startswith("- "): + break + split = _split_key(line["text"]) + if split is None: + raise YamlError("line %d: expected 'key: value'" % line["n"]) + key, rest = split + value, idx = self.value(rest, idx + 1, indent, line["n"]) + out[key] = value + self.lineno.setdefault(id(out), {})[key] = line["n"] + return out, idx + + def sequence(self, idx, indent): + out = [] + while idx < len(self.lines) and self.lines[idx]["indent"] == indent: + line = self.lines[idx] + if not (line["text"] == "-" or line["text"].startswith("- ")): + break + rest = line["text"][2:].strip() if line["text"] != "-" else "" + if not rest: + idx += 1 + if idx < len(self.lines) and self.lines[idx]["indent"] > indent: + value, idx = self.block(idx, self.lines[idx]["indent"]) + else: + value = None + out.append(value) + continue + # Item content sits on the dash line: re-enter with a virtual line at indent + 2. + self.lines[idx] = {"n": line["n"], "indent": indent + 2, "text": rest} + if _split_key(rest) is not None: + value, idx = self.mapping(idx, indent + 2) + else: + value, idx = self.value(rest, idx + 1, indent + 2, line["n"]) + out.append(value) + return out, idx + + def value(self, rest, idx, indent, lineno): + anchor = None + match = re.match(r"^&(\S+)\s*(.*)$", rest) + if match: + anchor, rest = match.group(1), match.group(2).strip() + alias = re.match(r"^\*(\S+)$", rest) + if alias: + if alias.group(1) not in self.anchors: + raise YamlError("line %d: unknown alias '*%s'" % (lineno, alias.group(1))) + return json.loads(json.dumps(self.anchors[alias.group(1)])), idx + + if rest in ("|", "|-", "|+", ">", ">-", ">+"): + buf, fold = [], rest[0] == ">" + base = None + while idx < len(self.lines) and self.lines[idx]["indent"] > indent: + if base is None: + base = self.lines[idx]["indent"] + buf.append(" " * max(0, self.lines[idx]["indent"] - base) + self.lines[idx]["text"]) + idx += 1 + value = (" " if fold else "\n").join(buf) + if rest.endswith("-"): + value = value.rstrip("\n") + else: + value += "" if fold else "\n" + elif rest == "": + if idx < len(self.lines) and self.lines[idx]["indent"] > indent: + value, idx = self.block(idx, self.lines[idx]["indent"]) + else: + value = None + else: + value, _ = _parse_flow(rest, top=True) + + if anchor: + self.anchors[anchor] = value + return value, idx + + +def load_yaml(text): + try: + import yaml # noqa: F401 + return yaml.safe_load(text) + except ImportError: + return _Block(text).parse() + + +def load_file(path): + text = open(path, encoding="utf-8-sig").read() + if path.lower().endswith(".json"): + return json.loads(text) + return load_yaml(text) + + +# ------------------------------------------------------------------------------- report + +RESET, RED, YELLOW, DIM = "\033[0m", "\033[31m", "\033[33m", "\033[2m" + + +class Report: + def __init__(self): + self.items = [] + + def error(self, path, test, message): + self.items.append(("ERROR", path, test, message)) + + def warn(self, path, test, message): + self.items.append(("WARN", path, test, message)) + + def print(self, root): + colour = sys.stdout.isatty() + by_path = {} + for level, path, test, message in self.items: + by_path.setdefault(path, []).append((level, test, message)) + for path in sorted(by_path): + print("\n" + os.path.relpath(path, root)) + for level, test, message in by_path[path]: + tag = level.ljust(5) + if colour: + tag = (RED if level == "ERROR" else YELLOW) + tag + RESET + where = (DIM + test + RESET) if colour and test else test + print(" %s %s%s%s" % (tag, where, " " if test else "", message)) + errors = sum(1 for i in self.items if i[0] == "ERROR") + warns = len(self.items) - errors + print("\n%d error(s), %d warning(s)" % (errors, warns)) + return errors + + +# -------------------------------------------------------------------------- suite config + + +def find_suite_config(root): + for base, dirs, files in os.walk(root): + dirs[:] = [d for d in dirs if d not in ("bin", "obj", ".git")] + if "suite.config.yaml" in files: + return os.path.join(base, "suite.config.yaml") + return None + + +def read_suite(root): + """Returns (kind, mock_url, folders, filter_strategy, config_path).""" + path = find_suite_config(root) + if not path: + return None, None, {}, None, None + try: + cfg = load_yaml(open(path, encoding="utf-8-sig").read()) or {} + except Exception as exc: # noqa: BLE001 + print("could not parse %s: %s" % (path, exc), file=sys.stderr) + return None, None, {}, None, path + if "component" in cfg: + section = cfg["component"] or {} + kind = "component" + elif "integration" in cfg: + integration = cfg["integration"] or {} + env = os.environ.get("TEST_ENVIRONMENT") or integration.get("default") + section = (integration.get(env) or {}) if env else {} + kind = "integration" + else: + return None, None, {}, None, path + mock = (section.get("mock") or {}).get("url") + folders = section.get("folders") or {} + strategy = (section.get("filter") or {}).get("strategy") + return kind, mock, folders, strategy, path + + +# ---------------------------------------------------------------------------- discovery + + +def is_test_file(data): + return ( + isinstance(data, dict) + and len(data) > 0 + and all(isinstance(v, dict) and "api" in v for v in data.values()) + ) + + +def discover(root, report): + """Returns [(path, ordered [(name, case)])] sorted by folder then filename. + + Groups candidates by directory first: a directory with at least one file shaped like tests + is a test directory, and any dict-shaped sibling there that is NOT correctly shaped gets + reported instead of silently vanishing — ConfIT's own TestReader loads every file in a wired + folder unconditionally, so a malformed sibling fails at load time, not skips quietly. + Directories with no test-shaped file at all (fixture folders such as RequestBody/ResponseBody) + are left alone. + """ + by_dir = {} + for base, dirs, files in os.walk(root): + dirs[:] = [d for d in dirs if d not in ("bin", "obj", ".git", "node_modules")] + for name in sorted(files): + if not name.lower().endswith((".yaml", ".yml", ".json")): + continue + if name in ("suite.config.yaml", "packages.lock.json") or name.startswith("appsettings"): + continue + path = os.path.join(base, name) + try: + data = load_file(path) + except Exception as exc: # noqa: BLE001 + text = open(path, encoding="utf-8-sig", errors="replace").read() + if re.search(r"^\s*api\s*:", text, re.M): + report.error(path, "", "could not parse this file: %s" % exc) + continue + by_dir.setdefault(base, []).append((path, data)) + + found = [] + for entries in by_dir.values(): + test_shaped = [(path, data) for path, data in entries if is_test_file(data)] + if not test_shaped: + continue + test_paths = {path for path, _ in test_shaped} + for path, data in entries: + if path in test_paths or not isinstance(data, dict) or not data: + continue + for case_name, case in data.items(): + if not (isinstance(case, dict) and "api" in case): + report.error(path, case_name, "not a valid test case — missing an 'api' section") + found += [(path, list(data.items())) for path, data in test_shaped] + return sorted(found, key=lambda item: (os.path.dirname(item[0]), os.path.basename(item[0]))) + + +# ------------------------------------------------------------------------------- checks + +VAR_REF = re.compile(r"\{\{\s*([A-Za-z_][\w]*)(?:\.([A-Za-z_][\w]*))?\s*\}\}") + +# Built-in matcher names are read from the library rather than hardcoded here, so adding a +# matcher upstream never leaves this checker rejecting valid tests. +SEMANTIC_SOURCE = "src/ConfIT/Matching/SemanticMatcher.cs" + + +def builtin_matchers(): + """Names from SemanticMatcher's BuiltIns dictionary, or None when source is unavailable.""" + root = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) + path = os.path.join(root, SEMANTIC_SOURCE) + if not os.path.isfile(path): + return None + text = open(path, encoding="utf-8-sig").read() + names = re.findall(r'\["([A-Za-z][A-Za-z0-9_]*)"\]\s*=', text) + return set(names) or None + + +def custom_matchers(root): + """Keys inside a 'Dictionary' initializer in the project's C#, + so registered custom matchers pass. Scoped to that type specifically — not every dictionary-key + string literal in the project — so an unrelated dictionary cannot accidentally whitelist a + typo'd or unregistered matcher name.""" + found = set() + block_re = re.compile(r'Dictionary\s*<\s*string\s*,\s*SemanticMatcherFunc\s*>.*?\n[ \t]*\}\s*;', re.S) + key_re = re.compile(r'\["([A-Za-z][A-Za-z0-9_]*)"\]\s*=') + for base, dirs, files in os.walk(root): + dirs[:] = [d for d in dirs if d not in ("bin", "obj", ".git")] + for name in files: + if not name.endswith(".cs"): + continue + try: + text = open(os.path.join(base, name), encoding="utf-8-sig", errors="replace").read() + except OSError: + continue + for block in block_re.findall(text): + found.update(key_re.findall(block)) + return found + + +def walk_strings(node): + if isinstance(node, dict): + for value in node.values(): + yield from walk_strings(value) + elif isinstance(node, list): + for value in node: + yield from walk_strings(value) + elif isinstance(node, str): + yield node + + +def check_structure(path, name, case, suite_kind, mock_url, folders, strategy, root, report): + api = case.get("api") + if not isinstance(api, dict): + report.error(path, name, "no 'api' section") + return + + request = api.get("request") + if not isinstance(request, dict): + report.error(path, name, "api.request is missing") + else: + graphql = request.get("graphql") + if graphql is not None: + if not isinstance(graphql, dict): + report.error(path, name, "api.request.graphql must be a mapping") + elif not graphql.get("query") and not graphql.get("queryFromFile"): + report.error(path, name, "graphql block sets neither 'query' nor 'queryFromFile'") + elif not request.get("method"): + report.error(path, name, "api.request.method is missing") + if not request.get("path"): + report.error(path, name, "api.request.path is missing") + + response = api.get("response") + if not isinstance(response, dict): + report.error(path, name, "api.response is missing") + return + if response.get("statusCode") is None: + report.error(path, name, "api.response.statusCode is missing") + if "body" not in response and "bodyFromFile" not in response: + report.error( + path, name, + "api.response has no 'body' — the structural diff compares the response against " + "nothing and always fails. Use 'body: {}' when every field is claimed by a matcher.") + + if case.get("mock") is not None: + if suite_kind == "integration": + report.error(path, name, "'mock:' block in an integration suite — no mock server runs there") + elif suite_kind == "component" and not mock_url: + report.error(path, name, "'mock:' block but suite.config.yaml sets no component.mock.url") + + if strategy == "tags" and not case.get("tags"): + report.warn(path, name, "no 'tags:' — this test is skipped whenever the tag filter is active") + + check_fixture_files(path, name, case, folders, root, report) + + +def check_fixture_files(path, name, case, folders, root, report): + request_dir = folders.get("requestBody") + response_dir = folders.get("responseBody") + + def exists(folder, filename): + if not folder: + return None + return os.path.isfile(os.path.join(root, folder, filename)) + + def check(payload, folder, label, key="bodyFromFile"): + if not isinstance(payload, dict): + return + filename = payload.get(key) + if not filename: + return + if not folder: + report.error(path, name, "%s: '%s: %s' but suite.config.yaml sets no matching folders entry" + % (label, key, filename)) + elif exists(folder, filename) is False: + report.error(path, name, "%s: '%s: %s' not found under %s/" % (label, key, filename, folder)) + + api = case.get("api") or {} + check(api.get("request"), request_dir, "api.request") + check(api.get("response"), response_dir, "api.response") + graphql = (api.get("request") or {}).get("graphql") + if isinstance(graphql, dict): + check(graphql, request_dir, "api.request.graphql", key="queryFromFile") + for interaction in (case.get("mock") or {}).get("interactions") or []: + if isinstance(interaction, dict): + check(interaction.get("request"), request_dir, "mock.request") + check(interaction.get("response"), response_dir, "mock.response") + + +def check_matchers(path, name, case, known, report): + matcher = (((case.get("api") or {}).get("response")) or {}).get("matcher") + if not isinstance(matcher, dict): + return + + semantic = matcher.get("semantic") or {} + if isinstance(semantic, dict): + for field, spec in semantic.items(): + if not isinstance(spec, str): + report.error(path, name, "matcher.semantic['%s'] must be a matcher name" % field) + continue + if "(" in spec and not spec.endswith(")"): + report.error(path, name, "matcher.semantic['%s']: '%s' is missing its closing parenthesis" + % (field, spec)) + continue + base = spec.split("(", 1)[0] + if known is not None and base not in known: + report.error(path, name, "matcher.semantic['%s']: '%s' is not a built-in matcher and is not " + "registered as a custom matcher in this project" % (field, base)) + if "*" in str(field): + report.error(path, name, "matcher.semantic['%s']: wildcards are not supported by semantic " + "matchers — use ignore or pattern for per-element fields" % field) + elif any(seg.isdigit() for seg in str(field).split("__")): + report.error(path, name, "matcher.semantic['%s']: array indexes do not resolve in semantic " + "paths — assert the element literally, or use pattern/ignore" % field) + + for kind in ("ignore", "pattern"): + block = matcher.get(kind) + paths = block if isinstance(block, list) else list(block) if isinstance(block, dict) else [] + for field in paths: + if str(field).split("__")[-1] == "*": + report.error(path, name, "matcher.%s['%s']: '*' cannot be the final segment — it selects an " + "array to reach into, not the field being matched" % (kind, field)) + + +def check_depends(path, cases, report): + order = {name: i for i, (name, _) in enumerate(cases)} + for index, (name, case) in enumerate(cases): + depends = case.get("depends") or [] + if isinstance(depends, str): + depends = [depends] + for prerequisite in depends: + if prerequisite not in order: + report.error(path, name, "depends: '%s' — no test with that name in this file " + "(depends is file-scoped)" % prerequisite) + elif order[prerequisite] >= index: + report.error(path, name, "depends: '%s' is defined later in the file — " + "prerequisites must come first" % prerequisite) + + +def check_variables(files, report): + extracted = {} # short name -> [(position, test name)] + for position, (path, name, case) in enumerate(flatten(files)): + block = ((case.get("api") or {}).get("response") or {}).get("extract") or {} + if isinstance(block, dict): + for variable in block: + extracted.setdefault(variable, []).append((position, name)) + + for position, (path, name, case) in enumerate(flatten(files)): + seen = set() + for text in walk_strings(case): + for match in VAR_REF.finditer(text): + prefix, suffix = match.group(1), match.group(2) + reference = match.group(0) + if reference in seen: + continue + seen.add(reference) + if suffix: # {{TestName.var}} + sources = [s for s in extracted.get(suffix, []) if s[1] == prefix] + if not sources: + report.error(path, name, "%s — test '%s' does not extract '%s'" + % (reference, prefix, suffix)) + elif sources[0][0] >= position: + report.error(path, name, "%s is extracted by a test that runs later" + % reference) + continue + sources = extracted.get(prefix, []) + if not sources: + report.error(path, name, "%s is never extracted by any test" % reference) + elif len(sources) > 1: + report.error(path, name, "%s is extracted by %s — ambiguous; use {{TestName.%s}}" + % (reference, " and ".join(s[1] for s in sources), prefix)) + elif sources[0][0] >= position: + report.error(path, name, "%s is extracted by '%s', which runs later" + % (reference, sources[0][1])) + + +def check_duplicate_names(files, report): + seen = {} + for path, name, _ in flatten(files): + if name in seen: + report.error(path, name, "duplicate test name — also defined in %s" + % os.path.basename(seen[name])) + else: + seen[name] = path + + +def flatten(files): + for path, cases in files: + for name, case in cases: + yield path, name, case + + +# -------------------------------------------------------------------------- csproj wiring + + +def check_registration(root, files, folders, config_path, report): + projects = [] + for base, dirs, names in os.walk(root): + dirs[:] = [d for d in dirs if d not in ("bin", "obj", ".git")] + projects += [os.path.join(base, n) for n in sorted(names) if n.endswith(".csproj")] + if not projects: + return + project = projects[0] + text = open(project, encoding="utf-8-sig").read() + registered = [] + for match in re.finditer(r'|>(.*?))', text, re.S): + registered.append((match.group(1).replace("\\", "/"), match.group(2) or "")) + + def is_registered(relative): + for pattern, body in registered: + normalized = pattern.replace("**/", "*/").replace("**", "*") + if pattern == relative or fnmatch.fnmatch(relative, normalized): + if "CopyToOutputDirectory" not in body: + report.error(project, "", "%s is listed but has no " % relative) + return True + return False + + targets = [path for path, _ in files] + if config_path: + targets.append(config_path) + for folder in {folders.get("requestBody"), folders.get("responseBody")} - {None}: + directory = os.path.join(root, folder) + if os.path.isdir(directory): + targets += [os.path.join(directory, f) for f in sorted(os.listdir(directory)) + if os.path.isfile(os.path.join(directory, f))] + + for target in targets: + relative = os.path.relpath(target, os.path.dirname(project)).replace(os.sep, "/") + if relative.startswith(".."): + continue + if not is_registered(relative): + report.error(project, "", "%s is not registered — add a entry with " + "Always" % relative) + + +# ----------------------------------------------------------------------------------- main + + +def main(): + root = os.path.abspath(sys.argv[1] if len(sys.argv) > 1 else ".") + report = Report() + + kind, mock_url, folders, strategy, config_path = read_suite(root) + if kind is None: + print("no suite.config.yaml with a 'component:' or 'integration:' section under %s — " + "suite-level checks skipped" % root, file=sys.stderr) + + files = discover(root, report) + if not files: + print("no ConfIT test definition files found under %s" % root) + if kind is not None: + report.error(config_path, "", "suite.config.yaml declares a '%s:' section but no " + "test definition files were found" % kind) + return 1 if report.print(root) else 0 + + known = builtin_matchers() + if known is not None: + known |= custom_matchers(root) + + for path, cases in files: + for name, case in cases: + check_structure(path, name, case, kind, mock_url, folders, strategy, root, report) + check_matchers(path, name, case, known, report) + check_depends(path, cases, report) + + check_duplicate_names(files, report) + check_variables(files, report) + check_registration(root, files, folders, config_path, report) + + print("checked %d test file(s), %d test(s)" + % (len(files), sum(len(cases) for _, cases in files))) + return 1 if report.print(root) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/verify-suite.sh b/tools/verify-suite.sh new file mode 100755 index 0000000..261d580 --- /dev/null +++ b/tools/verify-suite.sh @@ -0,0 +1,267 @@ +#!/usr/bin/env bash +# verify-suite.sh — wiring diagnostics for a ConfIT test project. +# +# Reports the setup faults that surface later as confusing runtime symptoms: a missing package +# reference, unregistered config or test files, a fixture that does not match the config +# section, a hardcoded filter, a framework mismatch. +# +# Usage: bash verify-suite.sh [TEST_PROJECT_DIR] # defaults to the current directory +# Exit: 1 when any ERROR is reported. + +set -u + +DIR="${1:-.}" +ERRORS=0 +WARNS=0 + +if [ -t 1 ]; then RED=$'\033[31m'; YEL=$'\033[33m'; GRN=$'\033[32m'; OFF=$'\033[0m' +else RED=""; YEL=""; GRN=""; OFF=""; fi + +err() { printf ' %sERROR%s %s\n' "$RED" "$OFF" "$1"; ERRORS=$((ERRORS + 1)); } +warn() { printf ' %sWARN %s %s\n' "$YEL" "$OFF" "$1"; WARNS=$((WARNS + 1)); } +ok() { printf ' %sok%s %s\n' "$GRN" "$OFF" "$1"; } +section() { printf '\n%s\n' "$1"; } + +[ -d "$DIR" ] || { echo "no such directory: $DIR" >&2; exit 2; } +DIR="$(cd "$DIR" && pwd)" + +CSPROJ="$(find "$DIR" -maxdepth 2 -name '*.csproj' -not -path '*/bin/*' -not -path '*/obj/*' | head -1)" +[ -n "$CSPROJ" ] || { echo "no .csproj found under $DIR" >&2; exit 2; } + +# Ground truth comes from the ConfIT checkout this script ships inside, never from a value +# hardcoded here. pwd -P so a symlinked invocation still lands on the real root. +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." 2>/dev/null && pwd -P || true)" +LIB_CSPROJ="$REPO_ROOT/src/ConfIT/ConfIT.csproj" +SUPPORTED_TFM="" +if [ -n "$REPO_ROOT" ] && [ -f "$LIB_CSPROJ" ]; then + SUPPORTED_TFM="$(sed -n 's/.*\([^<]*\)<.*/\1/p' "$LIB_CSPROJ" | head -1)" +fi + +# Every path that carries a , separators normalised. +# Splitting on '<' turns each XML tag into its own line, which keeps this awk-only. +REGISTERED="$(tr '<' '\n' < "$CSPROJ" | awk ' + /^None[[:space:]]+Update=/ { p = (match($0, /Update="[^"]*"/) ? substr($0, RSTART + 8, RLENGTH - 9) : ""); next } + /^CopyToOutputDirectory>/ { if (p != "") { print p; p = "" } next } + /^\/None>/ { p = "" } +' | tr '\\' '/')" + +registered() { + local rel="$1" pattern + while IFS= read -r pattern; do + [ -z "$pattern" ] && continue + [ "$pattern" = "$rel" ] && return 0 + case "$pattern" in + *'*'*) case "$rel" in ${pattern//\*\*/\*}) return 0 ;; esac ;; + esac + done <<< "$REGISTERED" + return 1 +} + +sources() { find "$DIR" -name '*.cs' -not -path '*/bin/*' -not -path '*/obj/*'; } +in_sources() { sources | tr '\n' '\0' | xargs -0 grep -l "$1" 2>/dev/null | head -1; } + +# Prints the lines nested under a 'key:' mapping (more indented than it), stopping at the next +# line back at or above that indent. Collects every occurrence of the key in the file, so an +# integration config with one 'api:' block per environment still gets each of them checked. +yaml_block() { + local file="$1" key="$2" + awk -v key="$key" ' + { + raw = $0 + gsub(/\t/, " ", raw) + trimmed = raw + sub(/^[ ]*/, "", trimmed) + if (trimmed == "") next + indent = length(raw) - length(trimmed) + if (active && indent <= key_indent) active = 0 + if (!active && trimmed ~ ("^" key ":")) { active = 1; key_indent = indent; next } + if (active) print raw + } + ' "$file" +} + +echo "ConfIT suite check: $(basename "$CSPROJ")" + +# --------------------------------------------------------------------- project references +section "project" +if grep -Eq '<(Package|Project)Reference[^>]*"[^"]*ConfIT' "$CSPROJ"; then + ok "ConfIT is referenced" +else + err "no ConfIT reference — run: dotnet add package ConfIT" +fi + +if grep -q ']*"xunit' "$CSPROJ"; then + ok "xunit is referenced" +else + warn "no xunit reference — BaseTest is built around xUnit's [Theory] / [MemberData]" +fi + +TFM="$(sed -n 's/.*\([^<]*\)<.*/\1/p' "$CSPROJ" | head -1)" +if [ -z "$TFM" ]; then + warn "no TargetFramework found in the csproj" +elif [ -z "$SUPPORTED_TFM" ]; then + ok "target framework: $TFM" + warn "could not read ConfIT's supported frameworks from src/ConfIT/ConfIT.csproj — not verified" +else + _matched=0 + for _want in $(printf '%s' "$TFM" | tr ';' ' '); do + for _have in $(printf '%s' "$SUPPORTED_TFM" | tr ';' ' '); do + [ "$_want" = "$_have" ] && _matched=1 + done + done + if [ "$_matched" -eq 1 ]; then + ok "target framework: $TFM (ConfIT ships $SUPPORTED_TFM)" + else + err "target framework '$TFM' — ConfIT ships $SUPPORTED_TFM" + fi +fi + +# ----------------------------------------------------------------------------- suite config +section "suite.config.yaml" +CONFIG="$DIR/suite.config.yaml" +SUITE_SECTION="" +if [ -f "$CONFIG" ]; then + ok "present" + if registered "suite.config.yaml"; then + ok "registered with CopyToOutputDirectory" + else + err "not registered — add with Always" + fi + + if grep -q '^component:' "$CONFIG"; then + SUITE_SECTION="component" + elif grep -q '^integration:' "$CONFIG"; then + SUITE_SECTION="integration" + else + err "neither a 'component:' nor an 'integration:' section" + fi + [ -n "$SUITE_SECTION" ] && ok "section: $SUITE_SECTION" + + API_BLOCK="$(yaml_block "$CONFIG" "api")" + if [ -z "$API_BLOCK" ] || ! printf '%s\n' "$API_BLOCK" | grep -Eq '^[[:space:]]*url:[[:space:]]*[^[:space:]]'; then + err "no 'api.url' — required in every mode" + fi + + if [ "$SUITE_SECTION" = "component" ]; then + MODE="$(sed -n 's/^[[:space:]]*mode:[[:space:]]*\([a-z-]*\).*/\1/p' "$CONFIG" | head -1)" + case "${MODE:-in-process}" in + in-process) + SETTINGS="$(sed -n 's/^[[:space:]]*settings:[[:space:]]*\([^[:space:]]*\).*/\1/p' "$CONFIG" | head -1)" + if [ -z "$SETTINGS" ]; then + err "mode is in-process but 'startup.settings' is not set" + elif [ -f "$DIR/$SETTINGS" ]; then + ok "startup.settings: $SETTINGS" + registered "$SETTINGS" || err "$SETTINGS is not registered with CopyToOutputDirectory" + else + err "startup.settings names '$SETTINGS', which is not in the project" + fi + ;; + command) + grep -q 'command:' "$CONFIG" || err "mode is command but 'startup.command' is not set" + if grep -q 'readiness:' "$CONFIG"; then + READINESS_BLOCK="$(yaml_block "$CONFIG" "readiness")" + HAS_URL=0; HAS_PORT=0 + printf '%s\n' "$READINESS_BLOCK" | grep -Eq '^[[:space:]]*url:[[:space:]]*[^[:space:]]' && HAS_URL=1 + printf '%s\n' "$READINESS_BLOCK" | grep -Eq '^[[:space:]]*port:[[:space:]]*[^[:space:]]' && HAS_PORT=1 + if [ $((HAS_URL + HAS_PORT)) -eq 1 ]; then + ok "readiness probe declared" + else + err "readiness: needs exactly one of 'port' or 'url'" + fi + else + err "mode is command but there is no 'readiness:' block — the launcher cannot tell when the app is up" + fi + grep -q 'stopCommand:' "$CONFIG" \ + || warn "no 'stopCommand' — add one if the port is not released between runs" + ;; + *) err "unknown startup mode '$MODE' — expected in-process or command" ;; + esac + fi + + grep -Eq 'strategy:[[:space:]]*tags' "$CONFIG" \ + && ok "tag filter configured — every test then needs a 'tags:' entry" + grep -q '\${' "$CONFIG" \ + && ok "uses \${ENV_VAR} interpolation — export those variables before running" +else + err "suite.config.yaml not found in $DIR" +fi + +# ---------------------------------------------------------------------------------- fixture +section "fixture" +FIXTURE="$(in_sources 'SuiteBootstrapper\.For')" +if [ -n "$FIXTURE" ]; then + ok "bootstrapped in $(basename "$FIXTURE")" + CALL="$(grep -o 'SuiteBootstrapper\.For[A-Za-z]*' "$FIXTURE" | head -1)" + ok "$CALL" + case "$SUITE_SECTION:$CALL" in + component:SuiteBootstrapper.ForIntegration) + err "config declares a 'component:' section but the fixture calls ForIntegration" ;; + integration:SuiteBootstrapper.ForComponent|integration:SuiteBootstrapper.ForCommand) + err "config declares an 'integration:' section but the fixture calls ${CALL#SuiteBootstrapper.}" ;; + esac + grep -q 'Dispose' "$FIXTURE" \ + || err "the fixture never disposes the suite — the summary will not print and infrastructure will leak" + grep -q 'TestSuiteContext' "$FIXTURE" \ + || warn "the fixture exposes no TestSuiteContext property for the test class to consume" +elif [ -n "$(in_sources 'new TestSuiteContext')" ]; then + ok "manually wired TestSuiteContext (adapter-chain or manual setup)" +else + err "no fixture found — neither a SuiteBootstrapper.For* call nor a TestSuiteContext construction" +fi + +# ------------------------------------------------------------------------------- test class +section "test class" +TESTCLASS="$(in_sources ': BaseTest')" +if [ -n "$TESTCLASS" ]; then + ok "BaseTest subclass: $(basename "$TESTCLASS")" + grep -q 'IClassFixture<' "$TESTCLASS" \ + || warn "no IClassFixture<> — the suite would start once per test instead of once per class" + grep -q 'TestReader\.GetTestsFor' "$TESTCLASS" \ + || err "no TestReader.GetTestsForAFolder / GetTestsForAFile — nothing will be discovered" + grep -q 'MemberData' "$TESTCLASS" \ + || err "no [MemberData] — the theory has no rows to run" +else + err "no class deriving from BaseTest" +fi + +[ -n "$(in_sources 'ITestOutputLogger')" ] \ + && ok "ITestOutputLogger adapter present" \ + || warn "no ITestOutputLogger implementation — ConfIT log output will not reach xUnit" + +[ -n "$(in_sources 'Environment\.SetEnvironmentVariable')" ] \ + && warn "Environment.SetEnvironmentVariable in test code — a hardcoded filter hides tests from CI" + +# ------------------------------------------------------------------------- test definitions +section "test definitions" +FOLDER="" +for candidate in TestCase TestCases Tests; do + [ -d "$DIR/$candidate" ] && { FOLDER="$candidate"; break; } +done +if [ -z "$FOLDER" ]; then + err "no TestCase folder found — create one and point [MemberData] at it" +else + COUNT=$(find "$DIR/$FOLDER" -maxdepth 1 \( -name '*.yaml' -o -name '*.yml' -o -name '*.json' \) | wc -l | tr -d ' ') + if [ "$COUNT" -gt 0 ]; then + ok "$COUNT test definition file(s) in $FOLDER/" + else + err "$FOLDER/ contains no .yaml / .yml / .json files" + fi + + MISSING=0 + while IFS= read -r file; do + [ -z "$file" ] && continue + rel="${file#"$DIR"/}" + registered "$rel" || { err "$rel is not registered with CopyToOutputDirectory"; MISSING=$((MISSING + 1)); } + done <<< "$(find "$DIR/$FOLDER" -type f \( -name '*.yaml' -o -name '*.yml' -o -name '*.json' -o -name '*.graphql' \))" + [ "$MISSING" -eq 0 ] && ok "all test definitions and body fixtures are registered" +fi + +OUT="$(find "$DIR/bin" -maxdepth 2 -type d -name 'net*' 2>/dev/null | head -1)" +if [ -n "$OUT" ]; then + [ -f "$OUT/suite.config.yaml" ] \ + && ok "suite.config.yaml reached the output directory" \ + || warn "suite.config.yaml is not in $OUT — rebuild, then re-check" +fi + +printf '\n%d error(s), %d warning(s)\n' "$ERRORS" "$WARNS" +[ "$ERRORS" -eq 0 ] || exit 1