From b81a00afa74405af8bd33c0629eaa3c4e3300bd3 Mon Sep 17 00:00:00 2001 From: nisargabd Date: Thu, 3 Sep 2026 15:01:56 +0530 Subject: [PATCH 001/130] chore: add test, security and build-and-push CI workflows [#5] Adds a run-tests/security/build-and-push flow: gated lint, gotestsum with diff-scoped coverage, dependency+image Trivy scans gated on HIGH/CRITICAL, multi-registry multi-arch build-and-push on release tags. Both Dockerfiles switch to dhi.io base images (with public overrides for a local build without a DHI subscription), so the security and build-and-push jobs authenticate to dhi.io first. --- .github/actions/dhi-login/action.yml | 28 +++ .github/workflows/build-and-push.yml | 276 +++++++++++++++++++++++++++ .github/workflows/run-tests.yml | 107 +++++++++++ .github/workflows/security.yml | 215 +++++++++++++++++++++ .gitignore | 8 + Dockerfile.adapter | 36 +++- Dockerfile.adapter-with-plugins | 34 +++- Makefile | 175 +++++++++++++++++ 8 files changed, 864 insertions(+), 15 deletions(-) create mode 100644 .github/actions/dhi-login/action.yml create mode 100644 .github/workflows/build-and-push.yml create mode 100644 .github/workflows/run-tests.yml create mode 100644 .github/workflows/security.yml create mode 100644 Makefile diff --git a/.github/actions/dhi-login/action.yml b/.github/actions/dhi-login/action.yml new file mode 100644 index 00000000..4dac2fdb --- /dev/null +++ b/.github/actions/dhi-login/action.yml @@ -0,0 +1,28 @@ +name: Log in to dhi.io +description: > + Authenticates to dhi.io (Docker Hardened Images), gated on a Docker + subscription — anonymous pulls 401. Both Dockerfiles default to dhi.io/* + base images (see each Dockerfile's FROM comment), so every job that builds + or scans an image needs this first. Written directly to + ~/.docker/config.json rather than through docker/login-action, whose + username/password inputs don't accept a pre-combined blob — this matches + the format `docker login` itself writes. + +inputs: + username: + description: DHI credential username (org secret DOCKERHUB_USERNAME — DHI's subscription is tied to a Docker Hub account) + required: true + token: + description: DHI credential token (org secret DOCKERHUB_TOKEN) + required: true + +runs: + using: composite + steps: + - shell: bash + run: | + AUTH=$(printf '%s:%s' "${{ inputs.username }}" "${{ inputs.token }}" | base64 -w0) + mkdir -p ~/.docker + cat > ~/.docker/config.json <> "$GITHUB_OUTPUT" + + - name: Cache Trivy binary and vulnerability DB + uses: actions/cache@v4 + with: + path: | + bin/trivy + ~/.cache/trivy + key: trivy-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Makefile') }}-${{ steps.trivy-date.outputs.date }} + restore-keys: | + trivy-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Makefile') }}- + + - uses: docker/setup-buildx-action@v3 + + # One flag per registry, read from repo variables (Settings > Secrets + # and variables > Actions > Variables), so a registry is switched on + # without a workflow edit. GHCR is the only one we use today, so it + # defaults on; everything else defaults off until its secrets exist. + # Adding a registry later is: a flag here, a login step below, and one + # more branch in "Tag and push" and the manifest job's create_manifest + # calls — the build/scan steps never change. + - name: Set registry flags + run: | + echo "GHCR_ENABLED=${{ vars.GHCR_ENABLED != 'false' }}" >> "$GITHUB_ENV" + echo "DOCKERHUB_ENABLED=${{ vars.DOCKERHUB_ENABLED == 'true' }}" >> "$GITHUB_ENV" + echo "AZURE_CONTAINER_REGISTRY_ENABLED=${{ vars.AZURE_CONTAINER_REGISTRY_ENABLED == 'true' }}" >> "$GITHUB_ENV" + echo "GOOGLE_ARTIFACT_REGISTRY_ENABLED=${{ vars.GOOGLE_ARTIFACT_REGISTRY_ENABLED == 'true' }}" >> "$GITHUB_ENV" + + - name: Log in to dhi.io + uses: ./.github/actions/dhi-login + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + token: ${{ secrets.DOCKERHUB_TOKEN }} + + # GITHUB_TOKEN can be denied write_package when creating a brand-new + # package under a personal (non-org) namespace, even with this + # workflow's `packages: write` and the repo's default token permissions + # both set to write — a documented GHCR quirk. GHCR_PAT (classic PAT, + # write:packages) is the standard workaround; falls back to + # GITHUB_TOKEN when no such secret is set, which is enough once the + # package exists or for an org-owned repo. + - name: Log in to GHCR + if: env.GHCR_ENABLED == 'true' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GHCR_PAT || secrets.GITHUB_TOKEN }} + + - name: Log in to Docker Hub + if: env.DOCKERHUB_ENABLED == 'true' + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Log in to Azure Container Registry + if: env.AZURE_CONTAINER_REGISTRY_ENABLED == 'true' + uses: docker/login-action@v3 + with: + registry: ${{ secrets.AZURE_REGISTRY_NAME }} + username: ${{ secrets.AZURE_REGISTRY_USERNAME }} + password: ${{ secrets.AZURE_REGISTRY_PASSWORD }} + + - name: Authenticate to Google Cloud + if: env.GOOGLE_ARTIFACT_REGISTRY_ENABLED == 'true' + uses: google-github-actions/auth@v3 + with: + token_format: "access_token" + workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_EMAIL }} + + - name: Configure Docker for Google Artifact Registry + if: env.GOOGLE_ARTIFACT_REGISTRY_ENABLED == 'true' + run: gcloud auth configure-docker ${{ secrets.GCP_REGION }}-docker.pkg.dev --quiet + + # GHCR image refs must be lowercase; github.repository_owner preserves + # the owner's real case, so it can't be used as-is. This resolves to + # whichever account or org owns the repo the tag was pushed to — a + # fork pushes under the fork owner, no per-person edit needed. + - name: Set image owner + run: echo "OWNER=$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_ENV" + + # Single source of truth for the build-time identity vars (see + # pkg/version) — written to $GITHUB_ENV so the build-push-action step + # below can reference them as build-args the same way a local + # `source install/scripts/version-vars.sh` would. + - name: Resolve version vars + run: | + source install/scripts/version-vars.sh + { + echo "ONIX_VERSION=$ONIX_VERSION" + echo "GIT_COMMIT=$GIT_COMMIT" + echo "GIT_TREE_STATE=$GIT_TREE_STATE" + echo "BUILD_DATE=$BUILD_DATE" + } >> "$GITHUB_ENV" + + # One local, unpushed tag per arch — the manifest job below stitches + # amd64 and arm64 into the single tag users actually pull. + # + # load: true instead of push: true — the image lands in this runner's + # local Docker daemon first so Trivy can scan the exact bytes about to + # ship, before it is tagged for or pushed to any registry. + - name: Build (local, unpushed) + uses: docker/build-push-action@v6 + with: + context: . + file: Dockerfile.adapter + platforms: ${{ matrix.platform }} + build-args: | + ONIX_VERSION=${{ env.ONIX_VERSION }} + GIT_COMMIT=${{ env.GIT_COMMIT }} + GIT_TREE_STATE=${{ env.GIT_TREE_STATE }} + BUILD_DATE=${{ env.BUILD_DATE }} + load: true + tags: ${{ env.IMAGE_NAME }}:${{ github.ref_name }}-${{ matrix.arch }} + + # trivy-release-gate exits 1 on a finding — the release gate, not just + # a report — before any push below runs. One scan covers every registry. + - name: Trivy image scan (gate) + run: make trivy-release-gate IMAGE=${{ env.IMAGE_NAME }}:${{ github.ref_name }}-${{ matrix.arch }} + + # Re-tag the already-scanned local image for each enabled registry and + # push it there. No rebuild, no re-scan. + - name: Tag and push + run: | + LOCAL="${{ env.IMAGE_NAME }}:${{ github.ref_name }}-${{ matrix.arch }}" + TAG="${{ github.ref_name }}-${{ matrix.arch }}" + + if [ "$GHCR_ENABLED" = "true" ]; then + DEST="ghcr.io/${OWNER}/${{ env.IMAGE_NAME }}:${TAG}" + docker tag "$LOCAL" "$DEST" + docker push "$DEST" + fi + if [ "$DOCKERHUB_ENABLED" = "true" ]; then + DEST="${{ secrets.DOCKERHUB_NAMESPACE }}/${{ env.IMAGE_NAME }}:${TAG}" + docker tag "$LOCAL" "$DEST" + docker push "$DEST" + fi + if [ "$AZURE_CONTAINER_REGISTRY_ENABLED" = "true" ]; then + DEST="${{ secrets.AZURE_REGISTRY_NAME }}/${{ env.IMAGE_NAME }}:${TAG}" + docker tag "$LOCAL" "$DEST" + docker push "$DEST" + fi + if [ "$GOOGLE_ARTIFACT_REGISTRY_ENABLED" = "true" ]; then + DEST="${{ secrets.GCP_REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ secrets.GOOGLE_ARTIFACT_REPO }}/${{ env.IMAGE_NAME }}:${TAG}" + docker tag "$LOCAL" "$DEST" + docker push "$DEST" + fi + + manifest: + runs-on: ubuntu-latest + needs: build + steps: + - uses: docker/setup-buildx-action@v3 + + - name: Set registry flags + run: | + echo "GHCR_ENABLED=${{ vars.GHCR_ENABLED != 'false' }}" >> "$GITHUB_ENV" + echo "DOCKERHUB_ENABLED=${{ vars.DOCKERHUB_ENABLED == 'true' }}" >> "$GITHUB_ENV" + echo "AZURE_CONTAINER_REGISTRY_ENABLED=${{ vars.AZURE_CONTAINER_REGISTRY_ENABLED == 'true' }}" >> "$GITHUB_ENV" + echo "GOOGLE_ARTIFACT_REGISTRY_ENABLED=${{ vars.GOOGLE_ARTIFACT_REGISTRY_ENABLED == 'true' }}" >> "$GITHUB_ENV" + + - name: Log in to GHCR + if: env.GHCR_ENABLED == 'true' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GHCR_PAT || secrets.GITHUB_TOKEN }} + + - name: Log in to Docker Hub + if: env.DOCKERHUB_ENABLED == 'true' + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Log in to Azure Container Registry + if: env.AZURE_CONTAINER_REGISTRY_ENABLED == 'true' + uses: docker/login-action@v3 + with: + registry: ${{ secrets.AZURE_REGISTRY_NAME }} + username: ${{ secrets.AZURE_REGISTRY_USERNAME }} + password: ${{ secrets.AZURE_REGISTRY_PASSWORD }} + + - name: Authenticate to Google Cloud + if: env.GOOGLE_ARTIFACT_REGISTRY_ENABLED == 'true' + uses: google-github-actions/auth@v3 + with: + token_format: "access_token" + workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_EMAIL }} + + - name: Configure Docker for Google Artifact Registry + if: env.GOOGLE_ARTIFACT_REGISTRY_ENABLED == 'true' + run: gcloud auth configure-docker ${{ secrets.GCP_REGION }}-docker.pkg.dev --quiet + + - name: Set image owner + run: echo "OWNER=$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_ENV" + + # imagetools create makes one manifest list from the two arch-specific + # images the build job just pushed to that registry — this is what + # lets `docker pull` resolve the right arch automatically on either + # amd64 or arm64. One create_manifest call per enabled registry. + - name: Create multi-arch manifests + run: | + create_manifest() { + local BASE="$1" + docker buildx imagetools create \ + -t "${BASE}:${{ github.ref_name }}" \ + -t "${BASE}:latest" \ + "${BASE}:${{ github.ref_name }}-amd64" \ + "${BASE}:${{ github.ref_name }}-arm64" + } + + if [ "$GHCR_ENABLED" = "true" ]; then + create_manifest "ghcr.io/${OWNER}/${{ env.IMAGE_NAME }}" + fi + if [ "$DOCKERHUB_ENABLED" = "true" ]; then + create_manifest "${{ secrets.DOCKERHUB_NAMESPACE }}/${{ env.IMAGE_NAME }}" + fi + if [ "$AZURE_CONTAINER_REGISTRY_ENABLED" = "true" ]; then + create_manifest "${{ secrets.AZURE_REGISTRY_NAME }}/${{ env.IMAGE_NAME }}" + fi + if [ "$GOOGLE_ARTIFACT_REGISTRY_ENABLED" = "true" ]; then + create_manifest "${{ secrets.GCP_REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ secrets.GOOGLE_ARTIFACT_REPO }}/${{ env.IMAGE_NAME }}" + fi diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 00000000..47f4136f --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,107 @@ +name: Run Tests + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + pull-requests: write + +concurrency: + group: test-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + # continue-on-error rather than dropping the job: CLAUDE.md keeps CI + # lint as the safety net behind the pre-commit hook, which is bypassable + # (--no-verify). Advisory here, not a gate — a red lint check must never + # be the reason a PR can't merge, only a reason someone looks. + - run: make lint + continue-on-error: true + + test: + runs-on: ubuntu-latest + steps: + # fetch-depth: 0 — cover-diff below needs the base branch's history + # locally to compute the PR's changed files; a PR checkout otherwise + # holds only the head commit. + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - run: make build + - run: make test-ci + + # continue-on-error so a coverage miss still posts its report; the last + # step in the job turns that outcome back into a failure. Always + # diff-scoped, never whole-repo — main only ever moves by PR merge + # (CLAUDE.md: never commit directly to main), so a PR's own diff is + # already the gate that matters. On a pull_request event that's the PR + # branch vs its target; on a push to main (the merge landing) it's + # github.event.before, the commit main pointed to right before this one. + - name: Check coverage + id: coverage + continue-on-error: true + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + make cover-diff BASE_REF="origin/${{ github.event.pull_request.base.ref }}" + else + make cover-diff BASE_REF="${{ github.event.before }}" + fi + + # Marker goes into the file itself, not just the step summary — it's + # what body-path feeds the PR comment below, and find-comment matches + # on this exact string to update rather than duplicate that comment. + - name: Publish coverage summary + run: | + printf '\n%s\n' "$(cat coverage-report.md)" > coverage-report.md + cat coverage-report.md >> "$GITHUB_STEP_SUMMARY" + + # Always posted, pass or fail or not-applicable — a comment that only + # shows up on failure is ambiguous from the PR itself: did coverage + # run at all, pass, or just never trigger? Always-visible with an + # explicit ✅/❌ (cover-diff's own output) answers that without a + # reviewer having to open the job. Updated in place across runs, not + # reposted, via find-comment below. + # + # continue-on-error on both: a fork PR's default GITHUB_TOKEN is + # read-only, so create-or-update-comment 403s there even with + # pull-requests: write declared. A comment that can't post must never + # fail the job — coverage already gated above, this step is cosmetic. + - name: Find existing coverage comment + if: github.event_name == 'pull_request' + continue-on-error: true + uses: peter-evans/find-comment@v3 + id: find-coverage-comment + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: "github-actions[bot]" + body-includes: "" + + - name: Comment coverage on PR + if: github.event_name == 'pull_request' + continue-on-error: true + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.find-coverage-comment.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body-path: coverage-report.md + edit-mode: replace + + - name: Gate on minimum coverage + if: steps.coverage.outcome == 'failure' + run: | + echo "::error::coverage is below the minimum — see the summary above" + exit 1 diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 00000000..f4a66049 --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,215 @@ +name: Security + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + # Findings are uploaded as SARIF for GitHub's own code scanning UI (Security + # tab, diff annotations) AND posted as a PR comment — the SARIF path alone + # took three clicks (check -> run -> results) to see a finding that used to + # be one scroll away in the PR conversation. pull-requests: write is for + # the comment; security-events: write is for the SARIF upload. + pull-requests: write + security-events: write + +concurrency: + group: security-${{ github.ref }} + cancel-in-progress: true + +jobs: + security: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # The trivy-action this replaced cached both the binary and its + # vulnerability DB across runs; `make trivy-deps`/`trivy-image` alone + # would re-download both (binary ~30MB, DB ~100MB+) on every run. + # Keyed on the Makefile's own hash so a TRIVY_VERSION bump there busts + # the cache automatically, plus a daily date so the DB itself doesn't + # go stale for longer than a day. + - name: Trivy cache date + id: trivy-date + run: echo "date=$(date -u +%F)" >> "$GITHUB_OUTPUT" + + - name: Cache Trivy binary and vulnerability DB + uses: actions/cache@v4 + with: + path: | + bin/trivy + ~/.cache/trivy + key: trivy-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Makefile') }}-${{ steps.trivy-date.outputs.date }} + restore-keys: | + trivy-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Makefile') }}- + + # T4, half one: the dependency graph as written. This catches what the + # image scan structurally cannot — a vulnerable module that only the test + # suite imports, so it is never linked into the binary and never appears + # in a layer. + # + # Logic lives in the Makefile's trivy-deps target — same command a local + # `make trivy-deps` runs. + - name: Trivy dependency scan (report) + run: make trivy-deps + + # A category per scan: without it the second upload replaces the first, + # because GitHub keys an analysis on (ref, category). + - name: Upload dependency findings + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: trivy-deps.sarif + category: trivy-dependencies + + # Posted right after the report, on every run — pass or fail, and + # before the image is even built — so a PR shows the dependency finding + # immediately rather than after the whole job finishes. Reads the same + # SARIF just uploaded above; no second scan. + # + # Trivy's SARIF has no structured per-field severity/version columns; + # each finding's detail lives as prose in message.text ("Package: + # ...\nSeverity: ...\n..."), so the table columns below are pulled out + # of that text rather than from dedicated JSON fields. + # continue-on-error: a report-formatting step must never be able to + # fail the job — the gate at the bottom is what decides that. + - name: Assemble Trivy dependency PR comment + if: github.event_name == 'pull_request' + continue-on-error: true + run: | + { + echo "" + echo "### 🛡️ Trivy — Go dependency scan (HIGH,CRITICAL)" + echo "[View full run](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" + echo + jq -r ' + def val(re; default): (capture(re) // {v: default}).v; + [ .runs[].results[]? ] as $found + | if ($found | length) == 0 then + "No HIGH or CRITICAL findings." + else + "| Package | Severity | Installed | Fixed in | Advisory |", + "|---|---|---|---|---|", + ( $found[] | .ruleId as $id | (.message.text // "") as $m | + "| `" + ($m | val("Package: (?[^\\n]+)"; "?")) + "` " + + "| " + ($m | val("Severity: (?[^\\n]+)"; "?")) + " " + + "| " + ($m | val("Installed Version: (?[^\\n]+)"; "?")) + " " + + "| " + ($m | val("Fixed Version: (?[^\\n]+)"; "—")) + " " + + "| [" + $id + "](" + ($m | val("Link: \\[[^]]+\\]\\((?[^)]+)\\)"; "")) + ") |" + ) + end + ' trivy-deps.sarif + } > trivy-deps-comment.md + + - name: Find existing Trivy dependency comment + if: github.event_name == 'pull_request' + continue-on-error: true + uses: peter-evans/find-comment@v3 + id: find-trivy-deps-comment + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: "github-actions[bot]" + body-includes: "" + + # continue-on-error: a fork PR's default GITHUB_TOKEN is read-only, so + # this 403s there even with pull-requests: write declared. The gate at + # the bottom of this job is what actually decides pass/fail; a comment + # that can't post must never fail the job over that. + - name: Comment Trivy dependency report on PR + if: github.event_name == 'pull_request' + continue-on-error: true + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.find-trivy-deps-comment.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body-path: trivy-deps-comment.md + edit-mode: replace + + - name: Log in to dhi.io + uses: ./.github/actions/dhi-login + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + token: ${{ secrets.DOCKERHUB_TOKEN }} + + # Both Dockerfiles default to dhi.io/* base images, and Trivy reads the + # base layers plus the Go build info embedded in the binary — including + # `stdlib`, so a Go toolchain CVE is reported here and nowhere else the + # dependency scan above cannot see. + - name: Build image + run: | + source install/scripts/version-vars.sh + docker build -f Dockerfile.adapter \ + --build-arg ONIX_VERSION="$ONIX_VERSION" \ + --build-arg GIT_COMMIT="$GIT_COMMIT" \ + --build-arg GIT_TREE_STATE="$GIT_TREE_STATE" \ + --build-arg BUILD_DATE="$BUILD_DATE" \ + -t network-adapter:${{ github.sha }} . + # Plain `docker build`, not build-push-action, matches security.yml's + # single-arch report scan — build-and-push.yml is where per-arch, + # buildx-driven builds actually matter. + + - name: Trivy image scan (report) + run: make trivy-image IMAGE=network-adapter:${{ github.sha }} + + - name: Upload image findings + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: trivy-image.sarif + category: trivy-image + + - name: Assemble Trivy image PR comment + if: github.event_name == 'pull_request' + continue-on-error: true + run: | + { + echo "" + echo "### 🛡️ Trivy — image scan (HIGH,CRITICAL)" + echo "[View full run](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" + echo + jq -r ' + def val(re; default): (capture(re) // {v: default}).v; + [ .runs[].results[]? ] as $found + | if ($found | length) == 0 then + "No HIGH or CRITICAL findings." + else + "| Package | Severity | Installed | Fixed in | Advisory |", + "|---|---|---|---|---|", + ( $found[] | .ruleId as $id | (.message.text // "") as $m | + "| `" + ($m | val("Package: (?[^\\n]+)"; "?")) + "` " + + "| " + ($m | val("Severity: (?[^\\n]+)"; "?")) + " " + + "| " + ($m | val("Installed Version: (?[^\\n]+)"; "?")) + " " + + "| " + ($m | val("Fixed Version: (?[^\\n]+)"; "—")) + " " + + "| [" + $id + "](" + ($m | val("Link: \\[[^]]+\\]\\((?[^)]+)\\)"; "")) + ") |" + ) + end + ' trivy-image.sarif + } > trivy-image-comment.md + + - name: Find existing Trivy image comment + if: github.event_name == 'pull_request' + continue-on-error: true + uses: peter-evans/find-comment@v3 + id: find-trivy-image-comment + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: "github-actions[bot]" + body-includes: "" + + - name: Comment Trivy image report on PR + if: github.event_name == 'pull_request' + continue-on-error: true + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.find-trivy-image-comment.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body-path: trivy-image-comment.md + edit-mode: replace + + # Gate on the reports already produced rather than scanning a third and + # fourth time — logic lives in the Makefile's trivy-gate target so a + # local `make trivy-gate` checks the exact same two files this job + # just wrote. Last in the job, so a red build still leaves both reports + # behind. + - name: Gate on HIGH or CRITICAL findings + run: make trivy-gate || { echo "::error::HIGH or CRITICAL Trivy findings — see job log above"; exit 1; } diff --git a/.gitignore b/.gitignore index 618e9b42..48d66b76 100644 --- a/.gitignore +++ b/.gitignore @@ -143,6 +143,14 @@ create_benchmark_issues.sh # Ignore coverage output files coverage.out coverage.html +coverage-report.md + +# Makefile tool/build artifacts (golangci-lint, gotestsum, trivy binaries) +/bin/ +trivy-deps.sarif +trivy-image.sarif +trivy-deps-comment.md +trivy-image-comment.md # Ignore the schema directory used for testing /schemas/ diff --git a/Dockerfile.adapter b/Dockerfile.adapter index 41dd7ca8..0b1fd641 100644 --- a/Dockerfile.adapter +++ b/Dockerfile.adapter @@ -1,4 +1,23 @@ -FROM golang:1.26.1-bookworm AS builder +# syntax=docker/dockerfile:1 + +# DHI (Docker Hardened Images) requires `docker login dhi.io` first — a Docker +# subscription credential this build cannot supply. Both stages default to it +# because it's what CI and every shipped image use, but a local build without +# that subscription can override both build-args with public equivalents: +# +# docker build -f Dockerfile.adapter \ +# --build-arg BUILD_IMAGE=golang:1.26.1-bookworm \ +# --build-arg RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base:latest \ +# -t network-adapter:dev . +# +# Tags verified against github.com/docker-hardened-images/catalog +# image/golang/debian-12/1.26-dev.yaml and image/debian-base/debian-12/12.yaml. +# No cgo here, but the -dev variant's gcc/g++/make come bundled regardless — +# nothing extra to install for either the default or the public override. +ARG BUILD_IMAGE=dhi.io/golang:1.26-debian12-dev +ARG RUNTIME_IMAGE=dhi.io/debian-base:bookworm + +FROM ${BUILD_IMAGE} AS builder WORKDIR /workspace/app COPY cmd/adapter ./cmd/adapter @@ -15,16 +34,15 @@ ARG BUILD_DATE=unknown RUN go build -ldflags "-X github.com/beckn-one/beckn-onix/pkg/version.Version=${ONIX_VERSION} -X github.com/beckn-one/beckn-onix/pkg/version.GitCommit=${GIT_COMMIT} -X github.com/beckn-one/beckn-onix/pkg/version.GitTreeState=${GIT_TREE_STATE} -X github.com/beckn-one/beckn-onix/pkg/version.BuildDate=${BUILD_DATE}" -o server cmd/adapter/main.go -# Create a minimal runtime image -FROM cgr.dev/chainguard/wolfi-base:latest -# ✅ Alpine is removed; using minimal Debian +# Debian-based hardened runtime, run-as nonroot by default — kept explicit +# below since RUNTIME_IMAGE can be overridden to wolfi-base, which uses the +# same uid 65532 convention. A shell is required here: CMD below needs +# `sh -c` to expand $CONFIG_FILE. +FROM ${RUNTIME_IMAGE} WORKDIR /app -# Copy only the built binary and plugin COPY --from=builder /workspace/app/server . -# Expose port 8080 EXPOSE 8080 - -# Run the Go server with the config flag from environment variable. -CMD ["sh", "-c", "./server --config=${CONFIG_FILE}"] \ No newline at end of file +USER 65532:65532 +CMD ["sh", "-c", "./server --config=${CONFIG_FILE}"] diff --git a/Dockerfile.adapter-with-plugins b/Dockerfile.adapter-with-plugins index a7ba5e98..de97fcd6 100644 --- a/Dockerfile.adapter-with-plugins +++ b/Dockerfile.adapter-with-plugins @@ -1,11 +1,30 @@ -FROM golang:1.26.1-bookworm AS builder +# syntax=docker/dockerfile:1 + +# DHI (Docker Hardened Images) requires `docker login dhi.io` first — a Docker +# subscription credential this build cannot supply. Both stages default to it +# because it's what CI and every shipped image use, but a local build without +# that subscription can override both build-args with public equivalents: +# +# docker build -f Dockerfile.adapter-with-plugins \ +# --build-arg BUILD_IMAGE=golang:1.26.1-bookworm \ +# --build-arg RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base:latest \ +# -t network-adapter-plugins:dev . +# +# Tags verified against github.com/docker-hardened-images/catalog +# image/golang/debian-12/1.26-dev.yaml and image/debian-base/debian-12/12.yaml. +# -buildmode=plugin needs a C toolchain regardless of base — the -dev +# variant's gcc/g++/make come bundled, same as the public bookworm override. +ARG BUILD_IMAGE=dhi.io/golang:1.26-debian12-dev +ARG RUNTIME_IMAGE=dhi.io/debian-base:bookworm + +FROM ${BUILD_IMAGE} AS builder WORKDIR /workspace/app # Specifically copy only the necessary files and directories COPY cmd/adapter/ ./cmd/adapter/ -COPY core/ ./core/ -COPY pkg/ ./pkg/ +COPY core/ ./core +COPY pkg/ ./pkg COPY install/build-plugins.sh ./install/build-plugins.sh COPY install/scripts/version-vars.sh ./install/scripts/version-vars.sh COPY go.mod . @@ -30,8 +49,11 @@ RUN chmod +x install/build-plugins.sh && \ ONIX_VERSION="${ONIX_VERSION}" GIT_COMMIT="${GIT_COMMIT}" GIT_TREE_STATE="${GIT_TREE_STATE}" BUILD_DATE="${BUILD_DATE}" \ ./install/build-plugins.sh -# Create minimal runtime image -FROM cgr.dev/chainguard/wolfi-base:latest +# Debian-based hardened runtime, run-as nonroot by default — kept explicit +# below since RUNTIME_IMAGE can be overridden to wolfi-base, which uses the +# same uid 65532 convention. A shell is required here: CMD below needs +# `sh -c` to expand $CONFIG_FILE. +FROM ${RUNTIME_IMAGE} WORKDIR /app # Copy binary and plugins built with same Go version @@ -39,5 +61,5 @@ COPY --from=builder /workspace/app/server . COPY --from=builder /workspace/app/plugins ./plugins EXPOSE 8081 - +USER 65532:65532 CMD ["sh", "-c", "./server --config=${CONFIG_FILE}"] diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..0bd4ab21 --- /dev/null +++ b/Makefile @@ -0,0 +1,175 @@ +# OAN Network Adapter — build, test and toolchain targets. +# +# Mirrors discovery-service's Makefile shape (same target names, same +# run-tests/security/build-and-push workflows call these, not raw commands), +# adapted to this module: no DB, no sqlc/migrate, no separate tools/ module — +# golangci-lint, gotestsum and trivy install straight into bin/ via `go +# install`/curl, same as discovery-service does for gotestsum and trivy. + +GO ?= go +BIN_DIR := bin +IMAGE ?= network-adapter:dev + +# CI thresholds/pins live here, not duplicated into workflow env blocks — one +# source of truth for both a local `make` run and the GitHub Actions runner. +MIN_COVERAGE ?= 80 +BASE_REF ?= origin/main +SEVERITY ?= HIGH,CRITICAL +GOLANGCI_LINT_VERSION := v2.5.0 +GOTESTSUM_VERSION := v1.13.0 +TRIVY_VERSION := v0.74.0 + +GOLANGCI_LINT := $(BIN_DIR)/golangci-lint +GOTESTSUM := $(BIN_DIR)/gotestsum +TRIVY := $(BIN_DIR)/trivy + +.DEFAULT_GOAL := help + +## help: list the available targets +help: + @grep -hE '^## ' $(MAKEFILE_LIST) | sed 's/^## / /' | sort + +## build: compile the adapter binary +# Scoped to cmd/adapter, not ./... — pkg/plugin/implementation/*/cmd holds +# `package main` sources meant only for `go build -buildmode=plugin` +# (install/build-plugins.sh), with no func main() for an ordinary build. +build: + $(GO) build -trimpath -o $(BIN_DIR)/ ./cmd/adapter/... + +## test: run the unit and integration suites +test: + $(GO) test -race ./... + +## cover: run the suites and write a coverage profile +cover: + $(GO) test -race -covermode=atomic -coverprofile=coverage.out ./... + +## test-ci: run the suites through gotestsum — one line per package, coverage +## profile written alongside. What run-tests.yml calls; `make test` +## stays the plain everyday entrypoint. +test-ci: $(GOTESTSUM) + $(GOTESTSUM) --format pkgname --format-hide-empty-pkg -- \ + -race -coverprofile=coverage.out -covermode=atomic ./... + +## cover-diff: coverage restricted to files changed vs BASE_REF — a PR review +## needs the diff's number, not the whole repo's. On failure, +## names the changed files dragging the number down (worst first). +cover-diff: coverage.out + @CHANGED=$$(git diff --name-only --diff-filter=ACMR "$(BASE_REF)...HEAD" -- '*.go' | grep -v '_test\.go$$' || true); \ + if [ -z "$$CHANGED" ]; then \ + echo "📊 **Test Coverage: ✅ Passed** — not applicable, no changed Go files vs $(BASE_REF)" | tee coverage-report.md; \ + exit 0; \ + fi; \ + MODULE=$$($(GO) list -m); \ + RESULT=$$(echo "$$CHANGED" | awk -v mod="$$MODULE/" -v min="$(MIN_COVERAGE)" ' \ + NR==FNR { want[mod $$0] = 1; next } \ + { f = $$1; sub(/:.*/, "", f); if (!(f in want)) next; \ + tot[f] += $$(NF-1); if ($$NF > 0) cov[f] += $$(NF-1) } \ + END { \ + T = 0; C = 0; \ + for (f in tot) { \ + T += tot[f]; C += cov[f]; \ + p = int(cov[f] * 100 / tot[f]); \ + disp = f; sub("^" mod, "", disp); \ + if (p < min) print "FILE\t" p "\t" disp; \ + } \ + if (T == 0) { print "EMPTY"; exit } \ + print "TOTAL\t" int(C * 100 / T) \ + }' - coverage.out); \ + if echo "$$RESULT" | grep -q '^EMPTY$$'; then \ + echo "📊 **Test Coverage: ✅ Passed** — not applicable, changed files carry no coverable statements" | tee coverage-report.md; \ + exit 0; \ + fi; \ + PCT=$$(echo "$$RESULT" | awk -F'\t' '$$1=="TOTAL"{print $$2}'); \ + if [ "$$PCT" -lt "$(MIN_COVERAGE)" ]; then \ + BELOW=$$(echo "$$RESULT" | awk -F'\t' '$$1=="FILE"{printf "%s\t%s\n",$$2,$$3}' | sort -n); \ + TOTAL_BELOW=$$(echo "$$BELOW" | wc -l); \ + { \ + echo "📊 **Test Coverage: ❌ Failed** — $${PCT}% of changed lines covered, min $(MIN_COVERAGE)%"; \ + echo; \ + echo "| File | Coverage |"; \ + echo "|---|---|"; \ + echo "$$BELOW" | head -15 | awk -F'\t' '{printf "| `%s` | %s%% |\n", $$2, $$1}'; \ + [ "$$TOTAL_BELOW" -gt 15 ] && echo "| … | $$((TOTAL_BELOW - 15)) more file(s) below $(MIN_COVERAGE)% |"; \ + } > coverage-report.md; \ + else \ + echo "📊 **Test Coverage: ✅ Passed** — $${PCT}% of changed lines covered, min $(MIN_COVERAGE)%" > coverage-report.md; \ + fi; \ + cat coverage-report.md; \ + [ "$$PCT" -ge "$(MIN_COVERAGE)" ] + +## trivy-deps: dependency graph scan (T4), SARIF report. Catches what the +## image scan structurally cannot — a vulnerable module only the +## test suite imports, so it's never linked into the binary. +trivy-deps: $(TRIVY) + $(TRIVY) fs . --severity $(SEVERITY) --exit-code 0 \ + --format sarif --output trivy-deps.sarif + +TRIVY_IMAGE_SCAN = $(TRIVY) image $(IMAGE) --severity $(SEVERITY) + +## trivy-image: shipped image scan (T4), SARIF report. IMAGE names the ref. +trivy-image: $(TRIVY) + $(TRIVY_IMAGE_SCAN) --exit-code 0 --format sarif --output trivy-image.sarif + +## trivy-release-gate: same image scan as trivy-image, but exit 1 on a +## finding instead of writing a report — the pre-push +## release gate build-and-push.yml runs once per +## arch-tagged local image, before anything is pushed. +trivy-release-gate: $(TRIVY) + $(TRIVY_IMAGE_SCAN) --exit-code 1 --format table + +## trivy-gate: fail if either SARIF report already produced by a scan step +## carries a finding. Reads the reports rather than rescanning. +trivy-gate: + @fail=0; \ + for report in trivy-deps.sarif trivy-image.sarif; do \ + count=$$(jq '[.runs[].results[]?] | length' "$$report"); \ + echo "$${report}: $${count} $(SEVERITY)"; \ + if [ "$$count" -gt 0 ]; then \ + jq -r '.runs[].results[]? | "\(.ruleId) \(.message.text)"' "$$report"; \ + fail=1; \ + fi; \ + done; \ + exit $$fail + +## lint: vet, format check and static analysis +lint: $(GOLANGCI_LINT) + $(GOLANGCI_LINT) run ./... + $(GOLANGCI_LINT) fmt --diff ./... + +## fmt: apply the formatters lint checks for +fmt: $(GOLANGCI_LINT) + $(GOLANGCI_LINT) fmt ./... + +## docker: build the adapter image +docker: + docker build -f Dockerfile.adapter -t $(IMAGE) . + +## tools: build the pinned toolchain into bin/ +tools: $(GOLANGCI_LINT) $(GOTESTSUM) $(TRIVY) + +## clean: remove build output and coverage/scan artifacts +clean: + rm -rf $(BIN_DIR) coverage.out coverage-report.md trivy-deps.sarif trivy-image.sarif + +$(GOLANGCI_LINT): + @mkdir -p $(BIN_DIR) + GOBIN=$(abspath $(BIN_DIR)) $(GO) install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) + +# gotestsum is CI-only (see run-tests.yml), so it doesn't belong in the +# adapter's or the linter's dependency graph either one. +$(GOTESTSUM): + @mkdir -p $(BIN_DIR) + GOBIN=$(abspath $(BIN_DIR)) $(GO) install gotest.tools/gotestsum@$(GOTESTSUM_VERSION) + +# The prebuilt release binary, not `go install`: trivy's rpm-db parser needs +# cgo, and its module graph is comparable in size to golangci-lint's for a +# tool nothing here imports — the official install script is what +# aquasecurity itself recommends over building from source for exactly this. +$(TRIVY): + @mkdir -p $(BIN_DIR) + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | \ + sh -s -- -b $(abspath $(BIN_DIR)) $(TRIVY_VERSION) + +.PHONY: help build test cover test-ci cover-diff lint fmt trivy-deps \ + trivy-image trivy-release-gate trivy-gate docker tools clean From 8f6092ded57dcfa46a56a3c2fa3a8254d4d8cc0c Mon Sep 17 00:00:00 2001 From: nisargabd Date: Thu, 3 Sep 2026 15:32:42 +0530 Subject: [PATCH 002/130] fix: switch adapter runtime to a glibc-only DHI image, drop the shell dependency [#5] security.yml's Trivy gate failed on unpatched CVEs in dhi.io/debian-base's bundled perl-base, ncurses, util-linux and zlib1g -- none of which the adapter uses. Neither can move to the fully static dhi.io/static image though: the adapter loads plugin .so files at runtime via Go's `plugin` package, which needs cgo and a real glibc dynamic linker to work at all. dhi.io/static's glibc variant is the middle ground -- keeps libc6 so plugin.Open still works, drops everything else the scan was flagging. That variant has no shell, so CMD's `sh -c "./server --config=$CONFIG_FILE"` can no longer expand the env var. Switched to an exec-form ENTRYPOINT and taught main.go to read CONFIG_FILE itself as the --config default. --- Dockerfile.adapter | 23 +++++++++++++++-------- Dockerfile.adapter-with-plugins | 20 ++++++++++++-------- cmd/adapter/main.go | 11 +++++++++-- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/Dockerfile.adapter b/Dockerfile.adapter index 0b1fd641..b3df00e8 100644 --- a/Dockerfile.adapter +++ b/Dockerfile.adapter @@ -11,11 +11,15 @@ # -t network-adapter:dev . # # Tags verified against github.com/docker-hardened-images/catalog -# image/golang/debian-12/1.26-dev.yaml and image/debian-base/debian-12/12.yaml. -# No cgo here, but the -dev variant's gcc/g++/make come bundled regardless — -# nothing extra to install for either the default or the public override. +# image/golang/debian-12/1.26-dev.yaml and image/static/debian-12/static-glibc.yaml. +# No cgo *required* for the plain build, but the adapter loads plugin .so +# files at runtime via Go's `plugin` package, which needs cgo and a real +# glibc dynamic linker regardless — so this stays a normal dynamic build, not +# -extldflags '-static'. The -dev variant's gcc/g++/make come bundled +# regardless — nothing extra to install for either the default or the public +# override. ARG BUILD_IMAGE=dhi.io/golang:1.26-debian12-dev -ARG RUNTIME_IMAGE=dhi.io/debian-base:bookworm +ARG RUNTIME_IMAGE=dhi.io/static:20250419-glibc-debian12 FROM ${BUILD_IMAGE} AS builder @@ -34,10 +38,11 @@ ARG BUILD_DATE=unknown RUN go build -ldflags "-X github.com/beckn-one/beckn-onix/pkg/version.Version=${ONIX_VERSION} -X github.com/beckn-one/beckn-onix/pkg/version.GitCommit=${GIT_COMMIT} -X github.com/beckn-one/beckn-onix/pkg/version.GitTreeState=${GIT_TREE_STATE} -X github.com/beckn-one/beckn-onix/pkg/version.BuildDate=${BUILD_DATE}" -o server cmd/adapter/main.go -# Debian-based hardened runtime, run-as nonroot by default — kept explicit +# glibc-only runtime — no shell, no perl/ncurses/util-linux (the packages +# that were carrying unpatched CVEs on the fuller debian-base image), but +# keeps libc6 so plugin.Open's dynamic linking still works. Explicit USER # below since RUNTIME_IMAGE can be overridden to wolfi-base, which uses the -# same uid 65532 convention. A shell is required here: CMD below needs -# `sh -c` to expand $CONFIG_FILE. +# same uid 65532 convention. FROM ${RUNTIME_IMAGE} WORKDIR /app @@ -45,4 +50,6 @@ COPY --from=builder /workspace/app/server . EXPOSE 8080 USER 65532:65532 -CMD ["sh", "-c", "./server --config=${CONFIG_FILE}"] +# Exec form, not `sh -c` — this base has no shell. main.go reads CONFIG_FILE +# itself as the --config default, so no shell expansion is needed here. +ENTRYPOINT ["/app/server"] diff --git a/Dockerfile.adapter-with-plugins b/Dockerfile.adapter-with-plugins index de97fcd6..043a91e6 100644 --- a/Dockerfile.adapter-with-plugins +++ b/Dockerfile.adapter-with-plugins @@ -11,11 +11,12 @@ # -t network-adapter-plugins:dev . # # Tags verified against github.com/docker-hardened-images/catalog -# image/golang/debian-12/1.26-dev.yaml and image/debian-base/debian-12/12.yaml. -# -buildmode=plugin needs a C toolchain regardless of base — the -dev -# variant's gcc/g++/make come bundled, same as the public bookworm override. +# image/golang/debian-12/1.26-dev.yaml and image/static/debian-12/static-glibc.yaml. +# -buildmode=plugin, and plugin.Open at runtime, both need a C toolchain and +# a real glibc dynamic linker — the -dev variant's gcc/g++/make come +# bundled, same as the public bookworm override. ARG BUILD_IMAGE=dhi.io/golang:1.26-debian12-dev -ARG RUNTIME_IMAGE=dhi.io/debian-base:bookworm +ARG RUNTIME_IMAGE=dhi.io/static:20250419-glibc-debian12 FROM ${BUILD_IMAGE} AS builder @@ -49,10 +50,11 @@ RUN chmod +x install/build-plugins.sh && \ ONIX_VERSION="${ONIX_VERSION}" GIT_COMMIT="${GIT_COMMIT}" GIT_TREE_STATE="${GIT_TREE_STATE}" BUILD_DATE="${BUILD_DATE}" \ ./install/build-plugins.sh -# Debian-based hardened runtime, run-as nonroot by default — kept explicit +# glibc-only runtime — no shell, no perl/ncurses/util-linux (the packages +# that were carrying unpatched CVEs on the fuller debian-base image), but +# keeps libc6 so plugin.Open's dynamic linking still works. Explicit USER # below since RUNTIME_IMAGE can be overridden to wolfi-base, which uses the -# same uid 65532 convention. A shell is required here: CMD below needs -# `sh -c` to expand $CONFIG_FILE. +# same uid 65532 convention. FROM ${RUNTIME_IMAGE} WORKDIR /app @@ -62,4 +64,6 @@ COPY --from=builder /workspace/app/plugins ./plugins EXPOSE 8081 USER 65532:65532 -CMD ["sh", "-c", "./server --config=${CONFIG_FILE}"] +# Exec form, not `sh -c` — this base has no shell. main.go reads CONFIG_FILE +# itself as the --config default, so no shell expansion is needed here. +ENTRYPOINT ["/app/server"] diff --git a/cmd/adapter/main.go b/cmd/adapter/main.go index f74d6fb0..09a88226 100644 --- a/cmd/adapter/main.go +++ b/cmd/adapter/main.go @@ -63,8 +63,15 @@ var configPath string var runFunc = run func main() { - // Define and parse command-line flags. - flag.StringVar(&configPath, "config", "../../config/onix/adapter.yaml", "Path to the configuration file") + // Define and parse command-line flags. CONFIG_FILE seeds the default so + // a shell-less container entrypoint (no `sh -c` to expand it into + // --config) can still select a config file via env var; --config still + // overrides it explicitly when passed. + defaultConfig := "../../config/onix/adapter.yaml" + if v := os.Getenv("CONFIG_FILE"); v != "" { + defaultConfig = v + } + flag.StringVar(&configPath, "config", defaultConfig, "Path to the configuration file") flag.Parse() // Use custom log for initial setup messages. From 6e2f9b8542a9d81919c0905f4e5913392200001b Mon Sep 17 00:00:00 2001 From: nisargabd Date: Thu, 3 Sep 2026 15:39:33 +0530 Subject: [PATCH 003/130] fix: bump golang.org/x/crypto and google.golang.org/grpc to close known CVEs [#5] security.yml's trivy-deps scan flagged CVE-2026-56854 (x/crypto, CRITICAL) and CVE-2026-84304 (grpc, HIGH), both with fixed versions already published upstream. --- go.mod | 22 +++++++++++----------- go.sum | 54 ++++++++++++++++++++++++++++-------------------------- 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/go.mod b/go.mod index fe8b8670..63fed0f4 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.26.1 require ( github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 - golang.org/x/crypto v0.54.0 + golang.org/x/crypto v0.55.0 ) require github.com/stretchr/testify v1.11.1 @@ -19,7 +19,7 @@ require ( require github.com/zenazn/pkcs7pad v0.0.0-20170308005700-253a5b1f0e03 -require golang.org/x/text v0.40.0 // indirect +require golang.org/x/text v0.41.0 // indirect require ( github.com/agnivade/levenshtein v1.2.1 // indirect @@ -80,11 +80,11 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/net v0.56.0 // indirect + golang.org/x/net v0.57.0 // indirect golang.org/x/sys v0.47.0 // indirect golang.org/x/time v0.15.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260427160629-7cedc36a6bc4 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/protobuf v1.36.11 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) @@ -107,20 +107,20 @@ require ( github.com/redis/go-redis/v9 v9.19.0 github.com/rs/zerolog v1.35.1 go.opentelemetry.io/contrib/instrumentation/runtime v0.68.0 - go.opentelemetry.io/otel v1.43.0 + go.opentelemetry.io/otel v1.44.0 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.19.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.43.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0 go.opentelemetry.io/otel/exporters/prometheus v0.65.0 go.opentelemetry.io/otel/log v0.19.0 - go.opentelemetry.io/otel/metric v1.43.0 - go.opentelemetry.io/otel/sdk v1.43.0 + go.opentelemetry.io/otel/metric v1.44.0 + go.opentelemetry.io/otel/sdk v1.44.0 go.opentelemetry.io/otel/sdk/log v0.19.0 - go.opentelemetry.io/otel/sdk/metric v1.43.0 - go.opentelemetry.io/otel/trace v1.43.0 + go.opentelemetry.io/otel/sdk/metric v1.44.0 + go.opentelemetry.io/otel/trace v1.44.0 go.uber.org/automaxprocs v1.6.0 golang.org/x/sync v0.22.0 - google.golang.org/grpc v1.82.1 + google.golang.org/grpc v1.83.1 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 055511f8..3f7b7a09 100644 --- a/go.sum +++ b/go.sum @@ -219,8 +219,8 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/runtime v0.68.0 h1:jhVIQEprwUTV+KfzzliLidclhoTOoHTgdz96kAyR8mU= go.opentelemetry.io/contrib/instrumentation/runtime v0.68.0/go.mod h1:4HsdbLUbernaTnA8CNaNE+1g026SciXb3juRYe3l8EY= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.19.0 h1:Dn8rkudDzY6KV9dr/D/bTUuWgqDf9xe0rr4G2elrn0Y= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.19.0/go.mod h1:gMk9F0xDgyN9M/3Ed5Y1wKcx/9mlU91NXY2SNq7RQuU= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.43.0 h1:8UQVDcZxOJLtX6gxtDt3vY2WTgvZqMQRzjsqiIHQdkc= @@ -233,18 +233,20 @@ go.opentelemetry.io/otel/exporters/prometheus v0.65.0 h1:jOveH/b4lU9HT7y+Gfamf18 go.opentelemetry.io/otel/exporters/prometheus v0.65.0/go.mod h1:i1P8pcumauPtUI4YNopea1dhzEMuEqWP1xoUZDylLHo= go.opentelemetry.io/otel/log v0.19.0 h1:KUZs/GOsw79TBBMfDWsXS+KZ4g2Ckzksd1ymzsIEbo4= go.opentelemetry.io/otel/log v0.19.0/go.mod h1:5DQYeGmxVIr4n0/BcJvF4upsraHjg6vudJJpnkL6Ipk= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA= +go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= go.opentelemetry.io/otel/sdk/log v0.19.0 h1:scYVLqT22D2gqXItnWiocLUKGH9yvkkeql5dBDiXyko= go.opentelemetry.io/otel/sdk/log v0.19.0/go.mod h1:vFBowwXGLlW9AvpuF7bMgnNI95LiW10szrOdvzBHlAg= go.opentelemetry.io/otel/sdk/log/logtest v0.19.0 h1:BEbF7ZBB6qQloV/Ub1+3NQoOUnVtcGkU3XX4Ws3GQfk= go.opentelemetry.io/otel/sdk/log/logtest v0.19.0/go.mod h1:Lua81/3yM0wOmoHTokLj9y9ADeA02v1naRrVrkAZuKk= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= @@ -257,30 +259,30 @@ go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= -golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= -golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= -golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= -golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= -golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260427160629-7cedc36a6bc4 h1:yOzSCGPx+cp5VO7IxvZ9SBFF7j1tZVcNtlHR2iYKtVo= -google.golang.org/genproto/googleapis/api v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:Q9HWtNeE7tM9npdIsEvqXj1QJIvVoeAV3rtXtS715Cw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 h1:tEkOQcXgF6dH1G+MVKZrfpYvozGrzb91k6ha7jireSM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= -google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= +google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 7acedd871c74447635f7668c17994b713271b75d Mon Sep 17 00:00:00 2001 From: nisargabd Date: Thu, 3 Sep 2026 15:56:01 +0530 Subject: [PATCH 004/130] fix: disable Terraform deploy's push trigger [#5] Ran on every push including this branch and always failed here -- missing Gerrit credentials for the private Terraform-CICD repo it clones. workflow_dispatch stays, so it's still runnable by hand; kept, not deleted, pending a decision on whether it's folded into build-and-push.yml or removed outright. --- .github/workflows/onix-gcp-terraform-deploy.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/onix-gcp-terraform-deploy.yml b/.github/workflows/onix-gcp-terraform-deploy.yml index 478979be..4d35fe94 100644 --- a/.github/workflows/onix-gcp-terraform-deploy.yml +++ b/.github/workflows/onix-gcp-terraform-deploy.yml @@ -1,7 +1,9 @@ name: Terraform Deploy to GCP on: - push: + # push trigger disabled — this always fails (missing Gerrit credentials in + # this repo) and ran on every push, including unrelated branches. Kept as + # manual-only until it's confirmed dead or folded into build-and-push.yml. workflow_dispatch: # Manual triggerr jobs: From 3edaf4236da86d6f1928e502fde3311db8d08748 Mon Sep 17 00:00:00 2001 From: nisargabd Date: Thu, 3 Sep 2026 16:37:32 +0530 Subject: [PATCH 005/130] chore: keep golang.org/x/crypto and google.golang.org/grpc at prior versions [#5] This is CI setup, not a dependency-bump PR -- the two CVEs trivy-deps flags (CVE-2026-56854, CVE-2026-84304) are real and the scan catching them is working as intended; the actual bump is deferred to its own change. --- go.mod | 22 +++++++++++----------- go.sum | 54 ++++++++++++++++++++++++++---------------------------- 2 files changed, 37 insertions(+), 39 deletions(-) diff --git a/go.mod b/go.mod index 63fed0f4..fe8b8670 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.26.1 require ( github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 - golang.org/x/crypto v0.55.0 + golang.org/x/crypto v0.54.0 ) require github.com/stretchr/testify v1.11.1 @@ -19,7 +19,7 @@ require ( require github.com/zenazn/pkcs7pad v0.0.0-20170308005700-253a5b1f0e03 -require golang.org/x/text v0.41.0 // indirect +require golang.org/x/text v0.40.0 // indirect require ( github.com/agnivade/levenshtein v1.2.1 // indirect @@ -80,11 +80,11 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/net v0.57.0 // indirect + golang.org/x/net v0.56.0 // indirect golang.org/x/sys v0.47.0 // indirect golang.org/x/time v0.15.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260427160629-7cedc36a6bc4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect google.golang.org/protobuf v1.36.11 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) @@ -107,20 +107,20 @@ require ( github.com/redis/go-redis/v9 v9.19.0 github.com/rs/zerolog v1.35.1 go.opentelemetry.io/contrib/instrumentation/runtime v0.68.0 - go.opentelemetry.io/otel v1.44.0 + go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.19.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.43.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0 go.opentelemetry.io/otel/exporters/prometheus v0.65.0 go.opentelemetry.io/otel/log v0.19.0 - go.opentelemetry.io/otel/metric v1.44.0 - go.opentelemetry.io/otel/sdk v1.44.0 + go.opentelemetry.io/otel/metric v1.43.0 + go.opentelemetry.io/otel/sdk v1.43.0 go.opentelemetry.io/otel/sdk/log v0.19.0 - go.opentelemetry.io/otel/sdk/metric v1.44.0 - go.opentelemetry.io/otel/trace v1.44.0 + go.opentelemetry.io/otel/sdk/metric v1.43.0 + go.opentelemetry.io/otel/trace v1.43.0 go.uber.org/automaxprocs v1.6.0 golang.org/x/sync v0.22.0 - google.golang.org/grpc v1.83.1 + google.golang.org/grpc v1.82.1 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 3f7b7a09..055511f8 100644 --- a/go.sum +++ b/go.sum @@ -219,8 +219,8 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/runtime v0.68.0 h1:jhVIQEprwUTV+KfzzliLidclhoTOoHTgdz96kAyR8mU= go.opentelemetry.io/contrib/instrumentation/runtime v0.68.0/go.mod h1:4HsdbLUbernaTnA8CNaNE+1g026SciXb3juRYe3l8EY= -go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= -go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.19.0 h1:Dn8rkudDzY6KV9dr/D/bTUuWgqDf9xe0rr4G2elrn0Y= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.19.0/go.mod h1:gMk9F0xDgyN9M/3Ed5Y1wKcx/9mlU91NXY2SNq7RQuU= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.43.0 h1:8UQVDcZxOJLtX6gxtDt3vY2WTgvZqMQRzjsqiIHQdkc= @@ -233,20 +233,18 @@ go.opentelemetry.io/otel/exporters/prometheus v0.65.0 h1:jOveH/b4lU9HT7y+Gfamf18 go.opentelemetry.io/otel/exporters/prometheus v0.65.0/go.mod h1:i1P8pcumauPtUI4YNopea1dhzEMuEqWP1xoUZDylLHo= go.opentelemetry.io/otel/log v0.19.0 h1:KUZs/GOsw79TBBMfDWsXS+KZ4g2Ckzksd1ymzsIEbo4= go.opentelemetry.io/otel/log v0.19.0/go.mod h1:5DQYeGmxVIr4n0/BcJvF4upsraHjg6vudJJpnkL6Ipk= -go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= -go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= -go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA= -go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk= -go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= -go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= go.opentelemetry.io/otel/sdk/log v0.19.0 h1:scYVLqT22D2gqXItnWiocLUKGH9yvkkeql5dBDiXyko= go.opentelemetry.io/otel/sdk/log v0.19.0/go.mod h1:vFBowwXGLlW9AvpuF7bMgnNI95LiW10szrOdvzBHlAg= go.opentelemetry.io/otel/sdk/log/logtest v0.19.0 h1:BEbF7ZBB6qQloV/Ub1+3NQoOUnVtcGkU3XX4Ws3GQfk= go.opentelemetry.io/otel/sdk/log/logtest v0.19.0/go.mod h1:Lua81/3yM0wOmoHTokLj9y9ADeA02v1naRrVrkAZuKk= -go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= -go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= -go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= -go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= @@ -259,30 +257,30 @@ go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= -golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= -golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= -golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= -golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= -golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= -google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= -google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/genproto/googleapis/api v0.0.0-20260427160629-7cedc36a6bc4 h1:yOzSCGPx+cp5VO7IxvZ9SBFF7j1tZVcNtlHR2iYKtVo= +google.golang.org/genproto/googleapis/api v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:Q9HWtNeE7tM9npdIsEvqXj1QJIvVoeAV3rtXtS715Cw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 h1:tEkOQcXgF6dH1G+MVKZrfpYvozGrzb91k6ha7jireSM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From e45b7bf91fa21c8627c6cb4063376f9cf15e8938 Mon Sep 17 00:00:00 2001 From: nisargabd Date: Thu, 3 Sep 2026 17:19:06 +0530 Subject: [PATCH 006/130] fix: scope the DHI runtime switch to the plugins image only [#5] Dockerfile.adapter goes back to the original public base images -- Dockerfile.adapter-with-plugins is the one actually deployed, so that's the one that needs the CVE fix. Keeps dhi.io/static's glibc-only variant (clears the unpatched perl-base/ncurses/util-linux/zlib1g CVEs, keeps libc6 so plugin.Open still works) but drops the main.go change entirely: Deployment/deployment.yaml now passes --config=$(CONFIG_FILE) via args, letting the kubelet do the substitution instead of a shell inside the container. --- Deployment/deployment.yaml | 5 +++++ Dockerfile.adapter | 23 ++++++++--------------- Dockerfile.adapter-with-plugins | 7 +++++-- cmd/adapter/main.go | 11 ++--------- 4 files changed, 20 insertions(+), 26 deletions(-) diff --git a/Deployment/deployment.yaml b/Deployment/deployment.yaml index 561fdb5a..443ac03c 100644 --- a/Deployment/deployment.yaml +++ b/Deployment/deployment.yaml @@ -19,6 +19,11 @@ spec: containers: - name: onix-adapter image: "asia-south1-docker.pkg.dev/trusty-relic-370809/onix-adapter-cicd/beckn-onix:latest" #------ + # $(CONFIG_FILE) is expanded by the kubelet against the env below + # before the container starts -- the image's shell-less DHI base + # has no `sh -c` to do that substitution itself. + args: + - "--config=$(CONFIG_FILE)" ports: - containerPort: 8080 env: diff --git a/Dockerfile.adapter b/Dockerfile.adapter index b3df00e8..0b1fd641 100644 --- a/Dockerfile.adapter +++ b/Dockerfile.adapter @@ -11,15 +11,11 @@ # -t network-adapter:dev . # # Tags verified against github.com/docker-hardened-images/catalog -# image/golang/debian-12/1.26-dev.yaml and image/static/debian-12/static-glibc.yaml. -# No cgo *required* for the plain build, but the adapter loads plugin .so -# files at runtime via Go's `plugin` package, which needs cgo and a real -# glibc dynamic linker regardless — so this stays a normal dynamic build, not -# -extldflags '-static'. The -dev variant's gcc/g++/make come bundled -# regardless — nothing extra to install for either the default or the public -# override. +# image/golang/debian-12/1.26-dev.yaml and image/debian-base/debian-12/12.yaml. +# No cgo here, but the -dev variant's gcc/g++/make come bundled regardless — +# nothing extra to install for either the default or the public override. ARG BUILD_IMAGE=dhi.io/golang:1.26-debian12-dev -ARG RUNTIME_IMAGE=dhi.io/static:20250419-glibc-debian12 +ARG RUNTIME_IMAGE=dhi.io/debian-base:bookworm FROM ${BUILD_IMAGE} AS builder @@ -38,11 +34,10 @@ ARG BUILD_DATE=unknown RUN go build -ldflags "-X github.com/beckn-one/beckn-onix/pkg/version.Version=${ONIX_VERSION} -X github.com/beckn-one/beckn-onix/pkg/version.GitCommit=${GIT_COMMIT} -X github.com/beckn-one/beckn-onix/pkg/version.GitTreeState=${GIT_TREE_STATE} -X github.com/beckn-one/beckn-onix/pkg/version.BuildDate=${BUILD_DATE}" -o server cmd/adapter/main.go -# glibc-only runtime — no shell, no perl/ncurses/util-linux (the packages -# that were carrying unpatched CVEs on the fuller debian-base image), but -# keeps libc6 so plugin.Open's dynamic linking still works. Explicit USER +# Debian-based hardened runtime, run-as nonroot by default — kept explicit # below since RUNTIME_IMAGE can be overridden to wolfi-base, which uses the -# same uid 65532 convention. +# same uid 65532 convention. A shell is required here: CMD below needs +# `sh -c` to expand $CONFIG_FILE. FROM ${RUNTIME_IMAGE} WORKDIR /app @@ -50,6 +45,4 @@ COPY --from=builder /workspace/app/server . EXPOSE 8080 USER 65532:65532 -# Exec form, not `sh -c` — this base has no shell. main.go reads CONFIG_FILE -# itself as the --config default, so no shell expansion is needed here. -ENTRYPOINT ["/app/server"] +CMD ["sh", "-c", "./server --config=${CONFIG_FILE}"] diff --git a/Dockerfile.adapter-with-plugins b/Dockerfile.adapter-with-plugins index 043a91e6..6f72a4c0 100644 --- a/Dockerfile.adapter-with-plugins +++ b/Dockerfile.adapter-with-plugins @@ -64,6 +64,9 @@ COPY --from=builder /workspace/app/plugins ./plugins EXPOSE 8081 USER 65532:65532 -# Exec form, not `sh -c` — this base has no shell. main.go reads CONFIG_FILE -# itself as the --config default, so no shell expansion is needed here. +# Exec form, not `sh -c` — this base has no shell, and no Go code reads +# CONFIG_FILE either. The deployment layer resolves it instead: Kubernetes +# expands $(CONFIG_FILE) in args itself before exec'ing the container (see +# Deployment/deployment.yaml) -- no shell involved on either side. A plain +# `docker run` without that substitution needs an explicit --config flag. ENTRYPOINT ["/app/server"] diff --git a/cmd/adapter/main.go b/cmd/adapter/main.go index 09a88226..f74d6fb0 100644 --- a/cmd/adapter/main.go +++ b/cmd/adapter/main.go @@ -63,15 +63,8 @@ var configPath string var runFunc = run func main() { - // Define and parse command-line flags. CONFIG_FILE seeds the default so - // a shell-less container entrypoint (no `sh -c` to expand it into - // --config) can still select a config file via env var; --config still - // overrides it explicitly when passed. - defaultConfig := "../../config/onix/adapter.yaml" - if v := os.Getenv("CONFIG_FILE"); v != "" { - defaultConfig = v - } - flag.StringVar(&configPath, "config", defaultConfig, "Path to the configuration file") + // Define and parse command-line flags. + flag.StringVar(&configPath, "config", "../../config/onix/adapter.yaml", "Path to the configuration file") flag.Parse() // Use custom log for initial setup messages. From 392e4bb46a9ead95ff022372e962f642e3376c85 Mon Sep 17 00:00:00 2001 From: nisargabd Date: Thu, 3 Sep 2026 17:25:16 +0530 Subject: [PATCH 007/130] fix: build and scan Dockerfile.adapter-with-plugins, not Dockerfile.adapter [#5] security.yml and build-and-push.yml still pointed at Dockerfile.adapter, which we just reverted back to the original, unpatched base images -- so the security gate was scanning the wrong Dockerfile and the CVE fix never took effect. Dockerfile.adapter-with-plugins is the one actually deployed and the one carrying the DHI glibc-only runtime fix. --- .github/workflows/build-and-push.yml | 2 +- .github/workflows/security.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 4ff2479f..2d49bbcf 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -153,7 +153,7 @@ jobs: uses: docker/build-push-action@v6 with: context: . - file: Dockerfile.adapter + file: Dockerfile.adapter-with-plugins platforms: ${{ matrix.platform }} build-args: | ONIX_VERSION=${{ env.ONIX_VERSION }} diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index f4a66049..cfc42c5c 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -139,7 +139,7 @@ jobs: - name: Build image run: | source install/scripts/version-vars.sh - docker build -f Dockerfile.adapter \ + docker build -f Dockerfile.adapter-with-plugins \ --build-arg ONIX_VERSION="$ONIX_VERSION" \ --build-arg GIT_COMMIT="$GIT_COMMIT" \ --build-arg GIT_TREE_STATE="$GIT_TREE_STATE" \ From 3e88a458b624c77023916cd3793e33272c2a929c Mon Sep 17 00:00:00 2001 From: nisargabd Date: Fri, 4 Sep 2026 14:48:24 +0530 Subject: [PATCH 008/130] fix: keep the plugin-building test packages out of the full-module race+coverage run [#5] pkg/plugin and benchmarks/e2e each build a real .so via a subprocess `go build -buildmode=plugin`, then load it with plugin.Open in the same test run. Instrumenting the whole module for coverage/race in one ./... build gives a shared package a different build identity than the plain subprocess build produces, so plugin.Open rejects the .so. Split them into their own go test invocation and merge the coverage output. --- Makefile | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0bd4ab21..4a9f1098 100644 --- a/Makefile +++ b/Makefile @@ -47,9 +47,25 @@ cover: ## test-ci: run the suites through gotestsum — one line per package, coverage ## profile written alongside. What run-tests.yml calls; `make test` ## stays the plain everyday entrypoint. +# +# pkg/plugin and benchmarks/e2e each build a real .so with a plain `go build +# -buildmode=plugin` subprocess, then load it with plugin.Open in the same +# test run. Instrumenting the *whole* module's coverage in one `./...` build +# gives shared packages (e.g. pkg/plugin/definition) a build identity that +# subprocess build doesn't share, so plugin.Open rejects the .so as built +# with a different version of that package. Splitting them into their own +# `go test` invocation keeps their build graph small enough to match. test-ci: $(GOTESTSUM) $(GOTESTSUM) --format pkgname --format-hide-empty-pkg -- \ - -race -coverprofile=coverage.out -covermode=atomic ./... + -race -coverprofile=coverage.out -covermode=atomic \ + $$(go list ./... | grep -vE '/pkg/plugin$$|/benchmarks/e2e$$') + # No -race here: these two packages build a plugin .so in a subprocess + # `go build` with no -race flag of its own, so a race-instrumented test + # binary and a non-race .so mismatch and plugin.Open refuses to load it. + $(GOTESTSUM) --format pkgname --format-hide-empty-pkg -- \ + -coverprofile=coverage-plugin.out -covermode=atomic \ + ./pkg/plugin ./benchmarks/e2e/... + @tail -n +2 coverage-plugin.out >> coverage.out && rm -f coverage-plugin.out ## cover-diff: coverage restricted to files changed vs BASE_REF — a PR review ## needs the diff's number, not the whole repo's. On failure, From a7bace36e9faeb6aa7c036d006def689be15bd70 Mon Sep 17 00:00:00 2001 From: nisargabd Date: Fri, 4 Sep 2026 14:50:29 +0530 Subject: [PATCH 009/130] docs: drop the discovery-service reference from the Makefile header [#5] --- Makefile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 4a9f1098..20e437f3 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,9 @@ # OAN Network Adapter — build, test and toolchain targets. # -# Mirrors discovery-service's Makefile shape (same target names, same -# run-tests/security/build-and-push workflows call these, not raw commands), -# adapted to this module: no DB, no sqlc/migrate, no separate tools/ module — -# golangci-lint, gotestsum and trivy install straight into bin/ via `go -# install`/curl, same as discovery-service does for gotestsum and trivy. +# Single source of truth for the run-tests/security/build-and-push workflows, +# which call these targets rather than raw commands. No DB, no sqlc/migrate, +# no separate tools/ module — golangci-lint, gotestsum and trivy install +# straight into bin/ via `go install`/curl. GO ?= go BIN_DIR := bin From 6220fc4ec80a2688c908ffffdc9e8e7fdc925e0f Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Mon, 7 Sep 2026 22:36:16 +0530 Subject: [PATCH 010/130] chore: drop the dead and deploy-only GitHub Actions workflows [#5] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit None of these six can run or need to run on this repo as it stands: beckn_ci.yml, beckn_ci_test.yml trigger on PRs to beckn-onix-v1.0-develop, a branch that does not exist here — they have never fired and cannot. deploy-to-gke.yml, deploy-to-gke-BS.yml, build-and-deploy-plugins.yml, onix-gcp-terraform-deploy.yml deployment, which is out of scope for now. Keeping them means keeping the GCP/registry secret surface and the maintenance cost of workflows nobody runs. What is left is CI, Security and sign-beckn-constants.yml. --- .github/workflows/beckn_ci.yml | 88 ------------- .github/workflows/beckn_ci_test.yml | 85 ------------- .../workflows/build-and-deploy-plugins.yml | 116 ------------------ .github/workflows/deploy-to-gke-BS.yml | 69 ----------- .github/workflows/deploy-to-gke.yml | 44 ------- .../workflows/onix-gcp-terraform-deploy.yml | 68 ---------- 6 files changed, 470 deletions(-) delete mode 100644 .github/workflows/beckn_ci.yml delete mode 100644 .github/workflows/beckn_ci_test.yml delete mode 100644 .github/workflows/build-and-deploy-plugins.yml delete mode 100644 .github/workflows/deploy-to-gke-BS.yml delete mode 100644 .github/workflows/deploy-to-gke.yml delete mode 100644 .github/workflows/onix-gcp-terraform-deploy.yml diff --git a/.github/workflows/beckn_ci.yml b/.github/workflows/beckn_ci.yml deleted file mode 100644 index df6c1051..00000000 --- a/.github/workflows/beckn_ci.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: CI/CD Pipeline - -on: - pull_request: - branches: - - beckn-onix-v1.0-develop - -env: - APP_DIRECTORY: "shared/plugin" # Root directory to start searching from - -jobs: - lint_and_test: - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' - timeout-minutes: 10 # Increased timeout due to additional steps - steps: - # 1. Checkout the code from the test branch (triggered by PR) - - name: Checkout code - uses: actions/checkout@v4 - - # 2. Set up Go environment - - name: Set up Go 1.24.0 - uses: actions/setup-go@v4 - with: - go-version: '1.24.0' - - # 3. Install golangci-lint - - name: Install golangci-lint - run: | - go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest - - # 4. Run golangci-lint on the entire repo, starting from the root directory - - name: Run golangci-lint - run: | - golangci-lint run ./... # This will lint all Go files in the repo and subdirectories - - # 5. Run unit tests with coverage in the entire repository - - name: Run unit tests with coverage - run: | - # Create a directory to store coverage files - mkdir -p $GITHUB_WORKSPACE/coverage_files - - # Find all *_test.go files and run `go test` for each - find ./ -type f -name '*_test.go' | while read test_file; do - # Get the directory of the test file - test_dir=$(dirname "$test_file") - # Get the name of the Go file associated with the test - go_file="${test_file/_test.go/.go}" - - # Run tests and store coverage for each Go file in a separate file - echo "Running tests in $test_dir for $go_file" - go test -v -coverprofile=$GITHUB_WORKSPACE/coverage_files/coverage_$(basename "$go_file" .go).out $test_dir - done - - # 6. List the generated coverage files for debugging purposes - #- name: List coverage files - #run: | - #echo "Listing all generated coverage files:" - #ls -l $GITHUB_WORKSPACE/coverage_files/ - - # 7. Check coverage for each generated coverage file - - name: Check coverage for each test file - run: | - # Loop through each coverage file in the coverage_files directory - for coverage_file in $GITHUB_WORKSPACE/coverage_files/coverage_*.out; do - echo "Checking coverage for $coverage_file" - - # Get the coverage percentage for each file - coverage=$(go tool cover -func=$coverage_file | grep total | awk '{print $3}' | sed 's/%//') - echo "Coverage for $coverage_file: $coverage%" - - # If coverage is below threshold (90%), fail the job - if (( $(echo "$coverage < 80" | bc -l) )); then - echo "Coverage for $coverage_file is below 90%. Failing the job." - exit 1 - fi - done - - # 7. Build the Go code - #- name: Build Go code - # run: | - # go build -o myapp ${{ env.APP_DIRECTORY }}/... - # if [ ! -f myapp ]; then - # echo "Build failed: myapp executable was not created." - # exit 1 - # else - # echo "Build succeeded: myapp executable created." - # fi diff --git a/.github/workflows/beckn_ci_test.yml b/.github/workflows/beckn_ci_test.yml deleted file mode 100644 index e8d9ae6b..00000000 --- a/.github/workflows/beckn_ci_test.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: CI/CD Test Pipeline - -on: - pull_request: - branches: - - beckn-onix-v1.0-develop - -env: - APP_DIRECTORY: "shared/plugin" - -jobs: - lint_and_test: - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' - timeout-minutes: 10 - outputs: - coverage_ok: ${{ steps.coverage_check.outputs.coverage_ok }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Go 1.24.0 - uses: actions/setup-go@v4 - with: - go-version: '1.24.0' - - - name: Install golangci-lint - run: go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest - - - name: Run golangci-lint - run: golangci-lint run ./... - - - name: Run unit tests with coverage - run: | - mkdir -p $GITHUB_WORKSPACE/coverage_files - test_files=$(find ./ -type f -name '*_test.go') - if [ -z "$test_files" ]; then - echo "No test cases found. Skipping." - exit 0 - fi - for test_file in $test_files; do - test_dir=$(dirname "$test_file") - go_file="${test_file/_test.go/.go}" - echo "Running tests in $test_dir for $go_file" - go test -v -coverprofile=$GITHUB_WORKSPACE/coverage_files/coverage_$(basename "$go_file" .go).out $test_dir || echo "Tests failed, but continuing." - done - - - name: Check coverage for each test file - id: coverage_check - run: | - echo "coverage_ok=true" >> $GITHUB_OUTPUT - coverage_files=$(find $GITHUB_WORKSPACE/coverage_files -name "coverage_*.out") - if [ -z "$coverage_files" ]; then - echo "No coverage files found. Skipping coverage check." - exit 0 - fi - for coverage_file in $coverage_files; do - echo "Checking coverage for $coverage_file" - coverage=$(go tool cover -func=$coverage_file | grep total | awk '{print $3}' | sed 's/%//') - echo "Coverage: $coverage%" - if (( $(echo "$coverage < 90" | bc -l) )); then - echo "coverage_ok=false" >> $GITHUB_OUTPUT - break - fi - done - - require_exception_approval: - needs: lint_and_test - if: needs.lint_and_test.outputs.coverage_ok == 'false' - runs-on: ubuntu-latest - environment: - name: coverage-exception - url: https://your-coverage-dashboard.com # Optional - steps: - - name: Manual approval required - run: echo "Coverage < 90%. Approval required to continue." - - proceed_with_merge: - needs: [lint_and_test, require_exception_approval] - if: | - needs.lint_and_test.outputs.coverage_ok == 'true' || success() - runs-on: ubuntu-latest - steps: - - name: Proceed with merge - run: echo "Coverage requirement met or exception approved. Merge allowed." diff --git a/.github/workflows/build-and-deploy-plugins.yml b/.github/workflows/build-and-deploy-plugins.yml deleted file mode 100644 index 79c33e07..00000000 --- a/.github/workflows/build-and-deploy-plugins.yml +++ /dev/null @@ -1,116 +0,0 @@ -name: Build and Upload Plugins - -on: - workflow_dispatch: - inputs: - target_branch: - description: 'Branch to deploy' - required: true - default: 'beckn-onix-v1.0-develop' - - -jobs: - build-and-upload: - runs-on: ubuntu-latest - env: - GCS_BUCKET: ${{ secrets.GCS_BUCKET }} - PLUGIN_OUTPUT_DIR: ./generated - ZIP_FILE: plugins_bundle.zip - - steps: - - name: Checkout this repo - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.target_branch }} - - - name: Show selected branch - run: echo "Deploying branch:${{ github.event.inputs.target_branch }}" - - - - name: Clone GitHub and Gerrit plugin repos - run: | - # Example GitHub clone - git clone -b beckn-onix-v1.0-develop https://${{ secrets.PAT_GITHUB }}:@github.com/beckn/beckn-onix.git github-repo - - # Example Gerrit clone - git clone https://${{ secrets.GERRIT_USERNAME }}:${{ secrets.GERRIT_PAT }}@open-networks.googlesource.com/onix-dev gerrit-repo - - - name: List directory structure - run: | - echo "📂 Contents of root:" - ls -alh - - echo "📂 Contents of GitHub repo:" - ls -alh github-repo - - echo "📂 Deep list of GitHub repo:" - find github-repo - - echo "📂 Contents of Gerrit repo:" - ls -alh gerrit-repo - - echo "📂 Deep list of Gerrit repo:" - find gerrit-repo - - - - name: Build Go plugins in Docker - run: | - set -e - mkdir -p $PLUGIN_OUTPUT_DIR - - BUILD_CMDS="" - - # GitHub plugins - for dir in github-repo/pkg/plugin/implementation/*; do - if [ -d "$dir/cmd" ]; then - plugin=$(basename "$dir") - BUILD_CMDS+="cd github-repo && go build -buildmode=plugin -buildvcs=false -o ../${PLUGIN_OUTPUT_DIR}/${plugin}.so ./pkg/plugin/implementation/${plugin}/cmd && cd - && " - fi - done - - # Gerrit plugins — build in their own repo/module context - for dir in gerrit-repo/plugins/*; do - if [ -d "$dir/cmd" ]; then - plugin=$(basename "$dir") - BUILD_CMDS+="cd gerrit-repo && go build -buildmode=plugin -buildvcs=false -o ../${PLUGIN_OUTPUT_DIR}/${plugin}.so ./plugins/${plugin}/cmd && cd - && " - fi - done - - BUILD_CMDS=${BUILD_CMDS%" && "} - echo "🛠️ Running build commands inside Docker:" - echo "$BUILD_CMDS" - - docker run --rm -v "$(pwd)":/app -w /app golang:1.24-bullseye sh -c "$BUILD_CMDS" - - - name: List built plugin files - run: | - echo "Looking in $PLUGIN_OUTPUT_DIR" - ls -lh $PLUGIN_OUTPUT_DIR || echo "⚠️ Directory does not exist" - find $PLUGIN_OUTPUT_DIR -name '*.so' || echo "⚠️ No .so files found" - - echo "Creating zip archive..." - cd "$PLUGIN_OUTPUT_DIR" - zip -r "../$ZIP_FILE" *.so - echo "Created $ZIP_FILE" - cd .. - - - name: List zip output - run: | - ls -lh plugins_bundle.zip - - - - name: Authenticate to GCP - run: | - echo '${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }}' > gcloud-key.json - gcloud auth activate-service-account --key-file=gcloud-key.json - gcloud config set project trusty-relic-370809 - env: - GOOGLE_APPLICATION_CREDENTIALS: gcloud-key.json - - - name: Upload to GCS - run: | - gsutil -m cp -r $ZIP_FILE gs://${GCS_BUCKET}/plugins/ - - - name: Cleanup - run: | - rm -rf $PLUGIN_OUTPUT_DIR $ZIP_FILE gcloud-key.json diff --git a/.github/workflows/deploy-to-gke-BS.yml b/.github/workflows/deploy-to-gke-BS.yml deleted file mode 100644 index 39a6156e..00000000 --- a/.github/workflows/deploy-to-gke-BS.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: CI/CD to GKE updated - -on: - #push: - workflow_dispatch: - -jobs: - deploy: - name: Build and Deploy to GKE - runs-on: ubuntu-latest - - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - # Full history + tags, so `git describe --tags` in version-vars.sh - # can actually resolve a release tag instead of always falling back - # to the bare commit SHA (the default shallow checkout fetches no - # tag refs at all). - fetch-depth: 0 - - - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v2 - with: - credentials_json: '${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }}' - - - name: Set up gcloud CLI - uses: google-github-actions/setup-gcloud@v1 - with: - project_id: ${{ secrets.GCP_PROJECT }} - export_default_credentials: true - - - name: Install GKE Auth Plugin - run: gcloud components install gke-gcloud-auth-plugin --quiet - - - name: Configure Docker to use Artifact Registry - run: gcloud auth configure-docker ${{ secrets.GCP_REGION }}-docker.pkg.dev - - - name: Build Docker Image - run: | - IMAGE_NAME=${{ secrets.GCP_REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT }}/${{ secrets.GCP_REPO }}/beckn-onix:${{ github.sha }} - source install/scripts/version-vars.sh - docker build -f Dockerfile.adapter \ - --build-arg ONIX_VERSION="$ONIX_VERSION" \ - --build-arg GIT_COMMIT="$GIT_COMMIT" \ - --build-arg GIT_TREE_STATE="$GIT_TREE_STATE" \ - --build-arg BUILD_DATE="$BUILD_DATE" \ - -t $IMAGE_NAME . - docker push $IMAGE_NAME - - - name: Get GKE Credentials - run: | - gcloud container clusters get-credentials ${{ secrets.GKE_CLUSTER }} \ - --zone ${{ secrets.GCP_REGION }} \ - --project ${{ secrets.GCP_PROJECT }} - - - name: Deploy to GKE using Kubernetes Manifests - run: | - IMAGE_NAME=${{ secrets.GCP_REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT }}/${{ secrets.GCP_REPO }}/beckn-onix:${{ github.sha }} - - # Replace image in deployment YAML - sed -i "s|image: .*|image: $IMAGE_NAME|g" Deployment/deployment.yaml - - # Apply Kubernetes manifests - kubectl apply -f Deployment/deployment.yaml --namespace=onix-adapter - kubectl apply -f Deployment/service.yaml --namespace=onix-adapter - - # Wait for rollout to complete - kubectl rollout status Deployment/onix-demo-adapter --namespace=onix-adapter diff --git a/.github/workflows/deploy-to-gke.yml b/.github/workflows/deploy-to-gke.yml deleted file mode 100644 index bcb43efd..00000000 --- a/.github/workflows/deploy-to-gke.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Deploy to GKE - -on: - workflow_dispatch: - inputs: - service_name: - description: 'Name of the Kubernetes service to deploy' - required: true - type: string - cluster_name: - description: 'Name of the GKE cluster' - required: true - type: string - -jobs: - deploy: - runs-on: ubuntu-latest - - env: - PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - REGION: ${{ secrets.GCP_REGION }} - GKE_CLUSTER: ${{ github.event.inputs.cluster_name }} - SERVICE_NAME: ${{ github.event.inputs.service_name }} - - steps: - - name: Checkout source - uses: actions/checkout@v3 - - - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - - name: Set up GKE credentials - uses: google-github-actions/get-gke-credentials@v1 - with: - cluster_name: ${{ env.GKE_CLUSTER }} - location: ${{ env.REGION }} - project_id: ${{ env.PROJECT_ID }} - - - name: Deploy to GKE - run: | - echo "Deploying service $SERVICE_NAME to cluster $GKE_CLUSTER" - kubectl set image deployment/$SERVICE_NAME $SERVICE_NAME=gcr.io/$PROJECT_ID/$SERVICE_NAME:latest --record diff --git a/.github/workflows/onix-gcp-terraform-deploy.yml b/.github/workflows/onix-gcp-terraform-deploy.yml deleted file mode 100644 index 4d35fe94..00000000 --- a/.github/workflows/onix-gcp-terraform-deploy.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Terraform Deploy to GCP - -on: - # push trigger disabled — this always fails (missing Gerrit credentials in - # this repo) and ran on every push, including unrelated branches. Kept as - # manual-only until it's confirmed dead or folded into build-and-push.yml. - workflow_dispatch: # Manual triggerr - -jobs: - plan: - name: Terraform Plan Only - runs-on: ubuntu-latest - - steps: - - name: Checkout this repository - uses: actions/checkout@v3 - - - name: Clone Terraform repo from Gerrit - run: | - git clone https://${{ secrets.GERRIT_USERNAME }}:${{ secrets.GERRIT_PAT }}@open-networks.googlesource.com/onix-dev gerrit-repo - echo "==== Contents of Terraform-dir ====" - pwd - cd gerrit-repo/Terraform-CICD - pwd - ls -la - - - name: Authenticate to Google Cloud - run: echo '${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }}' > gcp-key.json - - - name: Set up Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.5.0 - - - name: Write GCP credentials to file - run: echo '${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }}' > gcp-key.json - - - name: Export GCP credentials environment variable - run: echo "GOOGLE_APPLICATION_CREDENTIALS=$GITHUB_WORKSPACE/gcp-key.json" >> $GITHUB_ENV - - - name: Create backend.tf and Terraform Init - working-directory: ./gerrit-repo/Terraform-CICD - env: - GCS_BUCKET: beckn-cicd-tf-state-bucket - run: | - cat < backend.tf - terraform { - backend "gcs" { - bucket = "${GCS_BUCKET}" - prefix = "terraform/state" - credentials = "${{ github.workspace }}/gcp-key.json" - } - } - EOF - - terraform init - - - name: Terraform Plan - working-directory: ./gerrit-repo/Terraform-CICD - run: terraform plan - - - name: Terraform Apply - working-directory: ./gerrit-repo/Terraform-CICD - run: terraform apply -var="subnet_name=onix-gke-subnet" -auto-approve - - - name: Clean up credentials - run: rm -f gcp-key.json - From 56f7d8bc3a19223248087c201e3fd2d42043f377 Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Mon, 7 Sep 2026 22:36:37 +0530 Subject: [PATCH 011/130] chore: drop the release workflow and revert the DHI image migration [#5] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build-and-push.yml is release/deploy, which is out of scope for now, so it goes with the other deploy workflows. That also retires the review findings that only existed inside it (the tag globs, id-token: write with no consumer, :latest on a -rc tag, the duplicated registry-login block, and the multi-arch manifest step). The Docker Hardened Images switch went with it, and it had to: Dockerfile.adapter was migrated to dhi.io in a PR whose stated scope is CI workflows. dhi.io is subscription-gated, so the image is not pullable without the DHI entitlement. Dockerfile.adapter-with-plugins same switch, plus trailing-slash churn. Back to golang:1.26.1-bookworm building and cgr.dev/chainguard/wolfi-base as the runtime — which has a shell and glibc, so plugin.Open works, and is publicly pullable, so a fork PR's image scan still builds. Deployment/deployment.yaml the added args: ["--config=$(CONFIG_FILE)"] block does not match the adapter's flags and would CrashLoopBackOff on rollout. .github/actions/dhi-login only existed to authenticate to dhi.io. All three files are now byte-identical to development. --- .github/actions/dhi-login/action.yml | 28 --- .github/workflows/build-and-push.yml | 276 --------------------------- Deployment/deployment.yaml | 5 - Dockerfile.adapter | 36 +--- Dockerfile.adapter-with-plugins | 43 +---- 5 files changed, 16 insertions(+), 372 deletions(-) delete mode 100644 .github/actions/dhi-login/action.yml delete mode 100644 .github/workflows/build-and-push.yml diff --git a/.github/actions/dhi-login/action.yml b/.github/actions/dhi-login/action.yml deleted file mode 100644 index 4dac2fdb..00000000 --- a/.github/actions/dhi-login/action.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Log in to dhi.io -description: > - Authenticates to dhi.io (Docker Hardened Images), gated on a Docker - subscription — anonymous pulls 401. Both Dockerfiles default to dhi.io/* - base images (see each Dockerfile's FROM comment), so every job that builds - or scans an image needs this first. Written directly to - ~/.docker/config.json rather than through docker/login-action, whose - username/password inputs don't accept a pre-combined blob — this matches - the format `docker login` itself writes. - -inputs: - username: - description: DHI credential username (org secret DOCKERHUB_USERNAME — DHI's subscription is tied to a Docker Hub account) - required: true - token: - description: DHI credential token (org secret DOCKERHUB_TOKEN) - required: true - -runs: - using: composite - steps: - - shell: bash - run: | - AUTH=$(printf '%s:%s' "${{ inputs.username }}" "${{ inputs.token }}" | base64 -w0) - mkdir -p ~/.docker - cat > ~/.docker/config.json <> "$GITHUB_OUTPUT" - - - name: Cache Trivy binary and vulnerability DB - uses: actions/cache@v4 - with: - path: | - bin/trivy - ~/.cache/trivy - key: trivy-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Makefile') }}-${{ steps.trivy-date.outputs.date }} - restore-keys: | - trivy-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Makefile') }}- - - - uses: docker/setup-buildx-action@v3 - - # One flag per registry, read from repo variables (Settings > Secrets - # and variables > Actions > Variables), so a registry is switched on - # without a workflow edit. GHCR is the only one we use today, so it - # defaults on; everything else defaults off until its secrets exist. - # Adding a registry later is: a flag here, a login step below, and one - # more branch in "Tag and push" and the manifest job's create_manifest - # calls — the build/scan steps never change. - - name: Set registry flags - run: | - echo "GHCR_ENABLED=${{ vars.GHCR_ENABLED != 'false' }}" >> "$GITHUB_ENV" - echo "DOCKERHUB_ENABLED=${{ vars.DOCKERHUB_ENABLED == 'true' }}" >> "$GITHUB_ENV" - echo "AZURE_CONTAINER_REGISTRY_ENABLED=${{ vars.AZURE_CONTAINER_REGISTRY_ENABLED == 'true' }}" >> "$GITHUB_ENV" - echo "GOOGLE_ARTIFACT_REGISTRY_ENABLED=${{ vars.GOOGLE_ARTIFACT_REGISTRY_ENABLED == 'true' }}" >> "$GITHUB_ENV" - - - name: Log in to dhi.io - uses: ./.github/actions/dhi-login - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - token: ${{ secrets.DOCKERHUB_TOKEN }} - - # GITHUB_TOKEN can be denied write_package when creating a brand-new - # package under a personal (non-org) namespace, even with this - # workflow's `packages: write` and the repo's default token permissions - # both set to write — a documented GHCR quirk. GHCR_PAT (classic PAT, - # write:packages) is the standard workaround; falls back to - # GITHUB_TOKEN when no such secret is set, which is enough once the - # package exists or for an org-owned repo. - - name: Log in to GHCR - if: env.GHCR_ENABLED == 'true' - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GHCR_PAT || secrets.GITHUB_TOKEN }} - - - name: Log in to Docker Hub - if: env.DOCKERHUB_ENABLED == 'true' - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Log in to Azure Container Registry - if: env.AZURE_CONTAINER_REGISTRY_ENABLED == 'true' - uses: docker/login-action@v3 - with: - registry: ${{ secrets.AZURE_REGISTRY_NAME }} - username: ${{ secrets.AZURE_REGISTRY_USERNAME }} - password: ${{ secrets.AZURE_REGISTRY_PASSWORD }} - - - name: Authenticate to Google Cloud - if: env.GOOGLE_ARTIFACT_REGISTRY_ENABLED == 'true' - uses: google-github-actions/auth@v3 - with: - token_format: "access_token" - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_EMAIL }} - - - name: Configure Docker for Google Artifact Registry - if: env.GOOGLE_ARTIFACT_REGISTRY_ENABLED == 'true' - run: gcloud auth configure-docker ${{ secrets.GCP_REGION }}-docker.pkg.dev --quiet - - # GHCR image refs must be lowercase; github.repository_owner preserves - # the owner's real case, so it can't be used as-is. This resolves to - # whichever account or org owns the repo the tag was pushed to — a - # fork pushes under the fork owner, no per-person edit needed. - - name: Set image owner - run: echo "OWNER=$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_ENV" - - # Single source of truth for the build-time identity vars (see - # pkg/version) — written to $GITHUB_ENV so the build-push-action step - # below can reference them as build-args the same way a local - # `source install/scripts/version-vars.sh` would. - - name: Resolve version vars - run: | - source install/scripts/version-vars.sh - { - echo "ONIX_VERSION=$ONIX_VERSION" - echo "GIT_COMMIT=$GIT_COMMIT" - echo "GIT_TREE_STATE=$GIT_TREE_STATE" - echo "BUILD_DATE=$BUILD_DATE" - } >> "$GITHUB_ENV" - - # One local, unpushed tag per arch — the manifest job below stitches - # amd64 and arm64 into the single tag users actually pull. - # - # load: true instead of push: true — the image lands in this runner's - # local Docker daemon first so Trivy can scan the exact bytes about to - # ship, before it is tagged for or pushed to any registry. - - name: Build (local, unpushed) - uses: docker/build-push-action@v6 - with: - context: . - file: Dockerfile.adapter-with-plugins - platforms: ${{ matrix.platform }} - build-args: | - ONIX_VERSION=${{ env.ONIX_VERSION }} - GIT_COMMIT=${{ env.GIT_COMMIT }} - GIT_TREE_STATE=${{ env.GIT_TREE_STATE }} - BUILD_DATE=${{ env.BUILD_DATE }} - load: true - tags: ${{ env.IMAGE_NAME }}:${{ github.ref_name }}-${{ matrix.arch }} - - # trivy-release-gate exits 1 on a finding — the release gate, not just - # a report — before any push below runs. One scan covers every registry. - - name: Trivy image scan (gate) - run: make trivy-release-gate IMAGE=${{ env.IMAGE_NAME }}:${{ github.ref_name }}-${{ matrix.arch }} - - # Re-tag the already-scanned local image for each enabled registry and - # push it there. No rebuild, no re-scan. - - name: Tag and push - run: | - LOCAL="${{ env.IMAGE_NAME }}:${{ github.ref_name }}-${{ matrix.arch }}" - TAG="${{ github.ref_name }}-${{ matrix.arch }}" - - if [ "$GHCR_ENABLED" = "true" ]; then - DEST="ghcr.io/${OWNER}/${{ env.IMAGE_NAME }}:${TAG}" - docker tag "$LOCAL" "$DEST" - docker push "$DEST" - fi - if [ "$DOCKERHUB_ENABLED" = "true" ]; then - DEST="${{ secrets.DOCKERHUB_NAMESPACE }}/${{ env.IMAGE_NAME }}:${TAG}" - docker tag "$LOCAL" "$DEST" - docker push "$DEST" - fi - if [ "$AZURE_CONTAINER_REGISTRY_ENABLED" = "true" ]; then - DEST="${{ secrets.AZURE_REGISTRY_NAME }}/${{ env.IMAGE_NAME }}:${TAG}" - docker tag "$LOCAL" "$DEST" - docker push "$DEST" - fi - if [ "$GOOGLE_ARTIFACT_REGISTRY_ENABLED" = "true" ]; then - DEST="${{ secrets.GCP_REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ secrets.GOOGLE_ARTIFACT_REPO }}/${{ env.IMAGE_NAME }}:${TAG}" - docker tag "$LOCAL" "$DEST" - docker push "$DEST" - fi - - manifest: - runs-on: ubuntu-latest - needs: build - steps: - - uses: docker/setup-buildx-action@v3 - - - name: Set registry flags - run: | - echo "GHCR_ENABLED=${{ vars.GHCR_ENABLED != 'false' }}" >> "$GITHUB_ENV" - echo "DOCKERHUB_ENABLED=${{ vars.DOCKERHUB_ENABLED == 'true' }}" >> "$GITHUB_ENV" - echo "AZURE_CONTAINER_REGISTRY_ENABLED=${{ vars.AZURE_CONTAINER_REGISTRY_ENABLED == 'true' }}" >> "$GITHUB_ENV" - echo "GOOGLE_ARTIFACT_REGISTRY_ENABLED=${{ vars.GOOGLE_ARTIFACT_REGISTRY_ENABLED == 'true' }}" >> "$GITHUB_ENV" - - - name: Log in to GHCR - if: env.GHCR_ENABLED == 'true' - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GHCR_PAT || secrets.GITHUB_TOKEN }} - - - name: Log in to Docker Hub - if: env.DOCKERHUB_ENABLED == 'true' - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Log in to Azure Container Registry - if: env.AZURE_CONTAINER_REGISTRY_ENABLED == 'true' - uses: docker/login-action@v3 - with: - registry: ${{ secrets.AZURE_REGISTRY_NAME }} - username: ${{ secrets.AZURE_REGISTRY_USERNAME }} - password: ${{ secrets.AZURE_REGISTRY_PASSWORD }} - - - name: Authenticate to Google Cloud - if: env.GOOGLE_ARTIFACT_REGISTRY_ENABLED == 'true' - uses: google-github-actions/auth@v3 - with: - token_format: "access_token" - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_EMAIL }} - - - name: Configure Docker for Google Artifact Registry - if: env.GOOGLE_ARTIFACT_REGISTRY_ENABLED == 'true' - run: gcloud auth configure-docker ${{ secrets.GCP_REGION }}-docker.pkg.dev --quiet - - - name: Set image owner - run: echo "OWNER=$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_ENV" - - # imagetools create makes one manifest list from the two arch-specific - # images the build job just pushed to that registry — this is what - # lets `docker pull` resolve the right arch automatically on either - # amd64 or arm64. One create_manifest call per enabled registry. - - name: Create multi-arch manifests - run: | - create_manifest() { - local BASE="$1" - docker buildx imagetools create \ - -t "${BASE}:${{ github.ref_name }}" \ - -t "${BASE}:latest" \ - "${BASE}:${{ github.ref_name }}-amd64" \ - "${BASE}:${{ github.ref_name }}-arm64" - } - - if [ "$GHCR_ENABLED" = "true" ]; then - create_manifest "ghcr.io/${OWNER}/${{ env.IMAGE_NAME }}" - fi - if [ "$DOCKERHUB_ENABLED" = "true" ]; then - create_manifest "${{ secrets.DOCKERHUB_NAMESPACE }}/${{ env.IMAGE_NAME }}" - fi - if [ "$AZURE_CONTAINER_REGISTRY_ENABLED" = "true" ]; then - create_manifest "${{ secrets.AZURE_REGISTRY_NAME }}/${{ env.IMAGE_NAME }}" - fi - if [ "$GOOGLE_ARTIFACT_REGISTRY_ENABLED" = "true" ]; then - create_manifest "${{ secrets.GCP_REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ secrets.GOOGLE_ARTIFACT_REPO }}/${{ env.IMAGE_NAME }}" - fi diff --git a/Deployment/deployment.yaml b/Deployment/deployment.yaml index 443ac03c..561fdb5a 100644 --- a/Deployment/deployment.yaml +++ b/Deployment/deployment.yaml @@ -19,11 +19,6 @@ spec: containers: - name: onix-adapter image: "asia-south1-docker.pkg.dev/trusty-relic-370809/onix-adapter-cicd/beckn-onix:latest" #------ - # $(CONFIG_FILE) is expanded by the kubelet against the env below - # before the container starts -- the image's shell-less DHI base - # has no `sh -c` to do that substitution itself. - args: - - "--config=$(CONFIG_FILE)" ports: - containerPort: 8080 env: diff --git a/Dockerfile.adapter b/Dockerfile.adapter index 0b1fd641..41dd7ca8 100644 --- a/Dockerfile.adapter +++ b/Dockerfile.adapter @@ -1,23 +1,4 @@ -# syntax=docker/dockerfile:1 - -# DHI (Docker Hardened Images) requires `docker login dhi.io` first — a Docker -# subscription credential this build cannot supply. Both stages default to it -# because it's what CI and every shipped image use, but a local build without -# that subscription can override both build-args with public equivalents: -# -# docker build -f Dockerfile.adapter \ -# --build-arg BUILD_IMAGE=golang:1.26.1-bookworm \ -# --build-arg RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base:latest \ -# -t network-adapter:dev . -# -# Tags verified against github.com/docker-hardened-images/catalog -# image/golang/debian-12/1.26-dev.yaml and image/debian-base/debian-12/12.yaml. -# No cgo here, but the -dev variant's gcc/g++/make come bundled regardless — -# nothing extra to install for either the default or the public override. -ARG BUILD_IMAGE=dhi.io/golang:1.26-debian12-dev -ARG RUNTIME_IMAGE=dhi.io/debian-base:bookworm - -FROM ${BUILD_IMAGE} AS builder +FROM golang:1.26.1-bookworm AS builder WORKDIR /workspace/app COPY cmd/adapter ./cmd/adapter @@ -34,15 +15,16 @@ ARG BUILD_DATE=unknown RUN go build -ldflags "-X github.com/beckn-one/beckn-onix/pkg/version.Version=${ONIX_VERSION} -X github.com/beckn-one/beckn-onix/pkg/version.GitCommit=${GIT_COMMIT} -X github.com/beckn-one/beckn-onix/pkg/version.GitTreeState=${GIT_TREE_STATE} -X github.com/beckn-one/beckn-onix/pkg/version.BuildDate=${BUILD_DATE}" -o server cmd/adapter/main.go -# Debian-based hardened runtime, run-as nonroot by default — kept explicit -# below since RUNTIME_IMAGE can be overridden to wolfi-base, which uses the -# same uid 65532 convention. A shell is required here: CMD below needs -# `sh -c` to expand $CONFIG_FILE. -FROM ${RUNTIME_IMAGE} +# Create a minimal runtime image +FROM cgr.dev/chainguard/wolfi-base:latest +# ✅ Alpine is removed; using minimal Debian WORKDIR /app +# Copy only the built binary and plugin COPY --from=builder /workspace/app/server . +# Expose port 8080 EXPOSE 8080 -USER 65532:65532 -CMD ["sh", "-c", "./server --config=${CONFIG_FILE}"] + +# Run the Go server with the config flag from environment variable. +CMD ["sh", "-c", "./server --config=${CONFIG_FILE}"] \ No newline at end of file diff --git a/Dockerfile.adapter-with-plugins b/Dockerfile.adapter-with-plugins index 6f72a4c0..a7ba5e98 100644 --- a/Dockerfile.adapter-with-plugins +++ b/Dockerfile.adapter-with-plugins @@ -1,31 +1,11 @@ -# syntax=docker/dockerfile:1 - -# DHI (Docker Hardened Images) requires `docker login dhi.io` first — a Docker -# subscription credential this build cannot supply. Both stages default to it -# because it's what CI and every shipped image use, but a local build without -# that subscription can override both build-args with public equivalents: -# -# docker build -f Dockerfile.adapter-with-plugins \ -# --build-arg BUILD_IMAGE=golang:1.26.1-bookworm \ -# --build-arg RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base:latest \ -# -t network-adapter-plugins:dev . -# -# Tags verified against github.com/docker-hardened-images/catalog -# image/golang/debian-12/1.26-dev.yaml and image/static/debian-12/static-glibc.yaml. -# -buildmode=plugin, and plugin.Open at runtime, both need a C toolchain and -# a real glibc dynamic linker — the -dev variant's gcc/g++/make come -# bundled, same as the public bookworm override. -ARG BUILD_IMAGE=dhi.io/golang:1.26-debian12-dev -ARG RUNTIME_IMAGE=dhi.io/static:20250419-glibc-debian12 - -FROM ${BUILD_IMAGE} AS builder +FROM golang:1.26.1-bookworm AS builder WORKDIR /workspace/app # Specifically copy only the necessary files and directories COPY cmd/adapter/ ./cmd/adapter/ -COPY core/ ./core -COPY pkg/ ./pkg +COPY core/ ./core/ +COPY pkg/ ./pkg/ COPY install/build-plugins.sh ./install/build-plugins.sh COPY install/scripts/version-vars.sh ./install/scripts/version-vars.sh COPY go.mod . @@ -50,12 +30,8 @@ RUN chmod +x install/build-plugins.sh && \ ONIX_VERSION="${ONIX_VERSION}" GIT_COMMIT="${GIT_COMMIT}" GIT_TREE_STATE="${GIT_TREE_STATE}" BUILD_DATE="${BUILD_DATE}" \ ./install/build-plugins.sh -# glibc-only runtime — no shell, no perl/ncurses/util-linux (the packages -# that were carrying unpatched CVEs on the fuller debian-base image), but -# keeps libc6 so plugin.Open's dynamic linking still works. Explicit USER -# below since RUNTIME_IMAGE can be overridden to wolfi-base, which uses the -# same uid 65532 convention. -FROM ${RUNTIME_IMAGE} +# Create minimal runtime image +FROM cgr.dev/chainguard/wolfi-base:latest WORKDIR /app # Copy binary and plugins built with same Go version @@ -63,10 +39,5 @@ COPY --from=builder /workspace/app/server . COPY --from=builder /workspace/app/plugins ./plugins EXPOSE 8081 -USER 65532:65532 -# Exec form, not `sh -c` — this base has no shell, and no Go code reads -# CONFIG_FILE either. The deployment layer resolves it instead: Kubernetes -# expands $(CONFIG_FILE) in args itself before exec'ing the container (see -# Deployment/deployment.yaml) -- no shell involved on either side. A plain -# `docker run` without that substitution needs an explicit --config flag. -ENTRYPOINT ["/app/server"] + +CMD ["sh", "-c", "./server --config=${CONFIG_FILE}"] From 26e013f19b8c0fe05b39819f1b0c10f672c5f58d Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Mon, 7 Sep 2026 22:36:47 +0530 Subject: [PATCH 012/130] fix: bump crypto, grpc and amqp091-go past the HIGH/CRITICAL advisories [#5] The Security workflow this PR adds is only worth adding if it can be green, and on development it is not: trivy fs flags three HIGH/CRITICAL advisories in the dependency graph. golang.org/x/crypto v0.54.0 -> v0.55.0 google.golang.org/grpc v1.82.1 -> v1.83.1 github.com/rabbitmq/amqp091-go v1.11.0 -> v1.13.0 `go mod tidy` carried the transitive minimums along with them (otel 1.43 -> 1.44, x/net 0.56 -> 0.57, x/text 0.40 -> 0.41, genproto). No source change was needed: `go build -trimpath ./cmd/adapter/...` and `go vet ./...` are both clean, and the full suite passes. --- go.mod | 24 ++++++++++++------------ go.sum | 58 ++++++++++++++++++++++++++++++---------------------------- 2 files changed, 42 insertions(+), 40 deletions(-) diff --git a/go.mod b/go.mod index fe8b8670..10145dc5 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.26.1 require ( github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 - golang.org/x/crypto v0.54.0 + golang.org/x/crypto v0.55.0 ) require github.com/stretchr/testify v1.11.1 @@ -19,7 +19,7 @@ require ( require github.com/zenazn/pkcs7pad v0.0.0-20170308005700-253a5b1f0e03 -require golang.org/x/text v0.40.0 // indirect +require golang.org/x/text v0.41.0 // indirect require ( github.com/agnivade/levenshtein v1.2.1 // indirect @@ -80,11 +80,11 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/net v0.56.0 // indirect + golang.org/x/net v0.57.0 // indirect golang.org/x/sys v0.47.0 // indirect golang.org/x/time v0.15.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260427160629-7cedc36a6bc4 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/protobuf v1.36.11 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) @@ -102,25 +102,25 @@ require ( github.com/lestrrat-go/jwx/v3 v3.1.0 github.com/open-policy-agent/opa v1.15.2 github.com/prometheus/client_golang v1.23.2 - github.com/rabbitmq/amqp091-go v1.11.0 + github.com/rabbitmq/amqp091-go v1.13.0 github.com/redis/go-redis/extra/redisotel/v9 v9.19.0 github.com/redis/go-redis/v9 v9.19.0 github.com/rs/zerolog v1.35.1 go.opentelemetry.io/contrib/instrumentation/runtime v0.68.0 - go.opentelemetry.io/otel v1.43.0 + go.opentelemetry.io/otel v1.44.0 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.19.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.43.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0 go.opentelemetry.io/otel/exporters/prometheus v0.65.0 go.opentelemetry.io/otel/log v0.19.0 - go.opentelemetry.io/otel/metric v1.43.0 - go.opentelemetry.io/otel/sdk v1.43.0 + go.opentelemetry.io/otel/metric v1.44.0 + go.opentelemetry.io/otel/sdk v1.44.0 go.opentelemetry.io/otel/sdk/log v0.19.0 - go.opentelemetry.io/otel/sdk/metric v1.43.0 - go.opentelemetry.io/otel/trace v1.43.0 + go.opentelemetry.io/otel/sdk/metric v1.44.0 + go.opentelemetry.io/otel/trace v1.44.0 go.uber.org/automaxprocs v1.6.0 golang.org/x/sync v0.22.0 - google.golang.org/grpc v1.82.1 + google.golang.org/grpc v1.83.1 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 055511f8..35da526b 100644 --- a/go.sum +++ b/go.sum @@ -165,8 +165,8 @@ github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEo github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= -github.com/rabbitmq/amqp091-go v1.11.0 h1:HxIctVm9Gid/Vtn706necmZ7Wj6pgGI2eqplRbEY8O8= -github.com/rabbitmq/amqp091-go v1.11.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/rabbitmq/amqp091-go v1.13.0 h1:L8NA1WtF76C6KA3LAoufjfLgbist/If1UQYcsOjtxXA= +github.com/rabbitmq/amqp091-go v1.13.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg= github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/redis/go-redis/extra/rediscmd/v9 v9.19.0 h1:QL3vQTj64ZQpxiDZx6bFYS7oN37EdHHqiYGz3grgTRI= @@ -219,8 +219,8 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/runtime v0.68.0 h1:jhVIQEprwUTV+KfzzliLidclhoTOoHTgdz96kAyR8mU= go.opentelemetry.io/contrib/instrumentation/runtime v0.68.0/go.mod h1:4HsdbLUbernaTnA8CNaNE+1g026SciXb3juRYe3l8EY= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.19.0 h1:Dn8rkudDzY6KV9dr/D/bTUuWgqDf9xe0rr4G2elrn0Y= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.19.0/go.mod h1:gMk9F0xDgyN9M/3Ed5Y1wKcx/9mlU91NXY2SNq7RQuU= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.43.0 h1:8UQVDcZxOJLtX6gxtDt3vY2WTgvZqMQRzjsqiIHQdkc= @@ -233,18 +233,20 @@ go.opentelemetry.io/otel/exporters/prometheus v0.65.0 h1:jOveH/b4lU9HT7y+Gfamf18 go.opentelemetry.io/otel/exporters/prometheus v0.65.0/go.mod h1:i1P8pcumauPtUI4YNopea1dhzEMuEqWP1xoUZDylLHo= go.opentelemetry.io/otel/log v0.19.0 h1:KUZs/GOsw79TBBMfDWsXS+KZ4g2Ckzksd1ymzsIEbo4= go.opentelemetry.io/otel/log v0.19.0/go.mod h1:5DQYeGmxVIr4n0/BcJvF4upsraHjg6vudJJpnkL6Ipk= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA= +go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= go.opentelemetry.io/otel/sdk/log v0.19.0 h1:scYVLqT22D2gqXItnWiocLUKGH9yvkkeql5dBDiXyko= go.opentelemetry.io/otel/sdk/log v0.19.0/go.mod h1:vFBowwXGLlW9AvpuF7bMgnNI95LiW10szrOdvzBHlAg= go.opentelemetry.io/otel/sdk/log/logtest v0.19.0 h1:BEbF7ZBB6qQloV/Ub1+3NQoOUnVtcGkU3XX4Ws3GQfk= go.opentelemetry.io/otel/sdk/log/logtest v0.19.0/go.mod h1:Lua81/3yM0wOmoHTokLj9y9ADeA02v1naRrVrkAZuKk= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= @@ -257,30 +259,30 @@ go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= -golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= -golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= -golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= -golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= -golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260427160629-7cedc36a6bc4 h1:yOzSCGPx+cp5VO7IxvZ9SBFF7j1tZVcNtlHR2iYKtVo= -google.golang.org/genproto/googleapis/api v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:Q9HWtNeE7tM9npdIsEvqXj1QJIvVoeAV3rtXtS715Cw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 h1:tEkOQcXgF6dH1G+MVKZrfpYvozGrzb91k6ha7jireSM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= -google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= +google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From e99044b944515949838d096a18fe638301befb04 Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Mon, 7 Sep 2026 22:37:15 +0530 Subject: [PATCH 013/130] refactor: move the CI implementation into the Makefile [#5] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback was that the workflows carried the logic and the Makefile carried aliases — 132 lines of inline shell across the three workflows, none of it runnable locally. That is inverted here: the Makefile is the single source of truth and every CI step becomes a one-line `make ` call, so a red check reproduces by running the command the log shows. Fixes, not just relocation: test / cover both failed on development. pkg/plugin and benchmarks/e2e build a real .so and plugin.Open it in the same run, and both -race and whole-module coverage instrumentation make that .so unloadable. They now run as their own invocation. The carve-out is named once (PLUGIN_PKGS/MAIN_PKGS) and shared by test, cover and test-ci, so the three cannot disagree about it. -coverpkg answered in a comment rather than added: benchmarks/e2e is the only test-only package in the module and it is in the carve-out precisely because whole-module instrumentation is what breaks plugin.Open. It cannot be added without reintroducing the failure. cover-diff no longer swallows a broken base. An unresolvable BASE_REF and a failed `git diff` are both hard errors with an ::error:: annotation — an unknown changed-file set is not an empty one. It writes coverage-report.md on every exit path, including those two, so the workflow can read the file unconditionally. BASE_REF defaults to origin/development, the branch PRs actually target. merge-coverage `;` not `&&`, and always removes coverage-plugin.out, so a failed merge cannot leave a half-profile behind to be misread. coverage-plugin.out is gitignored as well. coverage.out a file rule, so cover-diff has a buildable prerequisite: it reuses the profile test-ci already wrote in CI, and runs the suites once on a clean local checkout. trivy-gate a missing or unparsable SARIF report is now a failure. It used to pass vacuously when a scan wrote nothing, which is the one case a security gate must not be green for. docker builds Dockerfile.adapter-with-plugins — the image that ships and that security.yml scans — not Dockerfile.adapter. Version build args come from install/scripts/version-vars.sh so they are spelled out in one place. trivy-deps --skip-dirs bin, so the scanner stops reporting on the trivy binary the cache restored next to it. help one line per target from the `## ` docstrings, instead of a wall of text. tools/trivy-comment.jq is the 16-line jq program that was pasted into four places. As a file it is lintable (`jq -n -f`), diffable, and free of Makefile `$$`/backslash escaping, and one copy means the dependency and image reports cannot drift apart. Removed as dead: trivy-release-gate (its only caller was build-and-push.yml, now deleted) and the `tools` alias (nothing invoked it — every tool is already built on demand as a prerequisite). Verified locally: build, test, cover, cover-diff (pass, no-changed-files, below-threshold and unresolvable-BASE_REF paths), trivy-deps, trivy-image, trivy-comment, all five trivy-gate states, and help. --- .gitignore | 1 + Makefile | 238 ++++++++++++++++++++++++++++------------- tools/trivy-comment.jq | 32 ++++++ 3 files changed, 197 insertions(+), 74 deletions(-) create mode 100644 tools/trivy-comment.jq diff --git a/.gitignore b/.gitignore index 48d66b76..4a9de6ae 100644 --- a/.gitignore +++ b/.gitignore @@ -143,6 +143,7 @@ create_benchmark_issues.sh # Ignore coverage output files coverage.out coverage.html +coverage-plugin.out coverage-report.md # Makefile tool/build artifacts (golangci-lint, gotestsum, trivy binaries) diff --git a/Makefile b/Makefile index 20e437f3..b28b905d 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,13 @@ -# OAN Network Adapter — build, test and toolchain targets. +# OAN Network Adapter — build, test, lint and security-scan targets. # -# Single source of truth for the run-tests/security/build-and-push workflows, -# which call these targets rather than raw commands. No DB, no sqlc/migrate, -# no separate tools/ module — golangci-lint, gotestsum and trivy install -# straight into bin/ via `go install`/curl. +# Single source of truth for the ci.yml and security.yml workflows: every CI +# step is a one-line `make ` call, so a red check reproduces locally +# by running the command the log shows. Anything left inline in the workflows +# is GitHub context (`${{ }}` expressions, $GITHUB_ENV / $GITHUB_STEP_SUMMARY +# writes) that has no meaning outside a runner. +# +# No DB, no sqlc/migrate, no separate tools/ module — golangci-lint, gotestsum +# and trivy install straight into bin/ via `go install` / curl. GO ?= go BIN_DIR := bin @@ -12,7 +16,9 @@ IMAGE ?= network-adapter:dev # CI thresholds/pins live here, not duplicated into workflow env blocks — one # source of truth for both a local `make` run and the GitHub Actions runner. MIN_COVERAGE ?= 80 -BASE_REF ?= origin/main +# development, not main: every branch in this repo is cut from development and +# PRs target it, so that is the base a local `make cover-diff` must compare to. +BASE_REF ?= origin/development SEVERITY ?= HIGH,CRITICAL GOLANGCI_LINT_VERSION := v2.5.0 GOTESTSUM_VERSION := v1.13.0 @@ -22,11 +28,41 @@ GOLANGCI_LINT := $(BIN_DIR)/golangci-lint GOTESTSUM := $(BIN_DIR)/gotestsum TRIVY := $(BIN_DIR)/trivy +# pkg/plugin and benchmarks/e2e each build a real .so with a plain `go build +# -buildmode=plugin` subprocess, then load it with plugin.Open in the same test +# run. Two things make that .so unloadable: +# +# -race — the subprocess build carries no -race flag of its own, so a +# race-instrumented test binary and a non-race .so mismatch. +# ./... — instrumenting the whole module's coverage in one build gives +# shared packages (e.g. pkg/plugin/definition) a build identity +# the subprocess build doesn't share, so plugin.Open rejects the +# .so as "built with a different version of" that package. +# +# So they run as their own invocation, without -race and with their own +# coverage profile. Named once here and shared by test, cover and test-ci — +# the three must never disagree about which packages are carved out. +# +# Deliberately no -coverpkg anywhere, and it can't be added: benchmarks/e2e is +# the only test-only package in the module (`go list` confirms it holds no +# non-test files), so it is the only place -coverpkg would credit coverage of +# the packages it drives — and it is in this carve-out precisely because that +# whole-module instrumentation is what makes plugin.Open reject the .so. The +# code benchmarks/e2e exercises is therefore credited only by its own +# packages' tests, which is why cover-diff gates on the diff rather than on a +# module-wide total. +PLUGIN_PKGS := ./pkg/plugin ./benchmarks/e2e/... +MAIN_PKGS = $$($(GO) list ./... | grep -vE '/pkg/plugin$$|/benchmarks/e2e$$') + +# Only used inside a workflow; a local run gets a placeholder rather than a +# broken link. +RUN_URL ?= $(if $(GITHUB_RUN_ID),$(GITHUB_SERVER_URL)/$(GITHUB_REPOSITORY)/actions/runs/$(GITHUB_RUN_ID),local run) + .DEFAULT_GOAL := help ## help: list the available targets help: - @grep -hE '^## ' $(MAKEFILE_LIST) | sed 's/^## / /' | sort + @grep -hE '^## [a-z]' $(MAKEFILE_LIST) | sed 's/^## / /' | sort ## build: compile the adapter binary # Scoped to cmd/adapter, not ./... — pkg/plugin/implementation/*/cmd holds @@ -35,44 +71,60 @@ help: build: $(GO) build -trimpath -o $(BIN_DIR)/ ./cmd/adapter/... -## test: run the unit and integration suites +## test: run the unit and integration suites (plugin packages without -race) test: - $(GO) test -race ./... + $(GO) test -race $(MAIN_PKGS) + $(GO) test $(PLUGIN_PKGS) -## cover: run the suites and write a coverage profile +## cover: run the suites and write a merged coverage profile to coverage.out cover: - $(GO) test -race -covermode=atomic -coverprofile=coverage.out ./... + $(GO) test -race -covermode=atomic -coverprofile=coverage.out $(MAIN_PKGS) + $(GO) test -covermode=atomic -coverprofile=coverage-plugin.out $(PLUGIN_PKGS) + @$(MAKE) --no-print-directory merge-coverage -## test-ci: run the suites through gotestsum — one line per package, coverage -## profile written alongside. What run-tests.yml calls; `make test` -## stays the plain everyday entrypoint. -# -# pkg/plugin and benchmarks/e2e each build a real .so with a plain `go build -# -buildmode=plugin` subprocess, then load it with plugin.Open in the same -# test run. Instrumenting the *whole* module's coverage in one `./...` build -# gives shared packages (e.g. pkg/plugin/definition) a build identity that -# subprocess build doesn't share, so plugin.Open rejects the .so as built -# with a different version of that package. Splitting them into their own -# `go test` invocation keeps their build graph small enough to match. +## test-ci: cover, through gotestsum — one line per package. What ci.yml calls. test-ci: $(GOTESTSUM) $(GOTESTSUM) --format pkgname --format-hide-empty-pkg -- \ - -race -coverprofile=coverage.out -covermode=atomic \ - $$(go list ./... | grep -vE '/pkg/plugin$$|/benchmarks/e2e$$') - # No -race here: these two packages build a plugin .so in a subprocess - # `go build` with no -race flag of its own, so a race-instrumented test - # binary and a non-race .so mismatch and plugin.Open refuses to load it. + -race -coverprofile=coverage.out -covermode=atomic $(MAIN_PKGS) $(GOTESTSUM) --format pkgname --format-hide-empty-pkg -- \ - -coverprofile=coverage-plugin.out -covermode=atomic \ - ./pkg/plugin ./benchmarks/e2e/... - @tail -n +2 coverage-plugin.out >> coverage.out && rm -f coverage-plugin.out + -coverprofile=coverage-plugin.out -covermode=atomic $(PLUGIN_PKGS) + @$(MAKE) --no-print-directory merge-coverage + +# `;` not `&&`, and always removes the intermediate: a failed tail must not +# leave coverage-plugin.out behind for someone to pick up by hand and misread. +merge-coverage: + @tail -n +2 coverage-plugin.out >> coverage.out; rm -f coverage-plugin.out + +# cover-diff needs a profile but must not re-run the suites in CI, where +# test-ci already wrote one. A file rule gives it both: present (CI) and make +# skips this; absent (clean local checkout) and it runs the suites once. +coverage.out: + @$(MAKE) --no-print-directory cover -## cover-diff: coverage restricted to files changed vs BASE_REF — a PR review -## needs the diff's number, not the whole repo's. On failure, -## names the changed files dragging the number down (worst first). +# The marker is written into the report itself, not added by the workflow: +# find-comment matches on this exact string to update its comment in place +# rather than posting a new one on every run. +COVER_MARKER := + +## cover-diff: coverage of the files changed vs BASE_REF, gated on MIN_COVERAGE +# A PR review needs the diff's number, not the whole repo's. On failure, names +# the changed files dragging it down, worst first. Always writes +# coverage-report.md — the workflow reads that file unconditionally, so every +# exit path here has to produce it. cover-diff: coverage.out - @CHANGED=$$(git diff --name-only --diff-filter=ACMR "$(BASE_REF)...HEAD" -- '*.go' | grep -v '_test\.go$$' || true); \ + @if ! git rev-parse --verify --quiet "$(BASE_REF)^{commit}" >/dev/null; then \ + echo "::error::BASE_REF '$(BASE_REF)' does not resolve to a commit — cannot compute the changed-file set"; \ + echo "📊 **Test Coverage: ❌ Failed** — BASE_REF \`$(BASE_REF)\` does not resolve to a commit" > coverage-report.md; \ + exit 1; \ + fi; \ + if ! DIFF=$$(git diff --name-only --diff-filter=ACMR "$(BASE_REF)...HEAD" -- '*.go'); then \ + echo "::error::git diff against '$(BASE_REF)' failed — the changed-file set is unknown, not empty"; \ + echo "📊 **Test Coverage: ❌ Failed** — \`git diff\` against \`$(BASE_REF)\` failed" > coverage-report.md; \ + exit 1; \ + fi; \ + CHANGED=$$(printf '%s\n' "$$DIFF" | grep -v '_test\.go$$'); \ if [ -z "$$CHANGED" ]; then \ - echo "📊 **Test Coverage: ✅ Passed** — not applicable, no changed Go files vs $(BASE_REF)" | tee coverage-report.md; \ + printf '%s\n' "$(COVER_MARKER)" "📊 **Test Coverage: ✅ Passed** — not applicable, no changed Go files vs $(BASE_REF)" | tee coverage-report.md; \ exit 0; \ fi; \ MODULE=$$($(GO) list -m); \ @@ -92,59 +144,87 @@ cover-diff: coverage.out print "TOTAL\t" int(C * 100 / T) \ }' - coverage.out); \ if echo "$$RESULT" | grep -q '^EMPTY$$'; then \ - echo "📊 **Test Coverage: ✅ Passed** — not applicable, changed files carry no coverable statements" | tee coverage-report.md; \ + printf '%s\n' "$(COVER_MARKER)" "📊 **Test Coverage: ✅ Passed** — not applicable, changed files carry no coverable statements" | tee coverage-report.md; \ exit 0; \ fi; \ PCT=$$(echo "$$RESULT" | awk -F'\t' '$$1=="TOTAL"{print $$2}'); \ - if [ "$$PCT" -lt "$(MIN_COVERAGE)" ]; then \ - BELOW=$$(echo "$$RESULT" | awk -F'\t' '$$1=="FILE"{printf "%s\t%s\n",$$2,$$3}' | sort -n); \ - TOTAL_BELOW=$$(echo "$$BELOW" | wc -l); \ - { \ + { \ + echo "$(COVER_MARKER)"; \ + if [ "$$PCT" -lt "$(MIN_COVERAGE)" ]; then \ + BELOW=$$(echo "$$RESULT" | awk -F'\t' '$$1=="FILE"{printf "%s\t%s\n",$$2,$$3}' | sort -n); \ + TOTAL_BELOW=$$(echo "$$BELOW" | wc -l); \ echo "📊 **Test Coverage: ❌ Failed** — $${PCT}% of changed lines covered, min $(MIN_COVERAGE)%"; \ echo; \ echo "| File | Coverage |"; \ echo "|---|---|"; \ echo "$$BELOW" | head -15 | awk -F'\t' '{printf "| `%s` | %s%% |\n", $$2, $$1}'; \ [ "$$TOTAL_BELOW" -gt 15 ] && echo "| … | $$((TOTAL_BELOW - 15)) more file(s) below $(MIN_COVERAGE)% |"; \ - } > coverage-report.md; \ - else \ - echo "📊 **Test Coverage: ✅ Passed** — $${PCT}% of changed lines covered, min $(MIN_COVERAGE)%" > coverage-report.md; \ - fi; \ + else \ + echo "📊 **Test Coverage: ✅ Passed** — $${PCT}% of changed lines covered, min $(MIN_COVERAGE)%"; \ + fi; \ + } > coverage-report.md; \ cat coverage-report.md; \ - [ "$$PCT" -ge "$(MIN_COVERAGE)" ] + if [ "$$PCT" -lt "$(MIN_COVERAGE)" ]; then \ + echo "::error::changed-file coverage is $${PCT}%, below the $(MIN_COVERAGE)% minimum"; \ + exit 1; \ + fi -## trivy-deps: dependency graph scan (T4), SARIF report. Catches what the -## image scan structurally cannot — a vulnerable module only the -## test suite imports, so it's never linked into the binary. +## trivy-deps: scan the dependency graph, SARIF report to trivy-deps.sarif +# Catches what the image scan structurally cannot — a vulnerable module only +# the test suite imports, so it is never linked into the binary or a layer. +# --skip-dirs bin: the workflow restores the cached trivy binary into bin/ +# before this runs, and a scanner reporting on its own binary is noise. trivy-deps: $(TRIVY) - $(TRIVY) fs . --severity $(SEVERITY) --exit-code 0 \ + $(TRIVY) fs . --skip-dirs $(BIN_DIR) --severity $(SEVERITY) --exit-code 0 \ --format sarif --output trivy-deps.sarif -TRIVY_IMAGE_SCAN = $(TRIVY) image $(IMAGE) --severity $(SEVERITY) - -## trivy-image: shipped image scan (T4), SARIF report. IMAGE names the ref. +## trivy-image: scan IMAGE, SARIF report to trivy-image.sarif +# Reads the base layers plus the Go build info embedded in the binary — +# including `stdlib`, so a Go toolchain CVE shows up here and nowhere else. trivy-image: $(TRIVY) - $(TRIVY_IMAGE_SCAN) --exit-code 0 --format sarif --output trivy-image.sarif + $(TRIVY) image $(IMAGE) --severity $(SEVERITY) --exit-code 0 \ + --format sarif --output trivy-image.sarif -## trivy-release-gate: same image scan as trivy-image, but exit 1 on a -## finding instead of writing a report — the pre-push -## release gate build-and-push.yml runs once per -## arch-tagged local image, before anything is pushed. -trivy-release-gate: $(TRIVY) - $(TRIVY_IMAGE_SCAN) --exit-code 1 --format table +## trivy-comment: render SARIF as a markdown PR comment — SARIF, TITLE, OUT +# The jq program lives in tools/trivy-comment.jq rather than inline: as a file +# it is lintable (`jq -n -f`), diffable, and free of Makefile `$$`/backslash +# escaping. One target serves both scans, so the two reports cannot drift. +trivy-comment: + @test -n "$(SARIF)" -a -n "$(TITLE)" -a -n "$(OUT)" || \ + { echo "::error::trivy-comment needs SARIF, TITLE and OUT"; exit 1; } + @test -s "$(SARIF)" || \ + { echo "::error::$(SARIF) missing or empty — no scan produced it"; exit 1; } + @{ \ + echo ""; \ + echo "### 🛡️ Trivy — $(TITLE) ($(SEVERITY))"; \ + echo "[View full run]($(RUN_URL))"; \ + echo; \ + jq -r -f tools/trivy-comment.jq "$(SARIF)"; \ + } > $(OUT) -## trivy-gate: fail if either SARIF report already produced by a scan step -## carries a finding. Reads the reports rather than rescanning. +## trivy-gate: fail if either SARIF report carries a finding, or is missing +# Reads the reports the scan jobs produced rather than scanning a third and +# fourth time. A missing or unparsable report is a failure, not a pass: a scan +# that silently wrote nothing must not turn the gate into a green no-op. trivy-gate: @fail=0; \ for report in trivy-deps.sarif trivy-image.sarif; do \ - count=$$(jq '[.runs[].results[]?] | length' "$$report"); \ - echo "$${report}: $${count} $(SEVERITY)"; \ + if [ ! -s "$$report" ]; then \ + echo "$$report: MISSING — no scan produced it"; \ + fail=1; continue; \ + fi; \ + count=$$(jq '[.runs[].results[]?] | length' "$$report" 2>/dev/null); \ + if [ -z "$$count" ]; then \ + echo "$$report: UNREADABLE — not valid SARIF"; \ + fail=1; continue; \ + fi; \ + echo "$$report: $$count $(SEVERITY)"; \ if [ "$$count" -gt 0 ]; then \ jq -r '.runs[].results[]? | "\(.ruleId) \(.message.text)"' "$$report"; \ fail=1; \ fi; \ done; \ + [ "$$fail" -eq 0 ] || echo "::error::HIGH or CRITICAL Trivy findings, or a missing report — see the log above"; \ exit $$fail ## lint: vet, format check and static analysis @@ -152,27 +232,37 @@ lint: $(GOLANGCI_LINT) $(GOLANGCI_LINT) run ./... $(GOLANGCI_LINT) fmt --diff ./... -## fmt: apply the formatters lint checks for +## fmt: apply the formatters that lint checks for fmt: $(GOLANGCI_LINT) $(GOLANGCI_LINT) fmt ./... -## docker: build the adapter image +## docker: build the shipped adapter image — the Dockerfile and build args CI scans +# Dockerfile.adapter-with-plugins, not Dockerfile.adapter: the plugins image is +# what security.yml scans and what deploys, so a local `make docker && +# make trivy-image` has to scan the same thing CI gates on. The version vars +# come from the script rather than being named again here, so ONIX_VERSION and +# friends are spelled out in exactly one place. docker: - docker build -f Dockerfile.adapter -t $(IMAGE) . - -## tools: build the pinned toolchain into bin/ -tools: $(GOLANGCI_LINT) $(GOTESTSUM) $(TRIVY) + . install/scripts/version-vars.sh && \ + docker build -f Dockerfile.adapter-with-plugins \ + --build-arg ONIX_VERSION="$$ONIX_VERSION" \ + --build-arg GIT_COMMIT="$$GIT_COMMIT" \ + --build-arg GIT_TREE_STATE="$$GIT_TREE_STATE" \ + --build-arg BUILD_DATE="$$BUILD_DATE" \ + -t $(IMAGE) . ## clean: remove build output and coverage/scan artifacts clean: - rm -rf $(BIN_DIR) coverage.out coverage-report.md trivy-deps.sarif trivy-image.sarif + rm -rf $(BIN_DIR) coverage.out coverage-plugin.out coverage-report.md \ + trivy-deps.sarif trivy-image.sarif \ + trivy-deps-comment.md trivy-image-comment.md $(GOLANGCI_LINT): @mkdir -p $(BIN_DIR) GOBIN=$(abspath $(BIN_DIR)) $(GO) install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) -# gotestsum is CI-only (see run-tests.yml), so it doesn't belong in the -# adapter's or the linter's dependency graph either one. +# gotestsum is CI-only (see ci.yml), so it doesn't belong in the adapter's or +# the linter's dependency graph either one. $(GOTESTSUM): @mkdir -p $(BIN_DIR) GOBIN=$(abspath $(BIN_DIR)) $(GO) install gotest.tools/gotestsum@$(GOTESTSUM_VERSION) @@ -186,5 +276,5 @@ $(TRIVY): curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | \ sh -s -- -b $(abspath $(BIN_DIR)) $(TRIVY_VERSION) -.PHONY: help build test cover test-ci cover-diff lint fmt trivy-deps \ - trivy-image trivy-release-gate trivy-gate docker tools clean +.PHONY: help build test cover test-ci merge-coverage cover-diff lint fmt \ + trivy-deps trivy-image trivy-comment trivy-gate docker clean diff --git a/tools/trivy-comment.jq b/tools/trivy-comment.jq new file mode 100644 index 00000000..54371ac1 --- /dev/null +++ b/tools/trivy-comment.jq @@ -0,0 +1,32 @@ +# Render a Trivy SARIF report as a markdown table for a PR comment. +# +# Used by the Makefile's trivy-comment target for both the dependency and the +# image scan — one program, not a copy per scan. +# +# Trivy's SARIF carries no structured per-field severity/version columns: each +# finding's detail lives as prose in message.text ("Package: ...\nSeverity: +# ...\n..."), so the columns below are pulled out of that text rather than read +# from dedicated JSON fields. +# +# Lint with: jq -n -f tools/trivy-comment.jq + +# capture returns null when the pattern doesn't match, so `// {v: default}` +# supplies the fallback rather than letting a missing field print "null". +def val(re; default): (capture(re) // {v: default}).v; + +[ .runs[]?.results[]? ] as $found +| if ($found | length) == 0 then + "No HIGH or CRITICAL findings." + else + "| Package | Severity | Installed | Fixed in | Advisory |", + "|---|---|---|---|---|", + ( $found[] + | .ruleId as $id + | (.message.text // "") as $m + | "| `" + ($m | val("Package: (?[^\\n]+)"; "?")) + "` " + + "| " + ($m | val("Severity: (?[^\\n]+)"; "?")) + " " + + "| " + ($m | val("Installed Version: (?[^\\n]+)"; "?")) + " " + + "| " + ($m | val("Fixed Version: (?[^\\n]+)"; "—")) + " " + + "| [" + $id + "](" + ($m | val("Link: \\[[^]]+\\]\\((?[^)]+)\\)"; "")) + ") |" + ) + end From 82366234e129a75a2e67de58cb5e811cbb7a355c Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Mon, 7 Sep 2026 22:40:42 +0530 Subject: [PATCH 014/130] refactor: rename Run Tests to CI and reduce it to make calls [#5] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run-tests.yml becomes ci.yml, replacing the dead ci.yml that triggered on `beck-onix-v1.0-develop` (a typo of a branch that does not exist, so it never fired). Names now read as what they gate: the workflow is `CI` and the jobs are `Lint` and `Test`, so branch protection matches `CI / Lint` and `CI / Test` rather than the old `Run Tests / lint`. The job bodies are four `make` calls plus GitHub plumbing: make lint advisory, continue-on-error. There is no .golangci.yml in this repo yet, so `run ./...` uses tool defaults against a codebase that has never been linted — enough pre-existing findings to block every PR on day one. This surfaces them. (The comment this replaced cited a CLAUDE.md and a pre-commit hook, neither of which exists here.) make build make test-ci make cover-diff push: added `development`, the branch PRs target and merge into, so a merge landing there runs the same gates the PR did. It previously only ran on main and PRs, leaving merges into development ungated. BASE_REF goes through `env:`, not string-interpolated into the run script — `?=` in the Makefile means the environment wins, so nothing untrusted ends up inside a shell command. permissions are least-privilege per job: the workflow grants contents: read and only the Test job adds pull-requests: write for the coverage comment. the coverage comment steps are continue-on-error — a fork PR's default GITHUB_TOKEN is read-only, so create-or-update-comment 403s there. Coverage is still gated by the last step; a comment that cannot post must not fail the job over it. find-comment v3 -> v4 and create-or-update-comment v4 -> v5. The final gate step is `exit 1`, nothing more: cover-diff already emitted the ::error:: annotation and the per-file breakdown. --- .github/workflows/ci.yml | 142 +++++++++++++++++++++++++------- .github/workflows/run-tests.yml | 107 ------------------------ 2 files changed, 114 insertions(+), 135 deletions(-) delete mode 100644 .github/workflows/run-tests.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cef2efdb..9e6a53e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,36 +1,122 @@ -name: Go CI +name: CI + +# Every step below is either a one-line `make` call or GitHub plumbing +# (`${{ }}` expressions, $GITHUB_STEP_SUMMARY). The build/test/coverage logic +# lives in the Makefile, so a red check here reproduces locally by running the +# exact command the log shows. on: - pull_request: - branches: - - beck-onix-v1.0-develop - - beck-onix-v1.0 push: - branches: - - beck-onix-v1.0-develop - - beck-onix-v1.0 + # development as well as main: this is the branch PRs target and merge + # into, so a merge landing there has to run the same gates the PR did. + branches: [main, development] + pull_request: + +# Least privilege at the workflow level; the one job that needs more declares +# it for itself. +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + # Advisory, not a gate. There is no .golangci.yml in this repo yet, so + # `run ./...` uses the tool's defaults against a codebase that has never + # been linted — enough pre-existing findings to block every PR on day + # one. This surfaces them; turn continue-on-error off once the baseline + # is clean and the config is pinned. + - run: make lint + continue-on-error: true + test: + name: Test runs-on: ubuntu-latest + # pull-requests: write is for the coverage comment below — only this job + # writes to the PR, so the lint job above stays read-only. + permissions: + contents: read + pull-requests: write steps: - - uses: actions/checkout@v2 - - name: Set up Go - uses: actions/setup-go@v2 + # fetch-depth: 0 — cover-diff needs the base branch's history locally to + # compute the changed-file set; a PR checkout otherwise holds only the + # head commit. + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - run: make build + - run: make test-ci + + # Diff-scoped, never whole-repo: the repo's existing coverage is well + # under the threshold, so a whole-repo gate would fail every PR + # regardless of what it changed. What a review can act on is the number + # for the lines the PR itself touched. On pull_request the base is the + # PR's target branch; on a push it's github.event.before, the commit the + # branch pointed at before this one landed. + # + # BASE_REF goes through env, not string-interpolated into the run script: + # `?=` in the Makefile means the environment wins, and nothing untrusted + # ends up inside a shell command. + # + # continue-on-error so a coverage miss still posts its report; the last + # step in the job turns that outcome back into a failure. + - name: Check coverage of changed files + id: coverage + continue-on-error: true + env: + BASE_REF: ${{ github.event_name == 'pull_request' && format('origin/{0}', github.event.pull_request.base.ref) || github.event.before }} + run: make cover-diff + + # cover-diff writes coverage-report.md on every exit path, including its + # own error paths, so this needs no existence guard. + - name: Publish coverage to the job summary + run: cat coverage-report.md >> "$GITHUB_STEP_SUMMARY" + + # Always posted — pass, fail or not-applicable. A comment that only shows + # up on failure is ambiguous from the PR itself: did coverage run at all, + # pass, or never trigger? Updated in place across runs via find-comment, + # which matches the marker cover-diff writes into the report. + # + # continue-on-error on both: a fork PR's default GITHUB_TOKEN is + # read-only, so create-or-update-comment 403s there even with + # pull-requests: write declared. Coverage is already gated below; a + # comment that can't post must never fail the job over that. + - name: Find existing coverage comment + if: github.event_name == 'pull_request' + continue-on-error: true + uses: peter-evans/find-comment@v4 + id: find-coverage-comment with: - go-version: '1.20' - - name: Install dependencies - run: go mod tidy - - name: Run tests - run: go test -coverprofile=coverage.out ./... - - name: Check coverage - run: | - coverage=$(go tool cover -func=coverage.out | grep total | awk '{print $3}' | sed 's/%//') - if (( $(echo "$coverage < 90" | bc -l) )); then - echo "Coverage is below 90%" - exit 1 - fi - - name: Run golangci-lint - run: golangci-lint run - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5 + issue-number: ${{ github.event.pull_request.number }} + comment-author: "github-actions[bot]" + body-includes: "" + + - name: Comment coverage on the PR + if: github.event_name == 'pull_request' + continue-on-error: true + uses: peter-evans/create-or-update-comment@v5 with: - files: ./coverage.out + comment-id: ${{ steps.find-coverage-comment.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body-path: coverage-report.md + edit-mode: replace + + # Turns the continue-on-error above back into a job failure. cover-diff + # already logged the ::error:: annotation and the per-file breakdown, so + # there is nothing to say here beyond the exit code. + - name: Gate on minimum coverage + if: steps.coverage.outcome == 'failure' + run: exit 1 diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml deleted file mode 100644 index 47f4136f..00000000 --- a/.github/workflows/run-tests.yml +++ /dev/null @@ -1,107 +0,0 @@ -name: Run Tests - -on: - push: - branches: [main] - pull_request: - -permissions: - contents: read - pull-requests: write - -concurrency: - group: test-${{ github.ref }} - cancel-in-progress: true - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - - # continue-on-error rather than dropping the job: CLAUDE.md keeps CI - # lint as the safety net behind the pre-commit hook, which is bypassable - # (--no-verify). Advisory here, not a gate — a red lint check must never - # be the reason a PR can't merge, only a reason someone looks. - - run: make lint - continue-on-error: true - - test: - runs-on: ubuntu-latest - steps: - # fetch-depth: 0 — cover-diff below needs the base branch's history - # locally to compute the PR's changed files; a PR checkout otherwise - # holds only the head commit. - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - - run: make build - - run: make test-ci - - # continue-on-error so a coverage miss still posts its report; the last - # step in the job turns that outcome back into a failure. Always - # diff-scoped, never whole-repo — main only ever moves by PR merge - # (CLAUDE.md: never commit directly to main), so a PR's own diff is - # already the gate that matters. On a pull_request event that's the PR - # branch vs its target; on a push to main (the merge landing) it's - # github.event.before, the commit main pointed to right before this one. - - name: Check coverage - id: coverage - continue-on-error: true - run: | - if [ "${{ github.event_name }}" = "pull_request" ]; then - make cover-diff BASE_REF="origin/${{ github.event.pull_request.base.ref }}" - else - make cover-diff BASE_REF="${{ github.event.before }}" - fi - - # Marker goes into the file itself, not just the step summary — it's - # what body-path feeds the PR comment below, and find-comment matches - # on this exact string to update rather than duplicate that comment. - - name: Publish coverage summary - run: | - printf '\n%s\n' "$(cat coverage-report.md)" > coverage-report.md - cat coverage-report.md >> "$GITHUB_STEP_SUMMARY" - - # Always posted, pass or fail or not-applicable — a comment that only - # shows up on failure is ambiguous from the PR itself: did coverage - # run at all, pass, or just never trigger? Always-visible with an - # explicit ✅/❌ (cover-diff's own output) answers that without a - # reviewer having to open the job. Updated in place across runs, not - # reposted, via find-comment below. - # - # continue-on-error on both: a fork PR's default GITHUB_TOKEN is - # read-only, so create-or-update-comment 403s there even with - # pull-requests: write declared. A comment that can't post must never - # fail the job — coverage already gated above, this step is cosmetic. - - name: Find existing coverage comment - if: github.event_name == 'pull_request' - continue-on-error: true - uses: peter-evans/find-comment@v3 - id: find-coverage-comment - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: "github-actions[bot]" - body-includes: "" - - - name: Comment coverage on PR - if: github.event_name == 'pull_request' - continue-on-error: true - uses: peter-evans/create-or-update-comment@v4 - with: - comment-id: ${{ steps.find-coverage-comment.outputs.comment-id }} - issue-number: ${{ github.event.pull_request.number }} - body-path: coverage-report.md - edit-mode: replace - - - name: Gate on minimum coverage - if: steps.coverage.outcome == 'failure' - run: | - echo "::error::coverage is below the minimum — see the summary above" - exit 1 From 4f7961419cf8d075e0de3c833fe22062e95bdc0c Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Mon, 7 Sep 2026 22:42:15 +0530 Subject: [PATCH 015/130] refactor: split Security into dependency-scan, image-scan and security-gate [#5] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The single `security` job scanned dependencies, then built the image, then scanned that, then gated — serially, in one job, so a dependency finding waited on an image build that has nothing to do with it, and the PR could not say which scan found what without opening a log. Three jobs now, named for what they do: `Dependency scan` and `Image scan` run in parallel (the dependency scan needs nothing but a checkout), and `Security gate` fans in with needs + if: always(). Branch protection matches `Security / Security gate`. Every step in the file is a one-line `make` call or GitHub plumbing — zero lines of inline shell, down from 132 across the three original workflows: make trivy-deps make trivy-comment SARIF=... TITLE=... OUT=... make docker IMAGE=network-adapter: make trivy-image IMAGE=network-adapter: make trivy-gate Also fixed: the gate now reads the SARIF the scan jobs produced, handed over as artifacts, rather than scanning a third and fourth time. if-no-files-found is deliberately `ignore`: a scan that produced nothing surfaces as the gate reporting MISSING — which now fails — with both verdicts visible in one place, rather than as a second red check on the scan job. push: added `development`, so a merge landing there runs the same scans the PR did. permissions are least-privilege per job: contents: read at the workflow level, with pull-requests: write and security-events: write only on the two scan jobs. The gate stays read-only. both upload-sarif steps are continue-on-error — a fork PR's GITHUB_TOKEN has no security-events scope, so they 403 there. The gate decides pass/fail; an upload that cannot happen must not fail the scan over it. a `category` per scan, so the image upload no longer replaces the dependency analysis (GitHub keys an analysis on ref + category). find-comment v3 -> v4, create-or-update-comment v4 -> v5. .github/actions/trivy-cache is the `Trivy cache date` + actions/cache pair, extracted now that two jobs need it — both jobs stay a one-line `uses:`. It keeps the trivy binary (~30MB) and the vulnerability DB (~100MB+) out of every run, which the trivy-action it replaced did for us. --- .github/actions/trivy-cache/action.yml | 26 +++ .github/workflows/security.yml | 278 +++++++++++-------------- 2 files changed, 152 insertions(+), 152 deletions(-) create mode 100644 .github/actions/trivy-cache/action.yml diff --git a/.github/actions/trivy-cache/action.yml b/.github/actions/trivy-cache/action.yml new file mode 100644 index 00000000..d92cf37c --- /dev/null +++ b/.github/actions/trivy-cache/action.yml @@ -0,0 +1,26 @@ +name: Cache Trivy +description: > + Restores the pinned trivy binary and its vulnerability DB from the Actions + cache. Both scan jobs need this, so it lives here rather than being pasted + into each one — the trivy-action this replaced cached both for us, and + `make trivy-deps`/`trivy-image` alone would re-download the binary (~30MB) + and the DB (~100MB+) on every run. + +runs: + using: composite + steps: + # Keyed on the Makefile's own hash so a TRIVY_VERSION bump there busts the + # cache automatically, plus a daily date so the DB itself never goes stale + # for longer than a day. + - id: date + shell: bash + run: echo "today=$(date -u +%F)" >> "$GITHUB_OUTPUT" + + - uses: actions/cache@v4 + with: + path: | + bin/trivy + ~/.cache/trivy + key: trivy-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Makefile') }}-${{ steps.date.outputs.today }} + restore-keys: | + trivy-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Makefile') }}- diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index cfc42c5c..bfe1e174 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -1,215 +1,189 @@ name: Security +# Three jobs, so the PR says *which* scan found something without anyone +# opening a log: the dependency scan needs nothing but a checkout, so it runs +# alongside the image build instead of queueing behind it, and the gate reads +# both reports once they land. +# +# Every step is a one-line `make` call or GitHub plumbing — the scan, report +# and gate logic lives in the Makefile, so `make trivy-deps` / `make docker && +# make trivy-image` / `make trivy-gate` reproduce any red check locally. on: push: - branches: [main] + # development as well as main: this is the branch PRs target and merge + # into, so a merge landing there has to run the same scans the PR did. + branches: [main, development] pull_request: +# Least privilege at the workflow level; the scan jobs declare the extra +# scopes they need, so the gate job stays read-only. permissions: contents: read - # Findings are uploaded as SARIF for GitHub's own code scanning UI (Security - # tab, diff annotations) AND posted as a PR comment — the SARIF path alone - # took three clicks (check -> run -> results) to see a finding that used to - # be one scroll away in the PR conversation. pull-requests: write is for - # the comment; security-events: write is for the SARIF upload. - pull-requests: write - security-events: write concurrency: group: security-${{ github.ref }} cancel-in-progress: true jobs: - security: + dependency-scan: + name: Dependency scan runs-on: ubuntu-latest + # Findings go to GitHub's code scanning UI as SARIF (Security tab, diff + # annotations) AND to a PR comment — the SARIF path alone took three clicks + # (check -> run -> results) to see a finding that is one scroll away in the + # PR conversation. pull-requests: write is the comment, security-events: + # write is the SARIF upload. + permissions: + contents: read + pull-requests: write + security-events: write steps: - uses: actions/checkout@v4 + - uses: ./.github/actions/trivy-cache - # The trivy-action this replaced cached both the binary and its - # vulnerability DB across runs; `make trivy-deps`/`trivy-image` alone - # would re-download both (binary ~30MB, DB ~100MB+) on every run. - # Keyed on the Makefile's own hash so a TRIVY_VERSION bump there busts - # the cache automatically, plus a daily date so the DB itself doesn't - # go stale for longer than a day. - - name: Trivy cache date - id: trivy-date - run: echo "date=$(date -u +%F)" >> "$GITHUB_OUTPUT" - - - name: Cache Trivy binary and vulnerability DB - uses: actions/cache@v4 - with: - path: | - bin/trivy - ~/.cache/trivy - key: trivy-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Makefile') }}-${{ steps.trivy-date.outputs.date }} - restore-keys: | - trivy-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Makefile') }}- - - # T4, half one: the dependency graph as written. This catches what the - # image scan structurally cannot — a vulnerable module that only the test - # suite imports, so it is never linked into the binary and never appears - # in a layer. + # Catches what the image scan structurally cannot: a vulnerable module + # only the test suite imports, so it is never linked into the binary and + # never appears in a layer. + - run: make trivy-deps + + # continue-on-error: a fork PR's GITHUB_TOKEN has no security-events + # write, so this 403s there. The gate job is what decides pass/fail; an + # upload that can't happen must not fail the scan over it. # - # Logic lives in the Makefile's trivy-deps target — same command a local - # `make trivy-deps` runs. - - name: Trivy dependency scan (report) - run: make trivy-deps - - # A category per scan: without it the second upload replaces the first, - # because GitHub keys an analysis on (ref, category). - - name: Upload dependency findings + # A category per scan — GitHub keys an analysis on (ref, category), so + # without it the image upload would replace this one. + - name: Upload findings to code scanning + continue-on-error: true uses: github/codeql-action/upload-sarif@v3 with: sarif_file: trivy-deps.sarif category: trivy-dependencies - # Posted right after the report, on every run — pass or fail, and - # before the image is even built — so a PR shows the dependency finding - # immediately rather than after the whole job finishes. Reads the same - # SARIF just uploaded above; no second scan. - # - # Trivy's SARIF has no structured per-field severity/version columns; - # each finding's detail lives as prose in message.text ("Package: - # ...\nSeverity: ...\n..."), so the table columns below are pulled out - # of that text rather than from dedicated JSON fields. - # continue-on-error: a report-formatting step must never be able to - # fail the job — the gate at the bottom is what decides that. - - name: Assemble Trivy dependency PR comment + # Rendered from the SARIF just uploaded, no second scan. Posted before + # the image scan has even finished, so a PR shows a dependency finding + # immediately rather than after the whole workflow completes. + - name: Assemble the PR comment if: github.event_name == 'pull_request' continue-on-error: true - run: | - { - echo "" - echo "### 🛡️ Trivy — Go dependency scan (HIGH,CRITICAL)" - echo "[View full run](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" - echo - jq -r ' - def val(re; default): (capture(re) // {v: default}).v; - [ .runs[].results[]? ] as $found - | if ($found | length) == 0 then - "No HIGH or CRITICAL findings." - else - "| Package | Severity | Installed | Fixed in | Advisory |", - "|---|---|---|---|---|", - ( $found[] | .ruleId as $id | (.message.text // "") as $m | - "| `" + ($m | val("Package: (?[^\\n]+)"; "?")) + "` " - + "| " + ($m | val("Severity: (?[^\\n]+)"; "?")) + " " - + "| " + ($m | val("Installed Version: (?[^\\n]+)"; "?")) + " " - + "| " + ($m | val("Fixed Version: (?[^\\n]+)"; "—")) + " " - + "| [" + $id + "](" + ($m | val("Link: \\[[^]]+\\]\\((?[^)]+)\\)"; "")) + ") |" - ) - end - ' trivy-deps.sarif - } > trivy-deps-comment.md - - - name: Find existing Trivy dependency comment + run: make trivy-comment SARIF=trivy-deps.sarif TITLE="Go dependency scan" OUT=trivy-deps-comment.md + + - name: Find the existing PR comment if: github.event_name == 'pull_request' continue-on-error: true - uses: peter-evans/find-comment@v3 - id: find-trivy-deps-comment + uses: peter-evans/find-comment@v4 + id: find-comment with: issue-number: ${{ github.event.pull_request.number }} comment-author: "github-actions[bot]" body-includes: "" # continue-on-error: a fork PR's default GITHUB_TOKEN is read-only, so - # this 403s there even with pull-requests: write declared. The gate at - # the bottom of this job is what actually decides pass/fail; a comment - # that can't post must never fail the job over that. - - name: Comment Trivy dependency report on PR + # this 403s there even with pull-requests: write declared. + - name: Comment the report on the PR if: github.event_name == 'pull_request' continue-on-error: true - uses: peter-evans/create-or-update-comment@v4 + uses: peter-evans/create-or-update-comment@v5 with: - comment-id: ${{ steps.find-trivy-deps-comment.outputs.comment-id }} + comment-id: ${{ steps.find-comment.outputs.comment-id }} issue-number: ${{ github.event.pull_request.number }} body-path: trivy-deps-comment.md edit-mode: replace - - name: Log in to dhi.io - uses: ./.github/actions/dhi-login + # Handed to the gate job, which runs on a different runner. if-no-files-found + # is deliberately `ignore` and not `error`: a scan that produced nothing + # should surface as the gate reporting a MISSING report, with both scans' + # verdicts visible, rather than as a second red check here. + - name: Hand the report to the gate + if: always() + uses: actions/upload-artifact@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - token: ${{ secrets.DOCKERHUB_TOKEN }} - - # Both Dockerfiles default to dhi.io/* base images, and Trivy reads the - # base layers plus the Go build info embedded in the binary — including - # `stdlib`, so a Go toolchain CVE is reported here and nowhere else the - # dependency scan above cannot see. - - name: Build image - run: | - source install/scripts/version-vars.sh - docker build -f Dockerfile.adapter-with-plugins \ - --build-arg ONIX_VERSION="$ONIX_VERSION" \ - --build-arg GIT_COMMIT="$GIT_COMMIT" \ - --build-arg GIT_TREE_STATE="$GIT_TREE_STATE" \ - --build-arg BUILD_DATE="$BUILD_DATE" \ - -t network-adapter:${{ github.sha }} . - # Plain `docker build`, not build-push-action, matches security.yml's - # single-arch report scan — build-and-push.yml is where per-arch, - # buildx-driven builds actually matter. - - - name: Trivy image scan (report) - run: make trivy-image IMAGE=network-adapter:${{ github.sha }} - - - name: Upload image findings + name: trivy-deps-sarif + path: trivy-deps.sarif + if-no-files-found: ignore + + image-scan: + name: Image scan + runs-on: ubuntu-latest + # Findings go to GitHub's code scanning UI as SARIF (Security tab, diff + # annotations) AND to a PR comment — the SARIF path alone took three clicks + # (check -> run -> results) to see a finding that is one scroll away in the + # PR conversation. pull-requests: write is the comment, security-events: + # write is the SARIF upload. + permissions: + contents: read + pull-requests: write + security-events: write + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/trivy-cache + + # The image the adapter actually ships — `make docker` builds + # Dockerfile.adapter-with-plugins with the same version build args, so a + # local `make docker && make trivy-image` scans what this gates on. + # + # Trivy reads the base layers plus the Go build info embedded in the + # binary, including `stdlib` — so a Go toolchain CVE shows up here and + # nowhere the dependency scan can see. + - run: make docker IMAGE=network-adapter:${{ github.sha }} + - run: make trivy-image IMAGE=network-adapter:${{ github.sha }} + + - name: Upload findings to code scanning + continue-on-error: true uses: github/codeql-action/upload-sarif@v3 with: sarif_file: trivy-image.sarif category: trivy-image - - name: Assemble Trivy image PR comment + - name: Assemble the PR comment if: github.event_name == 'pull_request' continue-on-error: true - run: | - { - echo "" - echo "### 🛡️ Trivy — image scan (HIGH,CRITICAL)" - echo "[View full run](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" - echo - jq -r ' - def val(re; default): (capture(re) // {v: default}).v; - [ .runs[].results[]? ] as $found - | if ($found | length) == 0 then - "No HIGH or CRITICAL findings." - else - "| Package | Severity | Installed | Fixed in | Advisory |", - "|---|---|---|---|---|", - ( $found[] | .ruleId as $id | (.message.text // "") as $m | - "| `" + ($m | val("Package: (?[^\\n]+)"; "?")) + "` " - + "| " + ($m | val("Severity: (?[^\\n]+)"; "?")) + " " - + "| " + ($m | val("Installed Version: (?[^\\n]+)"; "?")) + " " - + "| " + ($m | val("Fixed Version: (?[^\\n]+)"; "—")) + " " - + "| [" + $id + "](" + ($m | val("Link: \\[[^]]+\\]\\((?[^)]+)\\)"; "")) + ") |" - ) - end - ' trivy-image.sarif - } > trivy-image-comment.md - - - name: Find existing Trivy image comment + run: make trivy-comment SARIF=trivy-image.sarif TITLE="Image scan" OUT=trivy-image-comment.md + + - name: Find the existing PR comment if: github.event_name == 'pull_request' continue-on-error: true - uses: peter-evans/find-comment@v3 - id: find-trivy-image-comment + uses: peter-evans/find-comment@v4 + id: find-comment with: issue-number: ${{ github.event.pull_request.number }} comment-author: "github-actions[bot]" body-includes: "" - - name: Comment Trivy image report on PR + - name: Comment the report on the PR if: github.event_name == 'pull_request' continue-on-error: true - uses: peter-evans/create-or-update-comment@v4 + uses: peter-evans/create-or-update-comment@v5 with: - comment-id: ${{ steps.find-trivy-image-comment.outputs.comment-id }} + comment-id: ${{ steps.find-comment.outputs.comment-id }} issue-number: ${{ github.event.pull_request.number }} body-path: trivy-image-comment.md edit-mode: replace - # Gate on the reports already produced rather than scanning a third and - # fourth time — logic lives in the Makefile's trivy-gate target so a - # local `make trivy-gate` checks the exact same two files this job - # just wrote. Last in the job, so a red build still leaves both reports - # behind. - - name: Gate on HIGH or CRITICAL findings - run: make trivy-gate || { echo "::error::HIGH or CRITICAL Trivy findings — see job log above"; exit 1; } + - name: Hand the report to the gate + if: always() + uses: actions/upload-artifact@v4 + with: + name: trivy-image-sarif + path: trivy-image.sarif + if-no-files-found: ignore + + security-gate: + name: Security gate + needs: [dependency-scan, image-scan] + # always(): the gate has to run even when a scan job failed, so a failed + # scan is reported as a missing report rather than skipping the gate + # entirely. make trivy-gate fails on a missing or unparsable report for + # exactly this reason. + if: always() + runs-on: ubuntu-latest + steps: + # Checkout is for the Makefile, not the source — this job only reads the + # two SARIF reports the scan jobs produced. + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + pattern: trivy-*-sarif + merge-multiple: true + + - run: make trivy-gate From 6e684133132bb30634814d969c2fa562e6c8a3ae Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Mon, 7 Sep 2026 22:43:30 +0530 Subject: [PATCH 016/130] ci: cancel a PR's superseded runs, keep the verdict on trunk commits [#5] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both workflows already grouped on github.ref with cancel-in-progress, so a new commit on a PR did supersede the previous one's runs — but the intent was undocumented and the flag was a bare `true`, which also cancelled runs on main and development. cancel-in-progress is now scoped to pull_request. On a PR, pushing a new commit cancels the CI and Security runs still in flight for the commit before it: github.ref is refs/pull//merge for the whole life of the PR, so every push lands in the same group. On main and development nothing is cancelled — each commit there keeps its own recorded verdict, instead of being left with a grey check and no way to tell afterwards whether it was ever green. The two workflows keep separate groups, so cancelling a superseded CI run does not take that commit's Security run down with it. --- .github/workflows/ci.yml | 10 +++++++++- .github/workflows/security.yml | 7 ++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e6a53e7..868d6e84 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,9 +16,17 @@ on: permissions: contents: read +# A new commit on a PR makes the previous commit's runs worthless, so they are +# cancelled rather than left to finish. github.ref is refs/pull//merge for a +# pull_request event, so every push to the PR's head branch lands in the same +# group and supersedes the run before it; a push to main or development gets +# its own group and never collides with a PR's. concurrency: group: ci-${{ github.ref }} - cancel-in-progress: true + # PRs only. On main and development every commit keeps its own recorded + # verdict — cancelling there would leave trunk commits with a grey check and + # no way to tell afterwards whether that commit was ever green. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: lint: diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index bfe1e174..d4b3ea64 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -20,9 +20,14 @@ on: permissions: contents: read +# Same rule as ci.yml: a new commit on a PR supersedes the scans of the one +# before it. Its own group, not shared with ci.yml, so cancelling a superseded +# CI run does not take the Security run for that commit down with it. concurrency: group: security-${{ github.ref }} - cancel-in-progress: true + # PRs only — see ci.yml. A scan verdict on a trunk commit is a record worth + # keeping. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: dependency-scan: From c62e192affc4e35f201e35c89d13fa3da57a2f98 Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Mon, 7 Sep 2026 22:51:49 +0530 Subject: [PATCH 017/130] build: add lint-actions, lint-staged and a versioned pre-commit hook [#5] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing in this repo validated the workflow files. A bad ${{ }} expression, a mistyped `uses:` ref or a `needs:` pointing at a job that does not exist was only discoverable by running the workflow — or by running actionlint by hand. Deliberately not a CI step. The gate is a pre-commit hook, so the feedback lands before the commit exists rather than after a push: lint-actions actionlint (pinned v1.7.12, go-installable like the other tools) over the whole .github tree. Whole-repo rather than per-file because actionlint resolves `needs:` across a workflow's jobs and checks `uses: ./.github/actions/...` against the action on disk. lint-staged what the hook runs. Fires actionlint only when a workflow or composite action is staged, and gofmt only over the staged .go files. hooks sets core.hooksPath to .githooks. Run once per clone. .githooks/pre-commit is a one-line `exec make lint-staged`: the logic stays in the Makefile, so `make lint-staged` reproduces exactly what blocked a commit, and there is a single place to change what runs. core.hooksPath rather than copying into .git/hooks means the hook is versioned and reviewable, and a change to it reaches everyone on their next pull. Scope is what can pass today, so the hook never becomes something people reflexively --no-verify past: - actionlint reports nothing on the current tree, so it blocks from day one. - 22 files in the repo are not gofmt-clean. Staged-only means that history stays someone else's problem until you touch one of those files. - `golangci-lint run` is excluded: with no .golangci.yml it uses tool defaults against a codebase that has never been linted, so it would reject every commit. - the test suite is excluded: a pre-commit hook has to stay in seconds, and `make test-ci` in CI is where that belongs. Verified all six paths: nothing staged, staged Go/workflow with a clean tree, actionlint catching a bad steps. reference, gofmt rejecting and then accepting a staged file, a real `git commit` blocked by the installed hook, and --no-verify still getting through. --- .githooks/pre-commit | 13 ++++++++ Makefile | 75 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100755 .githooks/pre-commit diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 00000000..560e2e59 --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,13 @@ +#!/bin/sh +# +# Pre-commit gate. Installed by `make hooks`, which sets core.hooksPath to this +# directory — so this file is versioned, reviewable, and reaches everyone on +# their next pull rather than only the people who remember to re-copy it. +# +# A thin wrapper on purpose: the checks and the staged-file logic live in the +# Makefile's lint-staged target, so `make lint-staged` reproduces exactly what +# blocked a commit, and there is one place to change what runs. +# +# To bypass in an emergency: git commit --no-verify + +exec make --no-print-directory lint-staged diff --git a/Makefile b/Makefile index b28b905d..ec5405ba 100644 --- a/Makefile +++ b/Makefile @@ -23,10 +23,17 @@ SEVERITY ?= HIGH,CRITICAL GOLANGCI_LINT_VERSION := v2.5.0 GOTESTSUM_VERSION := v1.13.0 TRIVY_VERSION := v0.74.0 +ACTIONLINT_VERSION := v1.7.12 GOLANGCI_LINT := $(BIN_DIR)/golangci-lint GOTESTSUM := $(BIN_DIR)/gotestsum TRIVY := $(BIN_DIR)/trivy +ACTIONLINT := $(BIN_DIR)/actionlint + +# From GOROOT, not PATH: `go` is always resolvable here (every other target +# needs it), and gofmt sits next to it, so this works even where only the +# toolchain's bin dir is on PATH. Expanded at recipe time, hence the `$$`. +GOFMT = $$($(GO) env GOROOT)/bin/gofmt # pkg/plugin and benchmarks/e2e each build a real .so with a plain `go build # -buildmode=plugin` subprocess, then load it with plugin.Open in the same test @@ -236,6 +243,68 @@ lint: $(GOLANGCI_LINT) fmt: $(GOLANGCI_LINT) $(GOLANGCI_LINT) fmt ./... +## lint-actions: validate the workflows and composite actions +# Not wired into ci.yml on purpose — the pre-commit hook is the gate. Run +# whole-repo rather than per-file: actionlint resolves `needs:` across a +# workflow's jobs and checks `uses: ./.github/actions/...` against the action +# on disk, so a single file in isolation is not enough to judge either. +lint-actions: $(ACTIONLINT) + $(ACTIONLINT) + +## lint-staged: the pre-commit lints, against the staged files only. What the hook runs. +# Staged-only, and only these two checks, because both can pass today: +# +# workflows actionlint reports nothing on the current tree, so it blocks +# from day one. +# formatting 22 files in the repo are not gofmt-clean. Scoped to what you +# staged, that history is someone else's problem until you touch +# one of those files — at which point `make fmt` fixes it. +# +# Deliberately NOT `golangci-lint run`: with no .golangci.yml it uses tool +# defaults against a codebase that has never been linted, so it would reject +# every commit. Nor the test suite — a pre-commit hook has to stay in seconds, +# and CI is where `make test-ci` belongs. +# +# Reads the working tree, not the staged blob. A file staged clean but dirty in +# the working copy is reported here; that is the conservative direction, and +# avoids checking out the index to a temp dir on every commit. +lint-staged: + @STAGED=$$(git diff --cached --name-only --diff-filter=ACMR); \ + if [ -z "$$STAGED" ]; then \ + echo "lint-staged: nothing staged"; \ + exit 0; \ + fi; \ + fail=0; \ + if printf '%s\n' "$$STAGED" | grep -qE '^\.github/(workflows/.*\.ya?ml|actions/.*/action\.ya?ml)$$'; then \ + echo "==> lint-actions (staged workflow or action change)"; \ + $(MAKE) --no-print-directory lint-actions || fail=1; \ + fi; \ + GOFILES=$$(printf '%s\n' "$$STAGED" | grep '\.go$$' || true); \ + if [ -n "$$GOFILES" ]; then \ + echo "==> gofmt (staged Go files)"; \ + UNFMT=$$(printf '%s\n' "$$GOFILES" | xargs $(GOFMT) -l); \ + if [ -n "$$UNFMT" ]; then \ + echo "not gofmt-clean:"; \ + printf ' %s\n' $$UNFMT; \ + echo "run \`make fmt\` (or gofmt -w on the files above), then stage the result"; \ + fail=1; \ + fi; \ + fi; \ + if [ "$$fail" -ne 0 ]; then \ + echo; \ + echo "pre-commit checks failed — commit aborted"; \ + exit 1; \ + fi; \ + echo "lint-staged: ok" + +## hooks: point git at the repo's versioned hooks (run once per clone) +# core.hooksPath rather than copying into .git/hooks: the hook stays in the +# repo, under review, and a change to it reaches everyone on their next pull +# instead of only the people who re-copy it. +hooks: + git config core.hooksPath .githooks + @echo "core.hooksPath -> .githooks, running: $$(ls .githooks | tr '\n' ' ')" + ## docker: build the shipped adapter image — the Dockerfile and build args CI scans # Dockerfile.adapter-with-plugins, not Dockerfile.adapter: the plugins image is # what security.yml scans and what deploys, so a local `make docker && @@ -267,6 +336,11 @@ $(GOTESTSUM): @mkdir -p $(BIN_DIR) GOBIN=$(abspath $(BIN_DIR)) $(GO) install gotest.tools/gotestsum@$(GOTESTSUM_VERSION) +# Pinned like the others, and go-installable, so no curl | sh for this one. +$(ACTIONLINT): + @mkdir -p $(BIN_DIR) + GOBIN=$(abspath $(BIN_DIR)) $(GO) install github.com/rhysd/actionlint/cmd/actionlint@$(ACTIONLINT_VERSION) + # The prebuilt release binary, not `go install`: trivy's rpm-db parser needs # cgo, and its module graph is comparable in size to golangci-lint's for a # tool nothing here imports — the official install script is what @@ -277,4 +351,5 @@ $(TRIVY): sh -s -- -b $(abspath $(BIN_DIR)) $(TRIVY_VERSION) .PHONY: help build test cover test-ci merge-coverage cover-diff lint fmt \ + lint-actions lint-staged hooks \ trivy-deps trivy-image trivy-comment trivy-gate docker clean From 00140c9bbd7e7418e11d0268cce7d8730fe1af1a Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Mon, 7 Sep 2026 23:11:31 +0530 Subject: [PATCH 018/130] chore: cut CI down to four checks and publish the image on a tag [#5] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two checks on a PR, two on a tag, one workflow. CI / End to End verification build, test, changed-line coverage, result posted to the PR and the job summary CI / Security Scan trivy on the dependency graph and on the shipped image, one comment, one gate CI / Build Artifact (amd64) push the image to ghcr.io by digest, CI / Build Artifact (arm64) natively on a runner of that arch CI / Publish Artifact bind the version tag (and :latest for a plain vX.Y.Z) to both digests Dropped the Lint check. It was continue-on-error, so it gated nothing: there is no .golangci.yml yet and `golangci-lint run` against a never-linted codebase reports enough to block every PR, which is why it was advisory in the first place. `make lint` still exists for local use, and the .githooks pre-commit hook is what actually enforces gofmt on every commit. Folded security.yml into ci.yml. Dependency scan, image scan and the gate were three jobs and three checks to answer one question; they are one job now, which also removes the SARIF artifact handoff between runners and turns the two per-scan PR comments into one. trivy-comment's SARIF/TITLE/OUT parameters existed only to serve two jobs and are gone with them — trivy-report renders both reports from the SARIF_REPORTS list that trivy-gate reads, so the comment and the gate cannot disagree. Tag triggers accept 1.0.0 and 1.0.0-RC1 as well as the v-prefixed spelling every existing tag in this repo uses. Each arch builds on a runner of that arch rather than under QEMU, because the image compiles every plugin with `go build -buildmode=plugin` and a plugin .so has to match the adapter binary's GOARCH. Build Artifact pushes by digest with no tag, so only Publish Artifact creates something visible, and a tag can never resolve to one architecture. Job conditions test startsWith(github.ref, 'refs/tags/') rather than github.ref_type: on a pull_request the ref is refs/pull//merge and ref_type is not documented to be 'branch' there, so a ref_type test risked skipping both PR checks and reporting green for a run that did nothing. --- .github/workflows/ci.yml | 243 ++++++++++++++++++++++++++++----- .github/workflows/security.yml | 194 -------------------------- Makefile | 140 +++++++++++++++---- 3 files changed, 322 insertions(+), 255 deletions(-) delete mode 100644 .github/workflows/security.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 868d6e84..5b0cd5e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,56 +1,60 @@ name: CI +# Four checks and no more. On a PR or a trunk push: "End to End verification" +# and "Security Scan". On a version tag: "Build Artifact" and +# "Publish Artifact". Nothing runs on both event kinds, so a PR never shows a +# publish check and a tag never re-runs the tests the PR already gated on. +# # Every step below is either a one-line `make` call or GitHub plumbing -# (`${{ }}` expressions, $GITHUB_STEP_SUMMARY). The build/test/coverage logic -# lives in the Makefile, so a red check here reproduces locally by running the -# exact command the log shows. +# (`${{ }}` expressions, $GITHUB_STEP_SUMMARY). The build, test, scan and +# publish logic lives in the Makefile, so a red check reproduces locally by +# running the exact command the log shows. on: push: # development as well as main: this is the branch PRs target and merge - # into, so a merge landing there has to run the same gates the PR did. + # into, so a merge landing there runs the same gates the PR did. branches: [main, development] + # Both spellings on purpose. Every tag in this repo so far is v-prefixed + # (v1.8.2, v2.0.1-rc1), but a release cut as 1.0.0 or 1.0.0-RC1 should + # publish too rather than silently do nothing. These are GitHub filter + # patterns, not regexes: + and [] work, so the digits are real digits and + # a branch-shaped tag like `release/foo` cannot match. + tags: + - "[0-9]+.[0-9]+.[0-9]+" + - "[0-9]+.[0-9]+.[0-9]+-*" + - "v[0-9]+.[0-9]+.[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+-*" pull_request: -# Least privilege at the workflow level; the one job that needs more declares -# it for itself. +# Least privilege at the workflow level; the three jobs that need more declare +# it for themselves. permissions: contents: read # A new commit on a PR makes the previous commit's runs worthless, so they are # cancelled rather than left to finish. github.ref is refs/pull//merge for a # pull_request event, so every push to the PR's head branch lands in the same -# group and supersedes the run before it; a push to main or development gets -# its own group and never collides with a PR's. +# group and supersedes the run before it. concurrency: group: ci-${{ github.ref }} - # PRs only. On main and development every commit keeps its own recorded - # verdict — cancelling there would leave trunk commits with a grey check and - # no way to tell afterwards whether that commit was ever green. + # PRs only. On main, development and tags every commit keeps its own recorded + # verdict — cancelling there would leave a trunk commit with a grey check and + # no way to tell afterwards whether it was ever green, and would abort a + # half-finished publish. cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - lint: - name: Lint + verify: + name: End to End verification + # Everything except a tag push, i.e. PRs and trunk pushes. Tested on + # github.ref rather than github.ref_type: on a pull_request the ref is + # refs/pull//merge, which is neither refs/heads/ nor refs/tags/, and + # ref_type is not documented to be 'branch' there. Guessing wrong would + # skip this job on every PR and show a green check for a run that did + # nothing, so the condition names the one case it means to exclude. + if: ${{ !startsWith(github.ref, 'refs/tags/') }} runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - - # Advisory, not a gate. There is no .golangci.yml in this repo yet, so - # `run ./...` uses the tool's defaults against a codebase that has never - # been linted — enough pre-existing findings to block every PR on day - # one. This surfaces them; turn continue-on-error off once the baseline - # is clean and the config is pinned. - - run: make lint - continue-on-error: true - - test: - name: Test - runs-on: ubuntu-latest - # pull-requests: write is for the coverage comment below — only this job - # writes to the PR, so the lint job above stays read-only. + # pull-requests: write is for the coverage comment below. permissions: contents: read pull-requests: write @@ -90,7 +94,7 @@ jobs: # cover-diff writes coverage-report.md on every exit path, including its # own error paths, so this needs no existence guard. - - name: Publish coverage to the job summary + - name: Post the result to the job summary run: cat coverage-report.md >> "$GITHUB_STEP_SUMMARY" # Always posted — pass, fail or not-applicable. A comment that only shows @@ -102,7 +106,7 @@ jobs: # read-only, so create-or-update-comment 403s there even with # pull-requests: write declared. Coverage is already gated below; a # comment that can't post must never fail the job over that. - - name: Find existing coverage comment + - name: Find the existing result comment if: github.event_name == 'pull_request' continue-on-error: true uses: peter-evans/find-comment@v4 @@ -112,7 +116,7 @@ jobs: comment-author: "github-actions[bot]" body-includes: "" - - name: Comment coverage on the PR + - name: Post the result to the PR if: github.event_name == 'pull_request' continue-on-error: true uses: peter-evans/create-or-update-comment@v5 @@ -128,3 +132,172 @@ jobs: - name: Gate on minimum coverage if: steps.coverage.outcome == 'failure' run: exit 1 + + security-scan: + name: Security Scan + # Same condition as End to End verification above — see the note there. + if: ${{ !startsWith(github.ref, 'refs/tags/') }} + runs-on: ubuntu-latest + # Both scans in one job, one check. They were two jobs plus a gate job, and + # three red-or-green checks to say one thing — "is this branch shippable" — + # was more to read than it was worth. The dependency scan needs only a + # checkout, so it runs first and its findings are in the log before the + # image finishes building. + # + # pull-requests: write is the PR comment, security-events: write is the + # SARIF upload to the Security tab. + permissions: + contents: read + pull-requests: write + security-events: write + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/trivy-cache + + # Catches what the image scan structurally cannot: a vulnerable module + # only the test suite imports, so it is never linked into the binary and + # never appears in a layer. + - run: make trivy-deps + + # And the reverse — Trivy reads the image's base layers plus the Go build + # info embedded in the binary, including `stdlib`, so a Go toolchain CVE + # shows up here and nowhere the dependency scan can see. Hence both. + - run: make docker IMAGE=network-adapter:${{ github.sha }} + - run: make trivy-image IMAGE=network-adapter:${{ github.sha }} + + # continue-on-error: a fork PR's GITHUB_TOKEN has no security-events + # write, so these 403 there. make trivy-gate below is what decides + # pass/fail; an upload that can't happen must not fail the job over it. + # + # A category per scan — GitHub keys an analysis on (ref, category), so + # without them the second upload would replace the first. + - name: Upload dependency findings to code scanning + continue-on-error: true + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: trivy-deps.sarif + category: trivy-dependencies + + - name: Upload image findings to code scanning + continue-on-error: true + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: trivy-image.sarif + category: trivy-image + + # One comment covering both scans, rendered from the SARIF already + # written — no third and fourth scan. + - name: Assemble the PR comment + if: github.event_name == 'pull_request' + continue-on-error: true + run: make trivy-report + + - name: Find the existing scan comment + if: github.event_name == 'pull_request' + continue-on-error: true + uses: peter-evans/find-comment@v4 + id: find-comment + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: "github-actions[bot]" + body-includes: "" + + - name: Post the scan report to the PR + if: github.event_name == 'pull_request' + continue-on-error: true + uses: peter-evans/create-or-update-comment@v5 + with: + comment-id: ${{ steps.find-comment.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body-path: trivy-report.md + edit-mode: replace + + # The gate, last: it reads both reports and fails on a HIGH/CRITICAL + # finding or on a report that never got written, so the report is on the + # PR before the check goes red. + - run: make trivy-gate + + build-artifact: + # One leg per architecture, on a runner of that architecture. Not one check + # with QEMU: the image compiles every plugin with `go build + # -buildmode=plugin`, and emulating an arm64 Go toolchain build turns a + # few minutes into tens of them. Both runner labels are free for public + # repos, so the only cost of the second leg is that it is a second check. + name: Build Artifact (${{ matrix.arch }}) + if: startsWith(github.ref, 'refs/tags/') + runs-on: ${{ matrix.runner }} + strategy: + # A tag that can only ship one architecture should not ship at all, and + # stopping the other leg early keeps a doomed release from occupying a + # runner for another ten minutes. + fail-fast: true + matrix: + include: + - arch: amd64 + runner: ubuntu-latest + - arch: arm64 + runner: ubuntu-24.04-arm + permissions: + contents: read + # The push to ghcr.io. Read is not enough even for a push by digest. + packages: write + steps: + # fetch-depth: 0 — version-vars.sh derives ONIX_VERSION from + # `git describe --tags`, which needs the tag objects, not just the commit. + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # The default builder uses the `docker` driver, which supports neither + # push-by-digest nor --metadata-file. This switches to docker-container. + - uses: docker/setup-buildx-action@v3 + + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Pushes untagged, by digest, and writes digest-.txt. Nothing is + # visible under a version tag until Publish Artifact below has both. + - run: make image-build ARCH=${{ matrix.arch }} + + - uses: actions/upload-artifact@v4 + with: + name: digest-${{ matrix.arch }} + path: digest-${{ matrix.arch }}.txt + if-no-files-found: error + + publish-artifact: + name: Publish Artifact + if: startsWith(github.ref, 'refs/tags/') + # No always() and no if-failure handling: needs alone means a failed or + # cancelled build leg skips this job, which is the point — a version tag + # must never resolve to one architecture. + needs: build-artifact + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + # Checkout is for the Makefile and for the tag objects version-vars.sh + # reads; this job builds nothing. + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/download-artifact@v4 + with: + pattern: digest-* + merge-multiple: true + + # Binds the tag — and :latest, for a plain vX.Y.Z — to the digests the + # build legs pushed, then inspects the result so the log shows which + # platforms the published tag actually resolves to. + - run: make image-publish diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml deleted file mode 100644 index d4b3ea64..00000000 --- a/.github/workflows/security.yml +++ /dev/null @@ -1,194 +0,0 @@ -name: Security - -# Three jobs, so the PR says *which* scan found something without anyone -# opening a log: the dependency scan needs nothing but a checkout, so it runs -# alongside the image build instead of queueing behind it, and the gate reads -# both reports once they land. -# -# Every step is a one-line `make` call or GitHub plumbing — the scan, report -# and gate logic lives in the Makefile, so `make trivy-deps` / `make docker && -# make trivy-image` / `make trivy-gate` reproduce any red check locally. -on: - push: - # development as well as main: this is the branch PRs target and merge - # into, so a merge landing there has to run the same scans the PR did. - branches: [main, development] - pull_request: - -# Least privilege at the workflow level; the scan jobs declare the extra -# scopes they need, so the gate job stays read-only. -permissions: - contents: read - -# Same rule as ci.yml: a new commit on a PR supersedes the scans of the one -# before it. Its own group, not shared with ci.yml, so cancelling a superseded -# CI run does not take the Security run for that commit down with it. -concurrency: - group: security-${{ github.ref }} - # PRs only — see ci.yml. A scan verdict on a trunk commit is a record worth - # keeping. - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - dependency-scan: - name: Dependency scan - runs-on: ubuntu-latest - # Findings go to GitHub's code scanning UI as SARIF (Security tab, diff - # annotations) AND to a PR comment — the SARIF path alone took three clicks - # (check -> run -> results) to see a finding that is one scroll away in the - # PR conversation. pull-requests: write is the comment, security-events: - # write is the SARIF upload. - permissions: - contents: read - pull-requests: write - security-events: write - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/trivy-cache - - # Catches what the image scan structurally cannot: a vulnerable module - # only the test suite imports, so it is never linked into the binary and - # never appears in a layer. - - run: make trivy-deps - - # continue-on-error: a fork PR's GITHUB_TOKEN has no security-events - # write, so this 403s there. The gate job is what decides pass/fail; an - # upload that can't happen must not fail the scan over it. - # - # A category per scan — GitHub keys an analysis on (ref, category), so - # without it the image upload would replace this one. - - name: Upload findings to code scanning - continue-on-error: true - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: trivy-deps.sarif - category: trivy-dependencies - - # Rendered from the SARIF just uploaded, no second scan. Posted before - # the image scan has even finished, so a PR shows a dependency finding - # immediately rather than after the whole workflow completes. - - name: Assemble the PR comment - if: github.event_name == 'pull_request' - continue-on-error: true - run: make trivy-comment SARIF=trivy-deps.sarif TITLE="Go dependency scan" OUT=trivy-deps-comment.md - - - name: Find the existing PR comment - if: github.event_name == 'pull_request' - continue-on-error: true - uses: peter-evans/find-comment@v4 - id: find-comment - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: "github-actions[bot]" - body-includes: "" - - # continue-on-error: a fork PR's default GITHUB_TOKEN is read-only, so - # this 403s there even with pull-requests: write declared. - - name: Comment the report on the PR - if: github.event_name == 'pull_request' - continue-on-error: true - uses: peter-evans/create-or-update-comment@v5 - with: - comment-id: ${{ steps.find-comment.outputs.comment-id }} - issue-number: ${{ github.event.pull_request.number }} - body-path: trivy-deps-comment.md - edit-mode: replace - - # Handed to the gate job, which runs on a different runner. if-no-files-found - # is deliberately `ignore` and not `error`: a scan that produced nothing - # should surface as the gate reporting a MISSING report, with both scans' - # verdicts visible, rather than as a second red check here. - - name: Hand the report to the gate - if: always() - uses: actions/upload-artifact@v4 - with: - name: trivy-deps-sarif - path: trivy-deps.sarif - if-no-files-found: ignore - - image-scan: - name: Image scan - runs-on: ubuntu-latest - # Findings go to GitHub's code scanning UI as SARIF (Security tab, diff - # annotations) AND to a PR comment — the SARIF path alone took three clicks - # (check -> run -> results) to see a finding that is one scroll away in the - # PR conversation. pull-requests: write is the comment, security-events: - # write is the SARIF upload. - permissions: - contents: read - pull-requests: write - security-events: write - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/trivy-cache - - # The image the adapter actually ships — `make docker` builds - # Dockerfile.adapter-with-plugins with the same version build args, so a - # local `make docker && make trivy-image` scans what this gates on. - # - # Trivy reads the base layers plus the Go build info embedded in the - # binary, including `stdlib` — so a Go toolchain CVE shows up here and - # nowhere the dependency scan can see. - - run: make docker IMAGE=network-adapter:${{ github.sha }} - - run: make trivy-image IMAGE=network-adapter:${{ github.sha }} - - - name: Upload findings to code scanning - continue-on-error: true - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: trivy-image.sarif - category: trivy-image - - - name: Assemble the PR comment - if: github.event_name == 'pull_request' - continue-on-error: true - run: make trivy-comment SARIF=trivy-image.sarif TITLE="Image scan" OUT=trivy-image-comment.md - - - name: Find the existing PR comment - if: github.event_name == 'pull_request' - continue-on-error: true - uses: peter-evans/find-comment@v4 - id: find-comment - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: "github-actions[bot]" - body-includes: "" - - - name: Comment the report on the PR - if: github.event_name == 'pull_request' - continue-on-error: true - uses: peter-evans/create-or-update-comment@v5 - with: - comment-id: ${{ steps.find-comment.outputs.comment-id }} - issue-number: ${{ github.event.pull_request.number }} - body-path: trivy-image-comment.md - edit-mode: replace - - - name: Hand the report to the gate - if: always() - uses: actions/upload-artifact@v4 - with: - name: trivy-image-sarif - path: trivy-image.sarif - if-no-files-found: ignore - - security-gate: - name: Security gate - needs: [dependency-scan, image-scan] - # always(): the gate has to run even when a scan job failed, so a failed - # scan is reported as a missing report rather than skipping the gate - # entirely. make trivy-gate fails on a missing or unparsable report for - # exactly this reason. - if: always() - runs-on: ubuntu-latest - steps: - # Checkout is for the Makefile, not the source — this job only reads the - # two SARIF reports the scan jobs produced. - - uses: actions/checkout@v4 - - - uses: actions/download-artifact@v4 - with: - pattern: trivy-*-sarif - merge-multiple: true - - - run: make trivy-gate diff --git a/Makefile b/Makefile index ec5405ba..1f102355 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ # OAN Network Adapter — build, test, lint and security-scan targets. # -# Single source of truth for the ci.yml and security.yml workflows: every CI -# step is a one-line `make ` call, so a red check reproduces locally -# by running the command the log shows. Anything left inline in the workflows -# is GitHub context (`${{ }}` expressions, $GITHUB_ENV / $GITHUB_STEP_SUMMARY -# writes) that has no meaning outside a runner. +# Single source of truth for the ci.yml workflow: every CI step is a one-line +# `make ` call, so a red check reproduces locally by running the +# command the log shows. Anything left inline in the workflow is GitHub context +# (`${{ }}` expressions, $GITHUB_STEP_SUMMARY writes) that has no meaning +# outside a runner. # # No DB, no sqlc/migrate, no separate tools/ module — golangci-lint, gotestsum # and trivy install straight into bin/ via `go install` / curl. @@ -13,6 +13,20 @@ GO ?= go BIN_DIR := bin IMAGE ?= network-adapter:dev +# Where a tag push publishes to. Derived from GITHUB_REPOSITORY rather than +# written out, so a fork publishes to its own namespace and there is no repo +# name to update if this one is ever renamed; `tr` because ghcr.io rejects an +# uppercase path and GITHUB_REPOSITORY is mixed-case (OpenAgriNet/...). +REGISTRY ?= ghcr.io +IMAGE_REPO ?= $(REGISTRY)/$(shell printf '%s' '$(GITHUB_REPOSITORY)' | tr '[:upper:]' '[:lower:]') + +# The arch of the machine running make, so neither a local run nor the build +# matrix has to pass it: each arch is built on a runner of that arch, never +# under QEMU, because Dockerfile.adapter-with-plugins compiles every plugin +# with `go build -buildmode=plugin` and a plugin .so must match the adapter +# binary's GOARCH exactly. +ARCH ?= $(shell uname -m | sed -e 's/^x86_64$$/amd64/' -e 's/^aarch64$$/arm64/') + # CI thresholds/pins live here, not duplicated into workflow env blocks — one # source of truth for both a local `make` run and the GitHub Actions runner. MIN_COVERAGE ?= 80 @@ -112,6 +126,11 @@ coverage.out: # find-comment matches on this exact string to update its comment in place # rather than posting a new one on every run. COVER_MARKER := +SEC_MARKER := + +# Named once and shared by trivy-report and trivy-gate, so the report the PR +# shows and the report the gate reads can never be a different set of files. +SARIF_REPORTS := trivy-deps.sarif trivy-image.sarif ## cover-diff: coverage of the files changed vs BASE_REF, gated on MIN_COVERAGE # A PR review needs the diff's number, not the whole repo's. On failure, names @@ -192,30 +211,42 @@ trivy-image: $(TRIVY) $(TRIVY) image $(IMAGE) --severity $(SEVERITY) --exit-code 0 \ --format sarif --output trivy-image.sarif -## trivy-comment: render SARIF as a markdown PR comment — SARIF, TITLE, OUT +## trivy-report: render both SARIF reports as one PR comment, trivy-report.md +# One comment covering both scans, not one comment each: the two scans run in +# the same job now, and two bot comments per PR was the noise this is meant to +# cut. A missing report is written into the comment as missing rather than +# skipped — trivy-gate fails on it, and the comment has to agree with the gate. +# # The jq program lives in tools/trivy-comment.jq rather than inline: as a file # it is lintable (`jq -n -f`), diffable, and free of Makefile `$$`/backslash -# escaping. One target serves both scans, so the two reports cannot drift. -trivy-comment: - @test -n "$(SARIF)" -a -n "$(TITLE)" -a -n "$(OUT)" || \ - { echo "::error::trivy-comment needs SARIF, TITLE and OUT"; exit 1; } - @test -s "$(SARIF)" || \ - { echo "::error::$(SARIF) missing or empty — no scan produced it"; exit 1; } +# escaping. +trivy-report: @{ \ - echo ""; \ - echo "### 🛡️ Trivy — $(TITLE) ($(SEVERITY))"; \ + echo "$(SEC_MARKER)"; \ + echo "## 🛡️ Trivy security scan ($(SEVERITY))"; \ echo "[View full run]($(RUN_URL))"; \ - echo; \ - jq -r -f tools/trivy-comment.jq "$(SARIF)"; \ - } > $(OUT) + for report in $(SARIF_REPORTS); do \ + case "$$report" in \ + trivy-deps.sarif) title="Go dependencies";; \ + trivy-image.sarif) title="Container image";; \ + *) title="$$report";; \ + esac; \ + echo; echo "### $$title"; echo; \ + if [ -s "$$report" ]; then \ + jq -r -f tools/trivy-comment.jq "$$report"; \ + else \ + echo "⚠️ No report — the scan did not produce $$report."; \ + fi; \ + done; \ + } > trivy-report.md ## trivy-gate: fail if either SARIF report carries a finding, or is missing -# Reads the reports the scan jobs produced rather than scanning a third and -# fourth time. A missing or unparsable report is a failure, not a pass: a scan -# that silently wrote nothing must not turn the gate into a green no-op. +# Reads the reports the two scans already produced rather than scanning a third +# and fourth time. A missing or unparsable report is a failure, not a pass: a +# scan that silently wrote nothing must not turn the gate into a green no-op. trivy-gate: @fail=0; \ - for report in trivy-deps.sarif trivy-image.sarif; do \ + for report in $(SARIF_REPORTS); do \ if [ ! -s "$$report" ]; then \ echo "$$report: MISSING — no scan produced it"; \ fail=1; continue; \ @@ -307,8 +338,8 @@ hooks: ## docker: build the shipped adapter image — the Dockerfile and build args CI scans # Dockerfile.adapter-with-plugins, not Dockerfile.adapter: the plugins image is -# what security.yml scans and what deploys, so a local `make docker && -# make trivy-image` has to scan the same thing CI gates on. The version vars +# what the Security Scan job scans and what a tag publishes, so a local +# `make docker && make trivy-image` scans the same thing CI gates on. The vars # come from the script rather than being named again here, so ONIX_VERSION and # friends are spelled out in exactly one place. docker: @@ -320,11 +351,67 @@ docker: --build-arg BUILD_DATE="$$BUILD_DATE" \ -t $(IMAGE) . +## image-build: push this arch's image to IMAGE_REPO by digest (ARCH, no tag) +# Pushed untagged, by digest only. Nothing binds a version tag to it until +# image-publish has both arches, so a release that builds amd64 and then fails +# on arm64 never leaves behind a tag `docker pull` resolves on one platform and +# 404s on the other. +# +# --provenance=false: with provenance on, buildx wraps even a single-platform +# push in an OCI index to carry the attestation, and `imagetools create` would +# then compose indexes of indexes. A plain manifest per arch is what makes the +# two-platform index image-publish builds a clean one. +# +# Same Dockerfile and same version build args as `docker` above, so what a tag +# publishes is what CI scanned on the PR. +image-build: require-image-repo + . install/scripts/version-vars.sh && \ + docker buildx build -f Dockerfile.adapter-with-plugins \ + --build-arg ONIX_VERSION="$$ONIX_VERSION" \ + --build-arg GIT_COMMIT="$$GIT_COMMIT" \ + --build-arg GIT_TREE_STATE="$$GIT_TREE_STATE" \ + --build-arg BUILD_DATE="$$BUILD_DATE" \ + --platform linux/$(ARCH) \ + --provenance=false \ + --output type=image,name=$(IMAGE_REPO),push-by-digest=true,name-canonical=true,push=true \ + --metadata-file image-metadata-$(ARCH).json . + @jq -r '."containerimage.digest"' image-metadata-$(ARCH).json > digest-$(ARCH).txt + @echo "pushed $(IMAGE_REPO)@$$(cat digest-$(ARCH).txt) (linux/$(ARCH))" + +## image-publish: tag the digests image-build pushed as one multi-arch release +# The only step that creates a user-visible tag. Reads whatever digest-*.txt +# files are present rather than a fixed arch list, so adding an arch to the +# build matrix needs no change here. +# +# The tag comes from version-vars.sh, the same place the binary's -ldflags +# version comes from, so the image tag and `adapter --version` can't disagree. +# `latest` moves only for a plain vX.Y.Z: git describe renders a pre-release as +# v2.0.1-rc1 and an untagged commit as v1.8.2-3-gabc1234, and neither should be +# what `docker pull` gives someone who asked for no tag at all. +image-publish: require-image-repo + @ls digest-*.txt >/dev/null 2>&1 || \ + { echo "::error::no digest-*.txt — run image-build on each arch first"; exit 1; } + . install/scripts/version-vars.sh && \ + tags="-t $(IMAGE_REPO):$$ONIX_VERSION"; \ + case "$$ONIX_VERSION" in \ + *-*) echo "$$ONIX_VERSION is not a plain release — not moving :latest";; \ + *) tags="$$tags -t $(IMAGE_REPO):latest";; \ + esac; \ + docker buildx imagetools create $$tags \ + $$(for d in digest-*.txt; do echo "$(IMAGE_REPO)@$$(cat $$d)"; done); \ + docker buildx imagetools inspect $(IMAGE_REPO):$$ONIX_VERSION + +# Split out so both image targets fail the same way, naming the thing to set, +# instead of pushing to a repo path that is just the registry and a slash. +require-image-repo: + @test "$(IMAGE_REPO)" != "$(REGISTRY)/" || \ + { echo "::error::IMAGE_REPO is empty — set GITHUB_REPOSITORY=owner/repo, or IMAGE_REPO directly"; exit 1; } + ## clean: remove build output and coverage/scan artifacts clean: rm -rf $(BIN_DIR) coverage.out coverage-plugin.out coverage-report.md \ - trivy-deps.sarif trivy-image.sarif \ - trivy-deps-comment.md trivy-image-comment.md + $(SARIF_REPORTS) trivy-report.md \ + image-metadata-*.json digest-*.txt $(GOLANGCI_LINT): @mkdir -p $(BIN_DIR) @@ -352,4 +439,5 @@ $(TRIVY): .PHONY: help build test cover test-ci merge-coverage cover-diff lint fmt \ lint-actions lint-staged hooks \ - trivy-deps trivy-image trivy-comment trivy-gate docker clean + trivy-deps trivy-image trivy-report trivy-gate \ + docker image-build image-publish require-image-repo clean From 3ab7fc79dc33ecac062e2e2b63edc26df9ca0ea5 Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Mon, 7 Sep 2026 23:14:35 +0530 Subject: [PATCH 019/130] ci: move the tag jobs to their own workflow so PRs stop showing them [#5] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gating Build Artifact and Publish Artifact with `if: startsWith(github.ref, 'refs/tags/')` inside ci.yml put two permanently skipped checks on every PR: CI / Build Artifact (${{ matrix.arch }}) (pull_request) Skipped CI / Publish Artifact (pull_request) Skipped A job whose `if:` is false is still reported as a check, and its `name:` comes through with `${{ matrix.arch }}` unexpanded because the matrix leg never materialised — so the skipped-check noise and the raw expression in the name were the same bug. A workflow whose trigger does not match the event does not exist on that event at all, which is the only way to get zero checks. So: ci.yml is PR/trunk only (push to main and development, plus pull_request) and carries no `if:` on either job, and ci-release.yml is tag-only. Both are `name: CI`, so the check names still read "CI / Build Artifact (amd64)" and "CI / Publish Artifact". The one cost is two entries called "CI" in the Actions sidebar. A PR now shows exactly two checks, and a tag exactly three. --- .github/workflows/ci-release.yml | 121 +++++++++++++++++++++++++++++ .github/workflows/ci.yml | 129 ++++--------------------------- Makefile | 10 +-- 3 files changed, 141 insertions(+), 119 deletions(-) create mode 100644 .github/workflows/ci-release.yml diff --git a/.github/workflows/ci-release.yml b/.github/workflows/ci-release.yml new file mode 100644 index 00000000..5c283ea1 --- /dev/null +++ b/.github/workflows/ci-release.yml @@ -0,0 +1,121 @@ +# Named "CI" on purpose, same as ci.yml: the check names GitHub reports are +# " / ", so this keeps them reading +# "CI / Build Artifact (amd64)" and "CI / Publish Artifact". +# +# A separate file rather than two more jobs in ci.yml behind an `if:`. A job +# whose `if:` evaluates false still posts a *skipped* check to the PR, with +# `${{ matrix.* }}` left unexpanded in its name because the matrix leg never +# materialised. A workflow whose trigger does not match simply does not exist +# on that event, so a PR shows nothing from here at all. +# +# The one cost: the Actions sidebar lists two entries called "CI". Rename this +# workflow if that matters more than the check names do. +name: CI + +on: + push: + # Both spellings on purpose. Every tag in this repo so far is v-prefixed + # (v1.8.2, v2.0.1-rc1), but a release cut as 1.0.0 or 1.0.0-RC1 should + # publish rather than silently do nothing. These are GitHub filter + # patterns, not regexes: + and [] work, so the digits are real digits and + # a branch-shaped tag like `release/foo` cannot match. + tags: + - "[0-9]+.[0-9]+.[0-9]+" + - "[0-9]+.[0-9]+.[0-9]+-*" + - "v[0-9]+.[0-9]+.[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+-*" + +# Least privilege at the workflow level; both jobs add packages: write for the +# push to ghcr.io. +permissions: + contents: read + +# Per tag, and deliberately never cancelled: a publish that is half-done is +# worse than one that finishes and is then superseded. +concurrency: + group: ci-release-${{ github.ref }} + cancel-in-progress: false + +jobs: + build-artifact: + # One leg per architecture, on a runner of that architecture. Not one check + # with QEMU: the image compiles every plugin with `go build + # -buildmode=plugin`, and emulating an arm64 Go toolchain build turns a + # few minutes into tens of them. Both runner labels are free for public + # repos, so the only cost of the second leg is that it is a second check. + name: Build Artifact (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + strategy: + # A tag that can only ship one architecture should not ship at all, and + # stopping the other leg early keeps a doomed release from occupying a + # runner for another ten minutes. + fail-fast: true + matrix: + include: + - arch: amd64 + runner: ubuntu-latest + - arch: arm64 + runner: ubuntu-24.04-arm + permissions: + contents: read + # The push to ghcr.io. Read is not enough even for a push by digest. + packages: write + steps: + # fetch-depth: 0 — version-vars.sh derives ONIX_VERSION from + # `git describe --tags`, which needs the tag objects, not just the commit. + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # The default builder uses the `docker` driver, which supports neither + # push-by-digest nor --metadata-file. This switches to docker-container. + - uses: docker/setup-buildx-action@v3 + + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Pushes untagged, by digest, and writes digest-.txt. Nothing is + # visible under a version tag until Publish Artifact below has both. + - run: make image-build ARCH=${{ matrix.arch }} + + - uses: actions/upload-artifact@v4 + with: + name: digest-${{ matrix.arch }} + path: digest-${{ matrix.arch }}.txt + if-no-files-found: error + + publish-artifact: + name: Publish Artifact + # No always() and no if-failure handling: needs alone means a failed or + # cancelled build leg skips this job, which is the point — a version tag + # must never resolve to one architecture. + needs: build-artifact + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + # Checkout is for the Makefile and for the tag objects version-vars.sh + # reads; this job builds nothing. + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/download-artifact@v4 + with: + pattern: digest-* + merge-multiple: true + + # Binds the tag — and :latest, for a plain vX.Y.Z — to the digests the + # build legs pushed, then inspects the result so the log shows which + # platforms the published tag actually resolves to. + - run: make image-publish diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b0cd5e7..1b1504e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,9 +1,14 @@ name: CI -# Four checks and no more. On a PR or a trunk push: "End to End verification" -# and "Security Scan". On a version tag: "Build Artifact" and -# "Publish Artifact". Nothing runs on both event kinds, so a PR never shows a -# publish check and a tag never re-runs the tests the PR already gated on. +# Two checks and no more: "End to End verification" and "Security Scan". +# +# The two tag-time checks live in ci-release.yml, not here behind an `if:`. A +# job whose `if:` is false still posts a skipped check, and its `name:` is +# reported with `${{ matrix.* }}` unexpanded because the matrix leg never +# materialised — so gating tag jobs inside this file put two permanently +# skipped checks on every PR. A separate workflow with a tag-only trigger +# simply does not exist on a PR. That file is also named `CI`, so the check +# names still read "CI / Build Artifact (amd64)" and "CI / Publish Artifact". # # Every step below is either a one-line `make` call or GitHub plumbing # (`${{ }}` expressions, $GITHUB_STEP_SUMMARY). The build, test, scan and @@ -12,22 +17,13 @@ name: CI on: push: # development as well as main: this is the branch PRs target and merge - # into, so a merge landing there runs the same gates the PR did. + # into, so a merge landing there runs the same gates the PR did. No tags — + # a tag re-running the tests its PR already gated on proves nothing. branches: [main, development] - # Both spellings on purpose. Every tag in this repo so far is v-prefixed - # (v1.8.2, v2.0.1-rc1), but a release cut as 1.0.0 or 1.0.0-RC1 should - # publish too rather than silently do nothing. These are GitHub filter - # patterns, not regexes: + and [] work, so the digits are real digits and - # a branch-shaped tag like `release/foo` cannot match. - tags: - - "[0-9]+.[0-9]+.[0-9]+" - - "[0-9]+.[0-9]+.[0-9]+-*" - - "v[0-9]+.[0-9]+.[0-9]+" - - "v[0-9]+.[0-9]+.[0-9]+-*" pull_request: -# Least privilege at the workflow level; the three jobs that need more declare -# it for themselves. +# Least privilege at the workflow level; the jobs that need more declare it for +# themselves. permissions: contents: read @@ -37,22 +33,14 @@ permissions: # group and supersedes the run before it. concurrency: group: ci-${{ github.ref }} - # PRs only. On main, development and tags every commit keeps its own recorded + # PRs only. On main and development every commit keeps its own recorded # verdict — cancelling there would leave a trunk commit with a grey check and - # no way to tell afterwards whether it was ever green, and would abort a - # half-finished publish. + # no way to tell afterwards whether it was ever green. cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: verify: name: End to End verification - # Everything except a tag push, i.e. PRs and trunk pushes. Tested on - # github.ref rather than github.ref_type: on a pull_request the ref is - # refs/pull//merge, which is neither refs/heads/ nor refs/tags/, and - # ref_type is not documented to be 'branch' there. Guessing wrong would - # skip this job on every PR and show a green check for a run that did - # nothing, so the condition names the one case it means to exclude. - if: ${{ !startsWith(github.ref, 'refs/tags/') }} runs-on: ubuntu-latest # pull-requests: write is for the coverage comment below. permissions: @@ -135,8 +123,6 @@ jobs: security-scan: name: Security Scan - # Same condition as End to End verification above — see the note there. - if: ${{ !startsWith(github.ref, 'refs/tags/') }} runs-on: ubuntu-latest # Both scans in one job, one check. They were two jobs plus a gate job, and # three red-or-green checks to say one thing — "is this branch shippable" — @@ -216,88 +202,3 @@ jobs: # finding or on a report that never got written, so the report is on the # PR before the check goes red. - run: make trivy-gate - - build-artifact: - # One leg per architecture, on a runner of that architecture. Not one check - # with QEMU: the image compiles every plugin with `go build - # -buildmode=plugin`, and emulating an arm64 Go toolchain build turns a - # few minutes into tens of them. Both runner labels are free for public - # repos, so the only cost of the second leg is that it is a second check. - name: Build Artifact (${{ matrix.arch }}) - if: startsWith(github.ref, 'refs/tags/') - runs-on: ${{ matrix.runner }} - strategy: - # A tag that can only ship one architecture should not ship at all, and - # stopping the other leg early keeps a doomed release from occupying a - # runner for another ten minutes. - fail-fast: true - matrix: - include: - - arch: amd64 - runner: ubuntu-latest - - arch: arm64 - runner: ubuntu-24.04-arm - permissions: - contents: read - # The push to ghcr.io. Read is not enough even for a push by digest. - packages: write - steps: - # fetch-depth: 0 — version-vars.sh derives ONIX_VERSION from - # `git describe --tags`, which needs the tag objects, not just the commit. - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - # The default builder uses the `docker` driver, which supports neither - # push-by-digest nor --metadata-file. This switches to docker-container. - - uses: docker/setup-buildx-action@v3 - - - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # Pushes untagged, by digest, and writes digest-.txt. Nothing is - # visible under a version tag until Publish Artifact below has both. - - run: make image-build ARCH=${{ matrix.arch }} - - - uses: actions/upload-artifact@v4 - with: - name: digest-${{ matrix.arch }} - path: digest-${{ matrix.arch }}.txt - if-no-files-found: error - - publish-artifact: - name: Publish Artifact - if: startsWith(github.ref, 'refs/tags/') - # No always() and no if-failure handling: needs alone means a failed or - # cancelled build leg skips this job, which is the point — a version tag - # must never resolve to one architecture. - needs: build-artifact - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - # Checkout is for the Makefile and for the tag objects version-vars.sh - # reads; this job builds nothing. - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - uses: actions/download-artifact@v4 - with: - pattern: digest-* - merge-multiple: true - - # Binds the tag — and :latest, for a plain vX.Y.Z — to the digests the - # build legs pushed, then inspects the result so the log shows which - # platforms the published tag actually resolves to. - - run: make image-publish diff --git a/Makefile b/Makefile index 1f102355..a023cdfa 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ # OAN Network Adapter — build, test, lint and security-scan targets. # -# Single source of truth for the ci.yml workflow: every CI step is a one-line -# `make ` call, so a red check reproduces locally by running the -# command the log shows. Anything left inline in the workflow is GitHub context -# (`${{ }}` expressions, $GITHUB_STEP_SUMMARY writes) that has no meaning -# outside a runner. +# Single source of truth for the ci.yml and ci-release.yml workflows: every CI +# step is a one-line `make ` call, so a red check reproduces locally by +# running the command the log shows. Anything left inline in a workflow is +# GitHub context (`${{ }}` expressions, $GITHUB_STEP_SUMMARY writes) that has +# no meaning outside a runner. # # No DB, no sqlc/migrate, no separate tools/ module — golangci-lint, gotestsum # and trivy install straight into bin/ via `go install` / curl. From 5a0f1bd2e499e00efa65f7b1c0bf011ca9b76dd3 Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Mon, 7 Sep 2026 23:25:27 +0530 Subject: [PATCH 020/130] =?UTF-8?q?ci:=20drop=20the=20SARIF=20upload=20?= =?UTF-8?q?=E2=80=94=20it=20was=20a=20third=20check,=20and=20it=20contradi?= =?UTF-8?q?cted=20the=20gate=20[#5]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A PR reported three checks, not two: CI / End to End verification success CI / Security Scan failure Code scanning results / Trivy success — "No new alerts in code change" The third is not a job. github/codeql-action/upload-sarif registers its own check named after the SARIF tool driver, so it cannot be renamed, given an `if:`, or suppressed while the upload happens. And on that run it was green while `make trivy-gate` failed on the same SARIF: 20 HIGH stdlib CVEs, none of them *new to the diff*, which is the only thing the code-scanning check measures. Two checks disagreeing about one scan is worse than one check, and the gate is the one that is right. Every finding is already in the PR comment with its severity, installed version, fixed version and advisory link, and `make trivy-gate` decides pass/fail. `security-events: write` goes with the upload. The cost, stated: no Security-tab history or diff annotations for Trivy findings. The comment and the gate are the surface now. --- .github/workflows/ci.yml | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b1504e3..c5fd9dbf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -130,12 +130,11 @@ jobs: # checkout, so it runs first and its findings are in the log before the # image finishes building. # - # pull-requests: write is the PR comment, security-events: write is the - # SARIF upload to the Security tab. + # pull-requests: write is the PR comment. No security-events: write — the + # SARIF upload is gone, see below. permissions: contents: read pull-requests: write - security-events: write steps: - uses: actions/checkout@v4 - uses: ./.github/actions/trivy-cache @@ -151,25 +150,19 @@ jobs: - run: make docker IMAGE=network-adapter:${{ github.sha }} - run: make trivy-image IMAGE=network-adapter:${{ github.sha }} - # continue-on-error: a fork PR's GITHUB_TOKEN has no security-events - # write, so these 403 there. make trivy-gate below is what decides - # pass/fail; an upload that can't happen must not fail the job over it. + # No upload-sarif here, on purpose. github/codeql-action/upload-sarif + # posts its own check, "Code scanning results / Trivy", which is not a + # job and cannot be renamed or suppressed — so it was a third check on + # every PR. And it reported the opposite of the truth: on the run that + # found 20 HIGH stdlib CVEs it went green with "No new alerts in code + # change", because none of them were new to the diff, while + # make trivy-gate failed on the same SARIF. Two checks disagreeing about + # one scan is worse than one check, and the gate is the one that is + # right. Every finding is in the PR comment below with its severity and + # fixed version. # - # A category per scan — GitHub keys an analysis on (ref, category), so - # without them the second upload would replace the first. - - name: Upload dependency findings to code scanning - continue-on-error: true - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: trivy-deps.sarif - category: trivy-dependencies - - - name: Upload image findings to code scanning - continue-on-error: true - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: trivy-image.sarif - category: trivy-image + # The cost: no Security-tab history or diff annotations for Trivy. The + # comment and the gate are the surface. # One comment covering both scans, rendered from the SARIF already # written — no third and fourth scan. From eec90fdc9d3ba9f6846f811a8fa5e158e254da8c Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Mon, 7 Sep 2026 23:25:27 +0530 Subject: [PATCH 021/130] fix: bump the Go toolchain to 1.26.8 to clear 20 HIGH stdlib CVEs [#5] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The image scan the Security Scan job runs found 20 HIGH findings, all of them the same package: `stdlib` at v1.26.1. Zero came from application dependencies — `trivy-deps` is clean. Trivy reads the Go build info embedded in the binary, which is why these show up in the image scan and nowhere the dependency scan can see. The advisories' fixed versions top out at 1.26.6; 1.26.8 is the current 1.26 patch, so it clears all 20 with room before the next one lands. Bumped in three places that have to agree: go.mod's `go` directive (which is also what setup-go resolves through `go-version-file`, so the test toolchain follows) and both Dockerfiles' builder base. Leaving Dockerfile.adapter at 1.26.1 would have kept the two images on different toolchains for no reason. This re-touches the Dockerfiles, which this PR had otherwise reverted to byte-identical with development as out of scope. Different case: a one-token patch bump to the base image tag, made because the Security gate this PR introduces cannot go green without it — not the base-image migration that was reverted. Verified: `make build` and the full `make test` (every package, including pkg/plugin and benchmarks/e2e, which build and plugin.Open a real .so) pass on 1.26.8 with zero failures. --- Dockerfile.adapter | 2 +- Dockerfile.adapter-with-plugins | 2 +- go.mod | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile.adapter b/Dockerfile.adapter index 41dd7ca8..e7222efe 100644 --- a/Dockerfile.adapter +++ b/Dockerfile.adapter @@ -1,4 +1,4 @@ -FROM golang:1.26.1-bookworm AS builder +FROM golang:1.26.8-bookworm AS builder WORKDIR /workspace/app COPY cmd/adapter ./cmd/adapter diff --git a/Dockerfile.adapter-with-plugins b/Dockerfile.adapter-with-plugins index a7ba5e98..e333faea 100644 --- a/Dockerfile.adapter-with-plugins +++ b/Dockerfile.adapter-with-plugins @@ -1,4 +1,4 @@ -FROM golang:1.26.1-bookworm AS builder +FROM golang:1.26.8-bookworm AS builder WORKDIR /workspace/app diff --git a/go.mod b/go.mod index 10145dc5..5715fa56 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/beckn-one/beckn-onix -go 1.26.1 +go 1.26.8 require ( github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 From 84cb6b5fc200ca8b5462d67b13df0caa42b8a846 Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Mon, 7 Sep 2026 23:45:45 +0530 Subject: [PATCH 022/130] chore: widen the Trivy gate to CRITICAL,HIGH,MEDIUM,LOW [#5] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gate scanned at HIGH,CRITICAL only, so "0 findings" in the check meant "0 in that band" rather than "clean". Unfiltered, the tree has three findings in golang.org/x/crypto v0.55.0 that the gate never saw. UNKNOWN stays out of the band deliberately. All three of those findings carry no vendor severity, and one (GO-2026-5932, x/crypto/openpgp is unmaintained) has no fix at all, so including UNKNOWN would pin the gate red with no action available. None is reachable here: the only x/crypto import in the tree is blake2b, and the build closure of the adapter plus every plugin contains no x/crypto/ssh or x/crypto/openpgp. SEVERITY stays the single source of truth. The gate already counted rows in the SARIF rather than re-filtering, so widening the scan band was enough; the three places that restated "HIGH or CRITICAL" in prose now derive it, including the jq renderer, which takes the band via --arg so the "nothing found" line cannot drift from the scan that produced it. Verified with trivy v0.74.0, the version CI pins: deps 0 at the new band (the 3 UNKNOWN correctly excluded), wolfi-base runtime layer 0, shipped binary 0. The band change has teeth — the pre-bump go1.26.1 binary goes from 20 to 28 findings, the 8 extra all MEDIUM. Gate exit codes: 2 on findings, 2 on a missing report, 0 only when clean. --- .github/workflows/ci.yml | 2 +- Makefile | 6 +++--- tools/trivy-comment.jq | 10 +++++++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5fd9dbf..5aee8d29 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -191,7 +191,7 @@ jobs: body-path: trivy-report.md edit-mode: replace - # The gate, last: it reads both reports and fails on a HIGH/CRITICAL + # The gate, last: it reads both reports and fails on any in-band # finding or on a report that never got written, so the report is on the # PR before the check goes red. - run: make trivy-gate diff --git a/Makefile b/Makefile index a023cdfa..5a2104a7 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ MIN_COVERAGE ?= 80 # development, not main: every branch in this repo is cut from development and # PRs target it, so that is the base a local `make cover-diff` must compare to. BASE_REF ?= origin/development -SEVERITY ?= HIGH,CRITICAL +SEVERITY ?= CRITICAL,HIGH,MEDIUM,LOW GOLANGCI_LINT_VERSION := v2.5.0 GOTESTSUM_VERSION := v1.13.0 TRIVY_VERSION := v0.74.0 @@ -233,7 +233,7 @@ trivy-report: esac; \ echo; echo "### $$title"; echo; \ if [ -s "$$report" ]; then \ - jq -r -f tools/trivy-comment.jq "$$report"; \ + jq -r --arg severity "$(SEVERITY)" -f tools/trivy-comment.jq "$$report"; \ else \ echo "⚠️ No report — the scan did not produce $$report."; \ fi; \ @@ -262,7 +262,7 @@ trivy-gate: fail=1; \ fi; \ done; \ - [ "$$fail" -eq 0 ] || echo "::error::HIGH or CRITICAL Trivy findings, or a missing report — see the log above"; \ + [ "$$fail" -eq 0 ] || echo "::error::Trivy findings at $(SEVERITY), or a missing report — see the log above"; \ exit $$fail ## lint: vet, format check and static analysis diff --git a/tools/trivy-comment.jq b/tools/trivy-comment.jq index 54371ac1..b56ba9fd 100644 --- a/tools/trivy-comment.jq +++ b/tools/trivy-comment.jq @@ -1,14 +1,18 @@ # Render a Trivy SARIF report as a markdown table for a PR comment. # -# Used by the Makefile's trivy-comment target for both the dependency and the +# Used by the Makefile's trivy-report target for both the dependency and the # image scan — one program, not a copy per scan. # +# $severity is the band the scan was run at (the Makefile's SEVERITY), passed +# in with --arg so the "nothing found" line names the band it actually checked +# rather than hardcoding one that can drift from the scan. +# # Trivy's SARIF carries no structured per-field severity/version columns: each # finding's detail lives as prose in message.text ("Package: ...\nSeverity: # ...\n..."), so the columns below are pulled out of that text rather than read # from dedicated JSON fields. # -# Lint with: jq -n -f tools/trivy-comment.jq +# Lint with: jq -n --arg severity "" -f tools/trivy-comment.jq # capture returns null when the pattern doesn't match, so `// {v: default}` # supplies the fallback rather than letting a missing field print "null". @@ -16,7 +20,7 @@ def val(re; default): (capture(re) // {v: default}).v; [ .runs[]?.results[]? ] as $found | if ($found | length) == 0 then - "No HIGH or CRITICAL findings." + "No findings at " + $severity + "." else "| Package | Severity | Installed | Fixed in | Advisory |", "|---|---|---|---|---|", From 138ffbe6d451a8037471243c485c7aee170d9800 Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Tue, 8 Sep 2026 00:05:44 +0530 Subject: [PATCH 023/130] fix: fail image-publish on a failed create, and pin the trivy installer [#5] image-publish joined its steps with `;`, so the recipe exit status came from `imagetools inspect` alone. A `create` that failed against an already-existing tag left inspect reporting the previous index and the job green, publishing a tag bound to the wrong digests with nothing red to say so. Every step is now `&&`-chained. The trivy installer was fetched from the repo default branch while only the version it installs was pinned, so a job holding the runner GITHUB_TOKEN piped an unreviewed remote script into sh on every cache miss. It now comes from the $(TRIVY_VERSION) tag. .gitignore also drifted from the target set: trivy-report.md, image-metadata-*.json and digest-*.txt are produced but were not ignored, while trivy-deps-comment.md and trivy-image-comment.md outlived the trivy-comment target that wrote them. --- .gitignore | 10 ++++++++-- Makefile | 19 +++++++++++++++---- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 4a9de6ae..6292bb58 100644 --- a/.gitignore +++ b/.gitignore @@ -148,10 +148,16 @@ coverage-report.md # Makefile tool/build artifacts (golangci-lint, gotestsum, trivy binaries) /bin/ + +# Scan output: the two SARIF reports trivy-deps/trivy-image write and the +# single comment trivy-report renders from them. trivy-deps.sarif trivy-image.sarif -trivy-deps-comment.md -trivy-image-comment.md +trivy-report.md + +# Release output: image-build writes one of each per architecture. +image-metadata-*.json +digest-*.txt # Ignore the schema directory used for testing /schemas/ diff --git a/Makefile b/Makefile index 5a2104a7..98ceb306 100644 --- a/Makefile +++ b/Makefile @@ -383,6 +383,12 @@ image-build: require-image-repo # files are present rather than a fixed arch list, so adding an arch to the # build matrix needs no change here. # +# `&&` between every step, not `;`: a recipe is one shell invocation with no +# `set -e`, so with `;` the exit status would be `imagetools inspect`'s alone. +# A failed `create` on a tag that already exists would then leave inspect +# reporting the *previous* index and this job green — the published tag would +# point at the wrong digests with nothing red to say so. +# # The tag comes from version-vars.sh, the same place the binary's -ldflags # version comes from, so the image tag and `adapter --version` can't disagree. # `latest` moves only for a plain vX.Y.Z: git describe renders a pre-release as @@ -392,13 +398,13 @@ image-publish: require-image-repo @ls digest-*.txt >/dev/null 2>&1 || \ { echo "::error::no digest-*.txt — run image-build on each arch first"; exit 1; } . install/scripts/version-vars.sh && \ - tags="-t $(IMAGE_REPO):$$ONIX_VERSION"; \ + tags="-t $(IMAGE_REPO):$$ONIX_VERSION" && \ case "$$ONIX_VERSION" in \ *-*) echo "$$ONIX_VERSION is not a plain release — not moving :latest";; \ *) tags="$$tags -t $(IMAGE_REPO):latest";; \ - esac; \ + esac && \ docker buildx imagetools create $$tags \ - $$(for d in digest-*.txt; do echo "$(IMAGE_REPO)@$$(cat $$d)"; done); \ + $$(for d in digest-*.txt; do echo "$(IMAGE_REPO)@$$(cat $$d)"; done) && \ docker buildx imagetools inspect $(IMAGE_REPO):$$ONIX_VERSION # Split out so both image targets fail the same way, naming the thing to set, @@ -432,9 +438,14 @@ $(ACTIONLINT): # cgo, and its module graph is comparable in size to golangci-lint's for a # tool nothing here imports — the official install script is what # aquasecurity itself recommends over building from source for exactly this. +# +# The script is fetched at $(TRIVY_VERSION), not at main: this pipes a remote +# script into sh in a job that holds the runner's GITHUB_TOKEN, so what runs +# has to be the reviewed script for the pinned release rather than whatever is +# on the default branch at the time. Every other tool here is pinned too. $(TRIVY): @mkdir -p $(BIN_DIR) - curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | \ + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/$(TRIVY_VERSION)/contrib/install.sh | \ sh -s -- -b $(abspath $(BIN_DIR)) $(TRIVY_VERSION) .PHONY: help build test cover test-ci merge-coverage cover-diff lint fmt \ From 4f87932fef93c7c9594ba204da9cc42cebff4c26 Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Tue, 8 Sep 2026 00:28:37 +0530 Subject: [PATCH 024/130] fix: scan the release image before its version tag is bound [#5] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit image-build pushes by digest and image-publish binds the tag; nothing in between looked at what was being published. The PR-time Security Scan does not cover it — it grades an image built from the PR's tree on the day the PR ran, while a tag cut weeks later is a fresh build off a rolling wolfi-base and a freshly resolved module graph. trivy-release-gate was removed earlier in this branch as dead code, correctly at the time: its only caller was build-and-push.yml. Adding ci-release.yml put a release path back without putting the gate back with it. Reinstated against the pushed digest rather than a local tag, since push-by-digest leaves no local reference. The digest is unreachable by name until image-publish binds a tag, and the gate runs before that, so a finding means the version tag is never created. --exit-code 1 with a table: there is no PR to comment on, so the findings belong in the log the red check points at. Scanned per arch on its own native runner. Also adds timeout-minutes to all four jobs. The default is 360, and the new gate pulls from a registry — a stalled pull would otherwise hold a runner for six hours, and on a release leg it blocks Publish Artifact behind `needs`, so the tag simply never appears with no failure to point at. --- .github/workflows/ci-release.yml | 21 +++++++++++++++++++++ .github/workflows/ci.yml | 9 +++++++++ Makefile | 23 ++++++++++++++++++++++- 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-release.yml b/.github/workflows/ci-release.yml index 5c283ea1..e8222788 100644 --- a/.github/workflows/ci-release.yml +++ b/.github/workflows/ci-release.yml @@ -45,6 +45,11 @@ jobs: # repos, so the only cost of the second leg is that it is a second check. name: Build Artifact (${{ matrix.arch }}) runs-on: ${{ matrix.runner }} + # Runs in ~2 min per arch. Both legs build every plugin and now pull the + # pushed digest back for the scan, so a stalled registry is the realistic + # hang here — and a release leg that hangs also blocks Publish Artifact + # behind `needs`, so the tag just never appears with no failure to point at. + timeout-minutes: 45 strategy: # A tag that can only ship one architecture should not ship at all, and # stopping the other leg early keeps a doomed release from occupying a @@ -67,6 +72,8 @@ jobs: with: fetch-depth: 0 + - uses: ./.github/actions/trivy-cache + # The default builder uses the `docker` driver, which supports neither # push-by-digest nor --metadata-file. This switches to docker-container. - uses: docker/setup-buildx-action@v3 @@ -81,6 +88,18 @@ jobs: # visible under a version tag until Publish Artifact below has both. - run: make image-build ARCH=${{ matrix.arch }} + # The release image gets its own scan; the PR check does not cover it. + # Security Scan graded an image built from the PR's tree on the day the + # PR ran, and this is a fresh build off a rolling base image weeks later. + # Scanned per arch on its own native runner, so the arm64 artifact is + # scanned as arm64 rather than assumed to match amd64. + # + # Placed before upload-artifact and therefore before Publish Artifact, + # which needs both legs: a finding on either arch means no digest is + # handed on and no version tag is ever bound. fail-fast: true cancels + # the sibling leg rather than letting it finish a build nothing will use. + - run: make trivy-release-gate ARCH=${{ matrix.arch }} + - uses: actions/upload-artifact@v4 with: name: digest-${{ matrix.arch }} @@ -94,6 +113,8 @@ jobs: # must never resolve to one architecture. needs: build-artifact runs-on: ubuntu-latest + # Runs in ~20s: two registry API calls and no build. + timeout-minutes: 10 permissions: contents: read packages: write diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5aee8d29..14d32612 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,12 @@ jobs: verify: name: End to End verification runs-on: ubuntu-latest + # Runs in ~1 min. The default is 360, so a hung `go test` or a wedged + # module download would hold a runner for six hours and burn the whole + # concurrency allowance before anyone noticed. Generous multiples of the + # real duration, not tight budgets — this is a hang detector, not a + # performance gate that goes red when the suite legitimately grows. + timeout-minutes: 15 # pull-requests: write is for the coverage comment below. permissions: contents: read @@ -124,6 +130,9 @@ jobs: security-scan: name: Security Scan runs-on: ubuntu-latest + # Runs in ~3 min. Higher than verify: this one builds the image and pulls + # trivy's DB, so it has two network dependencies that can stall. + timeout-minutes: 30 # Both scans in one job, one check. They were two jobs plus a gate job, and # three red-or-green checks to say one thing — "is this branch shippable" — # was more to read than it was worth. The dependency scan needs only a diff --git a/Makefile b/Makefile index 98ceb306..8c1deffc 100644 --- a/Makefile +++ b/Makefile @@ -211,6 +211,27 @@ trivy-image: $(TRIVY) $(TRIVY) image $(IMAGE) --severity $(SEVERITY) --exit-code 0 \ --format sarif --output trivy-image.sarif +## trivy-release-gate: fail the release if the digest just pushed has a finding +# The PR-time Security Scan is not this gate. It scans an image built from the +# PR's tree on the day the PR ran; a tag cut weeks later rebuilds from a freshly +# pulled `wolfi-base` and a freshly resolved module graph, so the artifact that +# ships is not the artifact anything looked at. Without this, image-build +# publishes a digest no scan has ever seen. +# +# Scans the digest, not a local tag: image-build pushes by digest, so the only +# reference to the layers it just built is the one in digest-$(ARCH).txt. That +# digest is unreachable by name until image-publish binds a tag to it, and this +# runs first — so a finding here means the version tag is never created. +# +# --exit-code 1 and a table, not SARIF at --exit-code 0: there is no PR to +# comment on, so the findings belong in the log the red check points at, and +# the scan itself is the gate rather than a report something else grades. +trivy-release-gate: $(TRIVY) require-image-repo + @test -s digest-$(ARCH).txt || \ + { echo "::error::digest-$(ARCH).txt is missing or empty — run image-build first"; exit 1; } + $(TRIVY) image $(IMAGE_REPO)@$$(cat digest-$(ARCH).txt) \ + --severity $(SEVERITY) --exit-code 1 --format table + ## trivy-report: render both SARIF reports as one PR comment, trivy-report.md # One comment covering both scans, not one comment each: the two scans run in # the same job now, and two bot comments per PR was the noise this is meant to @@ -450,5 +471,5 @@ $(TRIVY): .PHONY: help build test cover test-ci merge-coverage cover-diff lint fmt \ lint-actions lint-staged hooks \ - trivy-deps trivy-image trivy-report trivy-gate \ + trivy-deps trivy-image trivy-report trivy-gate trivy-release-gate \ docker image-build image-publish require-image-repo clean From 3b0ba69f44d3b4710f3e8845b140af2e810d5507 Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Tue, 8 Sep 2026 00:38:18 +0530 Subject: [PATCH 025/130] fix: report not-applicable coverage as not-applicable, not as a pass [#5] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cover-diff has two paths that compute no percentage — no changed non-test .go files, so there is no denominator, and changed files carrying no coverable statements, which would be 0/0 — and both reported "Test Coverage: ✅ Passed — not applicable". Read from the PR that is a green pass, so a CI-only or docs-only PR looked indistinguishable from a well-tested one, and there was no number to reconcile the claim against. Both now report "➖ Not applicable" and say which case applies and why there is no percentage. Still exit 0 — a PR that changes no Go code must not be blocked on Go coverage — so the check verdict is unchanged; only the claim the comment makes about it is. Also corrects "no changed Go files" to "no non-test Go files changed": the changed-file set already filters _test.go, so a test-only PR took this path while the message said no Go files had changed at all. --- Makefile | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8c1deffc..4c7a629c 100644 --- a/Makefile +++ b/Makefile @@ -137,6 +137,14 @@ SARIF_REPORTS := trivy-deps.sarif trivy-image.sarif # the changed files dragging it down, worst first. Always writes # coverage-report.md — the workflow reads that file unconditionally, so every # exit path here has to produce it. +# Three verdicts, not two. "✅ Passed" and "❌ Failed" both mean a percentage was +# computed and compared against MIN_COVERAGE. The two not-applicable paths below +# compute nothing — no changed non-test .go files means no denominator, and +# changed files with no coverable statements means 0/0 — so they report +# "➖ Not applicable" and say why. They used to say "✅ Passed", which read as +# though the diff had been measured and cleared the bar; a CI-only PR looked +# indistinguishable from a well-tested one. Still exit 0: a PR that changes no +# Go code must not be blocked on Go coverage. cover-diff: coverage.out @if ! git rev-parse --verify --quiet "$(BASE_REF)^{commit}" >/dev/null; then \ echo "::error::BASE_REF '$(BASE_REF)' does not resolve to a commit — cannot compute the changed-file set"; \ @@ -150,7 +158,7 @@ cover-diff: coverage.out fi; \ CHANGED=$$(printf '%s\n' "$$DIFF" | grep -v '_test\.go$$'); \ if [ -z "$$CHANGED" ]; then \ - printf '%s\n' "$(COVER_MARKER)" "📊 **Test Coverage: ✅ Passed** — not applicable, no changed Go files vs $(BASE_REF)" | tee coverage-report.md; \ + printf '%s\n' "$(COVER_MARKER)" "📊 **Test Coverage: ➖ Not applicable** — no non-test Go files changed vs \`$(BASE_REF)\`, so there are no lines to measure and no percentage to report." | tee coverage-report.md; \ exit 0; \ fi; \ MODULE=$$($(GO) list -m); \ @@ -170,7 +178,7 @@ cover-diff: coverage.out print "TOTAL\t" int(C * 100 / T) \ }' - coverage.out); \ if echo "$$RESULT" | grep -q '^EMPTY$$'; then \ - printf '%s\n' "$(COVER_MARKER)" "📊 **Test Coverage: ✅ Passed** — not applicable, changed files carry no coverable statements" | tee coverage-report.md; \ + printf '%s\n' "$(COVER_MARKER)" "📊 **Test Coverage: ➖ Not applicable** — the changed Go files carry no coverable statements (declarations, types or constants only), so there is no percentage to compute." | tee coverage-report.md; \ exit 0; \ fi; \ PCT=$$(echo "$$RESULT" | awk -F'\t' '$$1=="TOTAL"{print $$2}'); \ From a7258fbd1d771b8083d25a7151e882844a691d8d Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 31 Aug 2026 11:47:21 +0530 Subject: [PATCH 026/130] feat: add the OAN registry plugin [OpenAgriNet/engineering-tracker#46] Resolves two different things from the OAN Registry, a SunbirdRC deployment, and keeps them apart because they answer different questions about different parties. RegistryLookup answers "who sent this": given the subscriber and key named in an inbound Authorization header, it returns that sender's signing key so the signature can be verified. This runs inside signature validation on every inbound message, so its timeout and retry budget are deliberately tighter than the sibling registry plugins' -- timeout x (retry_max + 1) is time a request spends waiting before it can even be rejected. ProviderRecordLookup answers "who do I call next" [#63]: given a capability binding taken from a request body, it reads the binding and the participant that owns it, and joins them into one call plan -- where the provider is, and per Beckn action, how to reach it. Every way of saying "this capability cannot be served" returns one sentinel, because a caller does the same thing with all of them; a registry that could not be CONSULTED returns its own error, since an outage is not an answer. Several decisions here were forced by the deployed registry rather than chosen: - records are read from the nested shape the registry actually serves, with keys under node.keys[] rather than flat on the record - a key is matched by its osid, which is what an Authorization header carries; the friendly keyId identifies nothing the registry indexes - the "base64:" label is stripped from key material, because model.Subscription carries the bare value signvalidator feeds straight to base64.StdEncoding.DecodeString - status is checked at both levels, since a participant stays active while one of its keys is retired - actions are read as an array: the registry treats every nested object as an entity and injects osid into it, which a map cannot carry Status is an allow-list throughout, not a deny-list. model.IsKeyStatusUsable treats anything it does not recognise as usable, so passing the registry's own vocabulary through unchanged would let a suspended participant's signature verify. Verified against a live registry and the recorded response it serves. --- install/build-plugins.sh | 1 + pkg/model/model.go | 42 + pkg/plugin/definition/registry.go | 33 + .../implementation/oanregistry/README.md | 242 +++ .../implementation/oanregistry/cmd/plugin.go | 151 ++ .../oanregistry/cmd/plugin_test.go | 249 +++ .../implementation/oanregistry/oanregistry.go | 652 +++++++ .../oanregistry/oanregistry_test.go | 1577 +++++++++++++++++ .../oanregistry/providerrecord.go | 419 +++++ .../oanregistry/providerrecord_test.go | 590 ++++++ 10 files changed, 3956 insertions(+) create mode 100644 pkg/plugin/implementation/oanregistry/README.md create mode 100644 pkg/plugin/implementation/oanregistry/cmd/plugin.go create mode 100644 pkg/plugin/implementation/oanregistry/cmd/plugin_test.go create mode 100644 pkg/plugin/implementation/oanregistry/oanregistry.go create mode 100644 pkg/plugin/implementation/oanregistry/oanregistry_test.go create mode 100644 pkg/plugin/implementation/oanregistry/providerrecord.go create mode 100644 pkg/plugin/implementation/oanregistry/providerrecord_test.go diff --git a/install/build-plugins.sh b/install/build-plugins.sh index 36e42ec7..d205616d 100755 --- a/install/build-plugins.sh +++ b/install/build-plugins.sh @@ -30,6 +30,7 @@ plugins=( "publisher" "registry" "dediregistry" + "oanregistry" "manifestloader" "reqpreprocessor" "otelsetup" diff --git a/pkg/model/model.go b/pkg/model/model.go index 7e1b0255..7dd4825b 100644 --- a/pkg/model/model.go +++ b/pkg/model/model.go @@ -71,6 +71,47 @@ type SubscriberRecord struct { MetaArrays map[string][]string // array-shaped meta values (e.g. NFH-014's meta.catalog_index_urls: [{url}, ...]) — kept separate from Meta rather than widening it to map[string]any, so every existing caller of Meta[key] keeps working unchanged } +// ProviderRecord is the resolved call plan for one provider capability: what to +// call, how to call it, and which mappings translate in and out. It is assembled +// from two registry records -- the capability binding and the participant that +// owns it -- so a caller resolves a whole plan in one lookup rather than knowing +// how the registry splits them. +// +// Mapping references are carried verbatim. They are fully-qualified URLs the +// mapper fetches; this type does not interpret them. +type ProviderRecord struct { + BindingKey string // "|" + ParticipantID string + CapabilityCode string + + // BaseURL comes from the participant and is shared by every action: one + // provider, one host. + BaseURL string + + // Actions is the call plan per Beckn action. A capability serves several -- + // a select that reads and a confirm that commits -- and they rarely share an + // endpoint or a method, so each carries its own. + // + // An action absent here is one this capability does not serve. + Actions map[string]ActionPlan + + RequestMapping string + ResponseMapping string +} + +// ActionPlan is how to make one action's upstream call. +type ActionPlan struct { + Method string + Path string + + // TimeoutMs and RetryMax are this action's own budget, and are zero when the + // registry does not set them -- the caller applies its defaults. They are + // per action because a confirm that commits deserves a different budget from + // a select that reads. + TimeoutMs int + RetryMax int +} + // Authorization-related constants for headers. const ( AuthHeaderSubscriber string = "Authorization" @@ -326,6 +367,7 @@ type StepContext struct { MessageID string // Message ID parsed from context.messageId in the request body InboundAuthSignature string // Raw Base64 signature from the inbound Authorization header's signature="..." attribute IsCallerHandler bool // True when the handler is a Caller (outbound); false for Receiver (inbound) + } // WithContext updates the existing StepContext with a new context. diff --git a/pkg/plugin/definition/registry.go b/pkg/plugin/definition/registry.go index 20d7bb98..57c3d69c 100644 --- a/pkg/plugin/definition/registry.go +++ b/pkg/plugin/definition/registry.go @@ -2,6 +2,7 @@ package definition import ( "context" + "errors" "github.com/beckn-one/beckn-onix/pkg/model" ) @@ -47,3 +48,35 @@ type RegistryMetadataLookup interface { type RegistryLookupProvider interface { New(context.Context, Cache, map[string]string) (RegistryLookup, func() error, error) } + +// ErrProviderRecordNotFound reports that no usable call plan exists for a +// binding key. It is returned for an absent binding, an absent participant, and +// for either of them being inactive -- all of which mean the same thing to a +// caller: this capability cannot be served right now. The distinction between +// them is observable in the plugin's own logs and metrics, and is not something +// a caller can act on differently. +var ErrProviderRecordNotFound = errors.New("provider record not found") + +// ProviderRecordLookup resolves a provider capability into a call plan. +// +// It is separate from RegistryLookup because it answers a different question +// about a different party. RegistryLookup answers "what is the SENDER's public +// key", keyed by the identity in an inbound Authorization header. +// ProviderRecordLookup answers "how do I call the UPSTREAM provider", keyed by +// a capability binding taken from the request body. The two have different +// subjects, different cache lifetimes, and different failure meanings, so they +// are not folded together. +// +// A registry plugin may implement one, the other, or both. Callers obtain this +// by type-asserting a RegistryLookup, the same way RegistryMetadataLookup is +// obtained -- a plugin that does not implement it yields nil, and the consumer +// decides whether that is fatal. +type ProviderRecordLookup interface { + // ProviderRecord resolves bindingKey ("|") + // into everything needed to call the provider. + // + // Returns ErrProviderRecordNotFound when no usable plan exists. Any other + // error is a transport or decoding failure -- the registry could not be + // consulted, which is not the same as it answering "no". + ProviderRecord(ctx context.Context, bindingKey string) (*model.ProviderRecord, error) +} diff --git a/pkg/plugin/implementation/oanregistry/README.md b/pkg/plugin/implementation/oanregistry/README.md new file mode 100644 index 00000000..099a7b84 --- /dev/null +++ b/pkg/plugin/implementation/oanregistry/README.md @@ -0,0 +1,242 @@ +# OAN Registry Plugin + +A **registry type plugin** for Beckn-ONIX that reads the OAN Registry, a +[SunbirdRC](https://docs.sunbirdrc.dev/) deployment. + +## Overview + +It answers two questions, and they are deliberately kept apart. + +**Who sent this?** `definition.RegistryLookup` — given the `subscriber_id` and +`key_id` carried in an inbound request's `Authorization` header, it returns that +**sender's** public key so the signature can be verified. + +**Who do I call next?** `definition.ProviderRecordLookup` — given a capability +binding taken from the request body, it returns the **upstream provider's** call +plan: where to call, how, and which mappings translate in and out. + +Different subject, different cache, different meaning of failure. They share +transport and nothing else. A caller reaches the second by type-asserting the +first, the same way `RegistryMetadataLookup` is reached elsewhere. + +It is read-only. Onboarding, key publication and status changes all happen +through the registry's own Participant APIs, not through this plugin. + +This call sits inside signature validation, so it runs on **every inbound +message**. Its timeout and retry budget are deliberately tighter than the +sibling registry plugins' for that reason: `timeout × (retry_max + 1)` is time a +request spends waiting before it can even be rejected. + +## Configuration + +```yaml +registry: + id: oanregistry + config: + url: http://registry:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + timeout: 2 + retry_max: 1 + retry_wait_min: 100ms + retry_wait_max: 500ms + cacheTTL: 60s +``` + +| Parameter | Required | Description | Default | +|-----------|----------|-------------|---------| +| `url` | **Yes** | Registry base URL, including the API version prefix. The plugin appends `/{entity}/search`. | — | +| `entity` | No | Registry entity to search. | `Participant` | +| `timeout` | No | Per-attempt request timeout, in seconds. Must be positive. | `2` | +| `retry_max` | No | Retry attempts after the first. `0` means do not retry, and is honoured as such. | `1` | +| `retry_wait_min` | No | Minimum backoff between attempts. | `100ms` | +| `retry_wait_max` | No | Maximum backoff between attempts. Also the ceiling a `Retry-After` header is clamped to. | `500ms` | +| `cacheTTL` | No | How long a resolved participant is reused. **Absent or `0` disables caching entirely.** | off | + +A `cache` plugin must also be configured for `cacheTTL` to have any effect. + +Startup fails on: a missing `url`, a `url` with no scheme or host, a +non-positive `timeout`, a negative `retry_max` or wait, an unparseable +`cacheTTL`, or `retry_wait_min` exceeding `retry_wait_max`. Catching +`registry:8081` (no scheme) at startup is cheaper than watching every lookup +fail once traffic arrives. + +### On `cacheTTL` + +The TTL is the **suspension-propagation window**: a cached participant keeps +verifying until the entry expires, even after the Network Operator suspends it. +That is why caching is off by default rather than something a deployment +inherits. + +The TTL is never taken from the key's own `validUntil`. That window is +typically a year, which would keep a suspended participant verifying for a year. + +Misses and refusals are never cached. Caching a miss would extend an outage; +caching a refusal would delay a reinstatement. + +## How a lookup works + +``` +Authorization: Signature keyId="||ed25519", ... + │ │ + ▼ ▼ + POST {url}/{entity}/search {"filters":{"participantId":{"eq":""}}} + │ + ▼ + walk node.keys[] for one whose osid == key_id, and whose use is signing + │ + ▼ + participant status == "active" + AND key status == "active" + AND key material present? → SUBSCRIBED + anything else → UNSUBSCRIBED +``` + +`key_id` is the **key's** `osid` (`node.keys[].osid`), not the participant's or +the node's. A record carries all three and they look alike; matching either of +the other two resolves the wrong thing, and keeps doing so the moment a second +key is published. + +**Only `participantId` is filtered on.** It is the schema's +`uniqueIndexFields`, so the registry already guarantees at most one match. +`osid` is system-generated and not indexed at all — on an Elasticsearch-backed +deployment, filtering on it matches nothing, which would turn every lookup into +a not-found. It is also nested inside the record, which a flat filter could not +reach in any case. The key identity is therefore checked client-side, where it +works on any backend and enforces exactly the same property. + +**`status` is not filtered on either.** Excluding suspended participants +server-side would return an empty result, making "suspended" indistinguishable +from "unknown" and losing the reason the caller reports. + +## How a provider record resolves + +``` +message.offer.provider.id ─┐ +message.resourceAttributes["@type"] ─┴─▶ bindingKey "|" + │ + ▼ + POST {url}/{providerEntity}/search {"filters":{"bindingKey":{"eq":"..."}}} + │ + ▼ the binding names its owner + POST {url}/{entity}/search {"filters":{"participantId":{"eq":"..."}}} + │ + ▼ + both statuses "active" and an upstream url present? → a call plan + anything else → ErrProviderRecordNotFound +``` + +Two reads, joined into one `model.ProviderRecord`: `baseUrl` from the +participant, the mappings and a **call plan per action** from the binding. + +```json +{ + "bindingKey": "mausamgram|openagrinet:WeatherObservation", + "requestMapping": "...", "responseMapping": "...", + "actions": { + "select": { "method": "GET", "path": "/get-daily", "timeoutMs": 30000, "retryMax": 3 }, + "confirm": { "method": "POST", "path": "/book" } + } +} +``` + +A capability serves several actions and they rarely share an endpoint — a +`confirm` that commits does not post where a `select` that reads gets — so the +endpoint, method and budget are per action. An action absent from `actions` is +one the capability does not serve, and a binding serving none at all is refused +outright rather than failing one action at a time. + +The owning participant is the one the **binding names**, not one parsed out of +the binding key — the registry owns that relationship, not the key format. + +Mapping references are carried **verbatim**. They are URLs the mapper resolves; +this plugin does not read, fetch or interpret them. + +`timeoutMs` and `retryMax` are zero when the registry omits them, meaning "the +caller applies its own default" — not "no timeout and no retries". + +Every way of saying *this capability cannot be served* — absent, withdrawn, +suspended, unroutable — returns `ErrProviderRecordNotFound`, because a caller +does the same thing with all of them. A registry that could not be **consulted** +returns its own error instead: that is an outage, not an answer. The two are +separated in metrics, never in the returned type. + +## What a caller gets back + +| Situation | Result | +|---|---| +| Registered, active, has a key | One `Subscription`, `Status: SUBSCRIBED` | +| Registered but suspended, or has no key | One `Subscription`, `Status: UNSUBSCRIBED` | +| Not registered | Empty slice, `nil` error | +| Registry unreachable or unreadable | `nil`, error | + +"Not found" is a legitimate answer, not an error — the caller turns an empty +slice into its own not-found. A participant that exists but may not sign comes +back with a status the caller rejects, so that **"unknown" and "suspended" stay +distinguishable** instead of collapsing into the same empty result. + +### Deny by default + +`model.IsKeyStatusUsable` is a deny-list: any status it does not recognise +counts as usable. Passing the registry's own `"inactive"` through unchanged +would therefore let a suspended participant's signature verify. Status mapping +here is a **security control, not a formatting step** — everything denies unless +explicitly allowed. + +Status is checked at **both** levels. A participant stays active while one of its +keys is retired, so a key carrying its own non-active status is refused even +though the participant is trading normally. + +The key validity window (`validFrom` / `validUntil`) is deliberately **not** +enforced. The Network Operator takes a participant off the network by setting +`status`, not by this plugin timing a key out. Both fields are mapped onto the +result for a caller to read, and nothing acts on them. + +Key material is published with an encoding label, e.g. +`"key": "base64:xq4+..."`. The label is stripped before the value reaches +`model.Subscription`, which carries the bare base64 that `signvalidator` hands +straight to `base64.StdEncoding.DecodeString`. Left on, it fails every +verification with a decode error pointing nowhere near the registry. + +## Observability + +Every lookup emits its duration and, when it did not resolve a key, the shared +plugin error counter. The `error_type` dimension is one of: + +Provider-record lookups report under `operation=provider_record`, with their own +outcomes: `binding_not_found` · `binding_inactive` · `binding_unowned` · +`participant_not_found` · `participant_inactive` · `no_upstream_url` · +`no_binding_key`. Each refusal is kept distinct: they all deny the call, but a +withdrawn capability and a suspended provider are different operational events. + +Signing-key lookups report under `operation=lookup`: + +`found` · `cache_hit` · `not_found` · `key_id_mismatch` · `key_not_signing` · +`inactive` · `key_inactive` · +`no_key` · `timeout` · `registry_error` · `decode_error` · `transport_error` + +Split on `error_type` when alerting. "Not a success" includes outcomes that are +the plugin working correctly — refusing a suspended participant is a successful +denial, and a routine suspension should not read as an incident. + +Two of these are worth watching separately. `not_found` means the caller is not +registered. `key_id_mismatch` means the participant **is** registered and the +key identity model is wrong — a sustained rate of that is a total outage that +would otherwise hide inside routine misses. + +## Notes for operators + +- **Service name, not `localhost`.** In a container the registry is reached by + its service name; `localhost` resolves to the adapter itself. +- **`Retry-After` is clamped** to `retry_wait_max`. The retry library honours it + unclamped, so a registry — or any ingress in front of one — answering + `Retry-After: 3600` would otherwise park a goroutine for an hour inside + signature validation, with no deadline on the inbound request to cut it short. +- **A record declaring an algorithm other than `ed25519` is logged as a + warning**, not refused. The header's algorithm is validated upstream, so a + disagreement cannot let a bad signature through — but it means the record and + the caller disagree about the key, which is worth seeing before it becomes a + verification failure nobody can explain. +- **More than one record for a `participantId`** is a registry integrity fault. + It is logged at error level and the lookup carries on, since the key check + still decides. diff --git a/pkg/plugin/implementation/oanregistry/cmd/plugin.go b/pkg/plugin/implementation/oanregistry/cmd/plugin.go new file mode 100644 index 00000000..efb35ca3 --- /dev/null +++ b/pkg/plugin/implementation/oanregistry/cmd/plugin.go @@ -0,0 +1,151 @@ +package main + +import ( + "context" + "errors" + "fmt" + "strconv" + "time" + + "github.com/beckn-one/beckn-onix/pkg/log" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/oanregistry" +) + +// Defaults for settings an operator leaves out. Only parseConfig can tell +// "absent" from "explicitly zero" -- retry_max of 0 is a legitimate "do not +// retry" -- so they are applied here. The values themselves live in the +// oanregistry package so there is exactly one place to change them. +const ( + defaultEntity = oanregistry.DefaultEntity + defaultProviderEntity = oanregistry.DefaultProviderEntity + defaultTimeout = oanregistry.DefaultTimeoutSeconds + defaultRetryMax = oanregistry.DefaultRetryMax + defaultRetryWaitMin = oanregistry.DefaultRetryWaitMin + defaultRetryWaitMax = oanregistry.DefaultRetryWaitMax +) + +// oanRegistryProvider implements the RegistryLookupProvider interface for the +// OAN registry plugin. +type oanRegistryProvider struct{} + +// newOANRegistryFunc creates a new OAN registry client. Indirected for tests. +var newOANRegistryFunc = oanregistry.New + +// parseConfig parses the configuration map into an oanregistry.Config, starting +// from the defaults and overriding whatever the operator supplied. +func (o oanRegistryProvider) parseConfig(config map[string]string) (*oanregistry.Config, error) { + cfg := &oanregistry.Config{ + URL: config["url"], + Entity: defaultEntity, + ProviderEntity: defaultProviderEntity, + Timeout: defaultTimeout, + RetryMax: defaultRetryMax, + RetryWaitMin: defaultRetryWaitMin, + RetryWaitMax: defaultRetryWaitMax, + } + + // Parse entity + if entity, exists := config["entity"]; exists && entity != "" { + cfg.Entity = entity + } + + // Parse providerEntity + if providerEntity, exists := config["providerEntity"]; exists && providerEntity != "" { + cfg.ProviderEntity = providerEntity + } + + // Parse cacheTTL. Absent means caching is off: the TTL is how long a + // suspended participant keeps verifying, so it is opt-in. + if cacheTTLStr, exists := config["cacheTTL"]; exists && cacheTTLStr != "" { + cacheTTL, err := time.ParseDuration(cacheTTLStr) + if err != nil { + return nil, fmt.Errorf("invalid cacheTTL value '%s': %w", cacheTTLStr, err) + } + if cacheTTL < 0 { + return nil, fmt.Errorf("cacheTTL must be non-negative, got %v", cacheTTL) + } + cfg.CacheTTL = cacheTTL + } + + // Parse timeout + if timeoutStr, exists := config["timeout"]; exists && timeoutStr != "" { + timeout, err := strconv.Atoi(timeoutStr) + if err != nil { + return nil, fmt.Errorf("invalid timeout value '%s': %w", timeoutStr, err) + } + if timeout <= 0 { + return nil, fmt.Errorf("timeout must be positive, got %d", timeout) + } + cfg.Timeout = timeout + } + + // Parse retry_max + if retryMaxStr, exists := config["retry_max"]; exists && retryMaxStr != "" { + retryMax, err := strconv.Atoi(retryMaxStr) + if err != nil { + return nil, fmt.Errorf("invalid retry_max value '%s': %w", retryMaxStr, err) + } + if retryMax < 0 { + return nil, fmt.Errorf("retry_max must be non-negative, got %d", retryMax) + } + cfg.RetryMax = retryMax + } + + // Parse retry_wait_min + if retryWaitMinStr, exists := config["retry_wait_min"]; exists && retryWaitMinStr != "" { + retryWaitMin, err := time.ParseDuration(retryWaitMinStr) + if err != nil { + return nil, fmt.Errorf("invalid retry_wait_min value '%s': %w", retryWaitMinStr, err) + } + if retryWaitMin < 0 { + return nil, fmt.Errorf("retry_wait_min must be non-negative, got %v", retryWaitMin) + } + cfg.RetryWaitMin = retryWaitMin + } + + // Parse retry_wait_max + if retryWaitMaxStr, exists := config["retry_wait_max"]; exists && retryWaitMaxStr != "" { + retryWaitMax, err := time.ParseDuration(retryWaitMaxStr) + if err != nil { + return nil, fmt.Errorf("invalid retry_wait_max value '%s': %w", retryWaitMaxStr, err) + } + if retryWaitMax < 0 { + return nil, fmt.Errorf("retry_wait_max must be non-negative, got %v", retryWaitMax) + } + cfg.RetryWaitMax = retryWaitMax + } + + if cfg.RetryWaitMin > cfg.RetryWaitMax { + return nil, fmt.Errorf("retry_wait_min (%v) must not exceed retry_wait_max (%v)", cfg.RetryWaitMin, cfg.RetryWaitMax) + } + + return cfg, nil +} + +// New creates a new OAN registry plugin instance. +func (o oanRegistryProvider) New(ctx context.Context, cache definition.Cache, config map[string]string) (definition.RegistryLookup, func() error, error) { + if ctx == nil { + return nil, nil, errors.New("context cannot be nil") + } + + cfg, err := o.parseConfig(config) + if err != nil { + log.Errorf(ctx, err, "Failed to parse OAN registry configuration") + return nil, nil, fmt.Errorf("failed to parse oan registry configuration: %w", err) + } + + log.Debugf(ctx, "OAN registry config mapped: %+v", cfg) + + client, closer, err := newOANRegistryFunc(ctx, cache, cfg) + if err != nil { + log.Errorf(ctx, err, "Failed to create OAN registry instance") + return nil, nil, err + } + + log.Infof(ctx, "OAN registry instance created successfully") + return client, closer, nil +} + +// Provider is the exported plugin instance. +var Provider = oanRegistryProvider{} diff --git a/pkg/plugin/implementation/oanregistry/cmd/plugin_test.go b/pkg/plugin/implementation/oanregistry/cmd/plugin_test.go new file mode 100644 index 00000000..8d36d67a --- /dev/null +++ b/pkg/plugin/implementation/oanregistry/cmd/plugin_test.go @@ -0,0 +1,249 @@ +package main + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + "time" + + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/oanregistry" +) + +func defaultConfig() *oanregistry.Config { + return &oanregistry.Config{ + Entity: defaultEntity, + ProviderEntity: defaultProviderEntity, + Timeout: defaultTimeout, + RetryMax: defaultRetryMax, + RetryWaitMin: defaultRetryWaitMin, + RetryWaitMax: defaultRetryWaitMax, + } +} + +func TestParseConfig(t *testing.T) { + t.Parallel() + + withDefaults := func(apply func(*oanregistry.Config)) *oanregistry.Config { + cfg := defaultConfig() + apply(cfg) + return cfg + } + + testCases := []struct { + name string + config map[string]string + expected *oanregistry.Config + expectedErr string + }{ + { + name: "applies defaults when only a URL is given", + config: map[string]string{"url": "http://registry:8081/api/v1"}, + expected: withDefaults(func(c *oanregistry.Config) { c.URL = "http://registry:8081/api/v1" }), + }, + { + name: "reads every supported setting", + config: map[string]string{ + "url": "http://registry:8081/api/v1", + "entity": "Subscriber", + "cacheTTL": "30s", + "timeout": "5", + "retry_max": "3", + "retry_wait_min": "200ms", + "retry_wait_max": "1s", + }, + expected: &oanregistry.Config{ + URL: "http://registry:8081/api/v1", + Entity: "Subscriber", + ProviderEntity: defaultProviderEntity, + CacheTTL: 30 * time.Second, + Timeout: 5, + RetryMax: 3, + RetryWaitMin: 200 * time.Millisecond, + RetryWaitMax: time.Second, + }, + }, + { + name: "reads an overridden provider entity", + config: map[string]string{ + "url": "http://registry:8081", + "providerEntity": "ProviderCapability", + }, + expected: withDefaults(func(c *oanregistry.Config) { + c.URL = "http://registry:8081" + c.ProviderEntity = "ProviderCapability" + }), + }, + { + name: "ignores an empty provider entity and keeps the default", + config: map[string]string{ + "url": "http://registry:8081", + "providerEntity": "", + }, + expected: withDefaults(func(c *oanregistry.Config) { + c.URL = "http://registry:8081" + }), + }, + { + // Caching is off unless asked for: the TTL is how long a suspended + // participant keeps verifying. + name: "leaves caching disabled when no TTL is set", + config: map[string]string{"url": "http://registry:8081"}, + expected: withDefaults(func(c *oanregistry.Config) { c.URL = "http://registry:8081" }), + }, + { + // Distinct from "unset", which yields the default of 1. + name: "honours an explicit retry_max of zero", + config: map[string]string{"url": "http://registry:8081", "retry_max": "0"}, + expected: withDefaults(func(c *oanregistry.Config) { c.URL = "http://registry:8081"; c.RetryMax = 0 }), + }, + { + name: "ignores empty values and keeps the defaults", + config: map[string]string{"url": "http://registry:8081", "entity": "", "timeout": ""}, + expected: withDefaults(func(c *oanregistry.Config) { c.URL = "http://registry:8081" }), + }, + { + name: "rejects a non-numeric timeout", + config: map[string]string{"url": "http://registry:8081", "timeout": "soon"}, + expectedErr: "invalid timeout value 'soon'", + }, + { + name: "rejects a non-positive timeout", + config: map[string]string{"url": "http://registry:8081", "timeout": "0"}, + expectedErr: "timeout must be positive, got 0", + }, + { + name: "rejects a negative retry_max", + config: map[string]string{"url": "http://registry:8081", "retry_max": "-1"}, + expectedErr: "retry_max must be non-negative, got -1", + }, + { + name: "rejects a malformed cacheTTL", + config: map[string]string{"url": "http://registry:8081", "cacheTTL": "3600"}, + expectedErr: "invalid cacheTTL value '3600'", + }, + { + name: "rejects a malformed retry_wait_min", + config: map[string]string{"url": "http://registry:8081", "retry_wait_min": "quick"}, + expectedErr: "invalid retry_wait_min value 'quick'", + }, + { + name: "rejects a minimum backoff above the maximum", + config: map[string]string{ + "url": "http://registry:8081", + "retry_wait_min": "2s", + "retry_wait_max": "1s", + }, + expectedErr: "retry_wait_min (2s) must not exceed retry_wait_max (1s)", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got, err := oanRegistryProvider{}.parseConfig(tc.config) + + if tc.expectedErr != "" { + if err == nil { + t.Fatalf("expected error %q but got none", tc.expectedErr) + } + if !strings.Contains(err.Error(), tc.expectedErr) { + t.Errorf("expected error containing %q, got %q", tc.expectedErr, err.Error()) + } + return + } + + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !reflect.DeepEqual(got, tc.expected) { + t.Errorf("expected config %+v, got %+v", tc.expected, got) + } + }) + } +} + +func TestNew(t *testing.T) { + t.Parallel() + + t.Run("rejects a nil context", func(t *testing.T) { + t.Parallel() + + //nolint:staticcheck // deliberately passing a nil context to assert the guard. + _, _, err := oanRegistryProvider{}.New(nil, nil, map[string]string{"url": "http://registry:8081"}) + if err == nil { + t.Fatal("expected an error for a nil context, got none") + } + }) + + t.Run("rejects a missing URL", func(t *testing.T) { + t.Parallel() + + _, _, err := oanRegistryProvider{}.New(context.Background(), nil, map[string]string{}) + if err == nil { + t.Fatal("expected an error for a missing URL, got none") + } + }) + + t.Run("rejects an unparseable config", func(t *testing.T) { + t.Parallel() + + _, _, err := oanRegistryProvider{}.New(context.Background(), nil, map[string]string{ + "url": "http://registry:8081", + "timeout": "soon", + }) + if err == nil { + t.Fatal("expected an error for an invalid timeout, got none") + } + }) + + t.Run("builds a client from a valid config", func(t *testing.T) { + t.Parallel() + + client, closer, err := oanRegistryProvider{}.New(context.Background(), nil, map[string]string{ + "url": "http://registry:8081/api/v1", + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if client == nil { + t.Fatal("expected a client, got nil") + } + if closer == nil { + t.Fatal("expected a closer, got nil") + } + if err := closer(); err != nil { + t.Errorf("expected the closer to succeed, got: %v", err) + } + }) + + // Deliberately NOT parallel: this swaps the package-level newOANRegistryFunc, + // so running it alongside its parallel siblings would race on that variable. + // Go never schedules a non-parallel subtest concurrently with parallel ones, + // which is what makes this safe -- do not add t.Parallel() "for consistency". + t.Run("propagates a client construction failure", func(t *testing.T) { + original := newOANRegistryFunc + t.Cleanup(func() { newOANRegistryFunc = original }) + + wantErr := errors.New("boom") + newOANRegistryFunc = func(context.Context, definition.Cache, *oanregistry.Config) (*oanregistry.Client, func() error, error) { + return nil, nil, wantErr + } + + _, _, err := oanRegistryProvider{}.New(context.Background(), nil, map[string]string{"url": "http://registry:8081"}) + if !errors.Is(err, wantErr) { + t.Fatalf("expected the underlying error to be propagated, got: %v", err) + } + }) +} + +// TestProviderSatisfiesTheInterface fails at compile time if the exported +// Provider ever stops matching what the plugin loader looks up. +func TestProviderSatisfiesTheInterface(t *testing.T) { + t.Parallel() + + var _ definition.RegistryLookupProvider = Provider +} diff --git a/pkg/plugin/implementation/oanregistry/oanregistry.go b/pkg/plugin/implementation/oanregistry/oanregistry.go new file mode 100644 index 00000000..cebf4219 --- /dev/null +++ b/pkg/plugin/implementation/oanregistry/oanregistry.go @@ -0,0 +1,652 @@ +// Package oanregistry resolves participant signing keys from the OAN Registry +// (a SunbirdRC deployment) so inbound Beckn signatures can be verified. +// +// It implements definition.RegistryLookup only. Onboarding, key publication and +// status changes all happen through the registry's own Participant APIs. +package oanregistry + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/beckn-one/beckn-onix/pkg/log" + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/telemetry" + "github.com/hashicorp/go-retryablehttp" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" +) + +// Defaults applied when an operator leaves a setting out. The timeout and retry +// budget are deliberately tighter than the sibling registry plugins': this call +// sits inside signature validation on every inbound message, so timeout x +// (retry_max + 1) is time a request spends waiting before it can be rejected. +// Exported so cmd/plugin.go applies the same values -- these are the single +// source of truth for them. +const ( + DefaultEntity = "Participant" + DefaultProviderEntity = "ProviderSchema" + DefaultTimeoutSeconds = 2 + DefaultRetryMax = 1 + DefaultRetryWaitMin = 100 * time.Millisecond + DefaultRetryWaitMax = 500 * time.Millisecond +) + +// Registry field names. They live here rather than in config because they +// change only when the registry API changes -- never between two deployments of +// this code -- and a typo in a config key would surface at runtime as a +// misleading "no key found" rather than at startup. +const ( + fieldParticipantID = "participantId" + fieldBindingKey = "bindingKey" + searchPath = "search" +) + +// Key "use" values. A key is scoped to a single purpose, so the signing key is +// the only one resolvable by the id in a request header; an encryption key is +// picked up alongside it by use. +const ( + useSign = "sign" + useEncr = "encr" +) + +// keyEncodingPrefix labels the encoding of a published key value, e.g. +// "base64:xq4+...". model.Subscription carries the bare base64 that signvalidator +// hands straight to base64.StdEncoding.DecodeString, so the label is stripped on +// the way through -- left on, it fails every verification with a decode error +// pointing nowhere near the registry. +const keyEncodingPrefix = "base64:" + +// statusActive is the only registry status that permits verification. It is +// checked at both levels: a participant stays active while one of its keys is +// retired, which is the whole point of per-key status. +const statusActive = "active" + +// expectedAlgorithm is the only signing algorithm on this network. The request +// header is checked against it upstream; this is only used to flag a record that +// disagrees. +const expectedAlgorithm = "ed25519" + +// Beckn subscription statuses, as understood by model.IsKeyStatusUsable. +const ( + statusSubscribed = "SUBSCRIBED" + statusUnsubscribed = "UNSUBSCRIBED" +) + +// Lookup outcomes, used as the error_type metric dimension and in logs. Failure +// outcomes are kept distinct so a dead registry and a malformed body do not +// collapse into one series -- splitting on error_type is the whole point of +// recording it. +const ( + outcomeFound = "found" + outcomeCacheHit = "cache_hit" + outcomeNotFound = "not_found" + outcomeKeyIDMismatch = "key_id_mismatch" + outcomeKeyNotSigning = "key_not_signing" + outcomeInactive = "inactive" + outcomeKeyInactive = "key_inactive" + outcomeNoKey = "no_key" + outcomeTimeout = "timeout" + outcomeRegistryError = "registry_error" + outcomeDecodeError = "decode_error" + outcomeTransportError = "transport_error" +) + +// Failure classes, wrapped so Lookup can tell them apart without inspecting +// error strings. +var ( + errRegistryStatus = errors.New("registry returned an error status") + errDecodeResponse = errors.New("registry response could not be decoded") +) + +// classify maps a search failure onto the outcome vocabulary above. +func classify(err error) string { + switch { + case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled): + return outcomeTimeout + case errors.Is(err, errRegistryStatus): + return outcomeRegistryError + case errors.Is(err, errDecodeResponse): + return outcomeDecodeError + default: + return outcomeTransportError + } +} + +const ( + pluginID = "oanregistry" + pluginType = "registry" + operationLookup = "lookup" + operationProviderRecord = "provider_record" +) + +// Config holds configuration parameters for the OAN registry client. +type Config struct { + // URL is the registry base including any API version prefix, + // e.g. "http://registry:8081/api/v1". + URL string `yaml:"url" json:"url"` + // Entity names the participant entity, ProviderEntity the capability-binding + // entity. They are separate registry collections and are searched separately. + Entity string `yaml:"entity" json:"entity"` + ProviderEntity string `yaml:"providerEntity" json:"providerEntity"` + CacheTTL time.Duration `yaml:"cacheTTL" json:"cacheTTL"` + Timeout int `yaml:"timeout" json:"timeout"` + RetryMax int `yaml:"retry_max" json:"retry_max"` + RetryWaitMin time.Duration `yaml:"retry_wait_min" json:"retry_wait_min"` + RetryWaitMax time.Duration `yaml:"retry_wait_max" json:"retry_wait_max"` +} + +// Client resolves participants from the OAN registry. It is safe for concurrent +// use: every field is set once in New and never mutated afterwards. +type Client struct { + searchURL string + providerSearchURL string + client *retryablehttp.Client + cache definition.Cache + cacheTTL time.Duration +} + +// participant is the subset of a registry record this plugin reads. The registry +// carries a good deal more -- Sunbird audit fields (osCreatedAt, osOwner, ...) and +// the participant and node osids among them -- and none of it is modelled here. +// encoding/json drops what it cannot place, so every field left out is one less +// thing to break when the registry schema moves. +// +// Only the participant's own status sits at this level. Keys hang off the node, +// each carrying its own identity and status, so resolving one is a walk rather +// than a field read. +type participant struct { + ParticipantID string `json:"participantId"` + Status string `json:"status"` + Node node `json:"node"` + Upstream upstream `json:"upstream"` +} + +// upstream is the backend a provider participant fronts. It is present only on +// records that front one: a participant is either a network peer publishing keys +// under node, or a provider publishing an upstream, and the two do not overlap. +type upstream struct { + BaseURL string `json:"baseUrl"` +} + +// node is the network-facing half of a participant record. +type node struct { + SubscriberURL string `json:"subscriberUrl"` + Type string `json:"type"` + Keys []key `json:"keys"` +} + +// key is one published key. A node publishes several -- separate signing and +// encryption keys, and more than one signing key while a rotation is in flight -- +// so a key is identified by its own OSID rather than by its position. +type key struct { + OSID string `json:"osid"` + KeyID string `json:"keyId"` + Use string `json:"use"` + Algorithm string `json:"alg"` + Value string `json:"key"` + Status string `json:"status"` + ValidFrom string `json:"validFrom"` + ValidUntil string `json:"validUntil"` +} + +// publicKey returns the key material with its encoding label removed, ready for +// the base64 decode the caller performs. +func (k key) publicKey() string { + return strings.TrimPrefix(k.Value, keyEncodingPrefix) +} + +// isSigning reports whether this key may verify a signature. +// +// An absent use is accepted: it predates the discriminator, and if the guess is +// wrong the signature simply fails to verify, which is the safe direction. An +// explicitly non-signing use is refused, so an encryption key's osid arriving in +// a signing header is reported as exactly that rather than as a missing key. +func (k key) isSigning() bool { + return k.Use == "" || strings.EqualFold(k.Use, useSign) +} + +type eqFilter struct { + Eq string `json:"eq"` +} + +type searchRequest struct { + Filters map[string]eqFilter `json:"filters"` +} + +// validate checks if the provided OAN registry configuration is valid. +func validate(cfg *Config) error { + if cfg == nil { + return fmt.Errorf("oan registry config cannot be nil") + } + if cfg.URL == "" { + return fmt.Errorf("oan registry URL cannot be empty") + } + // url.Parse accepts almost anything, so check the parts that actually have + // to be there. Catching "registry:8081" (no scheme) at startup is far + // cheaper than watching every lookup fail once traffic arrives. + parsed, err := url.Parse(cfg.URL) + if err != nil { + return fmt.Errorf("invalid oan registry URL %q: %w", cfg.URL, err) + } + if parsed.Scheme == "" || parsed.Host == "" { + return fmt.Errorf("oan registry URL %q must include a scheme and host, e.g. http://:/api/v1", cfg.URL) + } + return nil +} + +// New creates a new instance of Client. +func New(ctx context.Context, cache definition.Cache, cfg *Config) (*Client, func() error, error) { + log.Debugf(ctx, "Initializing OAN registry client with config: %+v", cfg) + + if err := validate(cfg); err != nil { + return nil, nil, err + } + + entity := cfg.Entity + if entity == "" { + entity = DefaultEntity + } + providerEntity := cfg.ProviderEntity + if providerEntity == "" { + providerEntity = DefaultProviderEntity + } + + rc := retryablehttp.NewClient() + + // retryablehttp logs every attempt and retry straight to stderr, outside + // pkg/log, so it is neither structured nor filterable. The same information is + // already emitted by this plugin's own logging and metrics. + rc.Logger = nil + + // Always bounded. The sibling registry plugins apply their timeout only when + // one is configured, which leaves it at zero -- meaning no timeout at all -- + // when it is not. That matters because the transport retryablehttp ships with + // sets no ResponseHeaderTimeout, so a peer that accepts the connection and + // then goes quiet is otherwise bounded by nothing. + timeout := cfg.Timeout + if timeout <= 0 { + timeout = DefaultTimeoutSeconds + } + rc.HTTPClient.Timeout = time.Duration(timeout) * time.Second + + // Retry settings are taken as given: RetryMax of 0 is a legitimate "do not + // retry", so it must not be confused with "unset". parseConfig supplies the + // defaults, since only it can tell the two apart. + rc.RetryMax = cfg.RetryMax + if cfg.RetryWaitMin > 0 { + rc.RetryWaitMin = cfg.RetryWaitMin + } + if cfg.RetryWaitMax > 0 { + rc.RetryWaitMax = cfg.RetryWaitMax + } + + // DefaultBackoff honours a Retry-After header on 429 and 503 and returns it + // *unclamped* -- it bypasses its own RetryWaitMax ceiling. A registry, or any + // ingress in front of one, answering "Retry-After: 3600" would park this + // goroutine for an hour inside signature validation. The inbound request + // context typically has no deadline of its own, so nothing else would cut it + // short. Clamp it back to the configured ceiling. + rc.Backoff = func(min, max time.Duration, attempt int, resp *http.Response) time.Duration { + if wait := retryablehttp.DefaultBackoff(min, max, attempt, resp); wait < max { + return wait + } + return max + } + + client := &Client{ + searchURL: searchURLFor(cfg.URL, entity), + providerSearchURL: searchURLFor(cfg.URL, providerEntity), + client: rc, + cache: cache, + cacheTTL: cfg.CacheTTL, + } + + closer := func() error { + log.Debugf(ctx, "Cleaning up OAN registry client resources") + if client.client != nil { + client.client.HTTPClient.CloseIdleConnections() + } + return nil + } + + log.Infof(ctx, "OAN registry client is created successfully") + return client, closer, nil +} + +// Lookup resolves the signing key for the participant and key named in the +// request. The caller populates only SubscriberID and KeyID; every other field +// on the request is zero. +// +// A missing participant returns (nil, nil) rather than an error: "not found" is +// a legitimate answer, and the caller turns an empty slice into its own +// not-found error. A participant that exists but may not sign is returned with +// a status the caller rejects, so that "unknown" and "suspended" stay +// distinguishable instead of collapsing into the same empty result. +func (c *Client) Lookup(ctx context.Context, req *model.Subscription) ([]model.Subscription, error) { + start := time.Now() + tracer := otel.Tracer(telemetry.ScopeName, trace.WithInstrumentationVersion(telemetry.ScopeVersion)) + ctx, span := tracer.Start(ctx, "oan registry lookup") + defer span.End() + + // M2: an empty key id would match any record whose OSID is absent. Unreachable + // under the current shape, where every record carries one -- but cheap, and it + // stops being unreachable the moment OSID maps to a field that can be missing. + if req.SubscriberID == "" || req.KeyID == "" { + span.SetAttributes(telemetry.AttrErrorType.String(outcomeNotFound)) + c.emitMetrics(ctx, start, operationLookup, outcomeNotFound) + return nil, nil + } + + cacheKey := fmt.Sprintf("oan_lookup_%s_%s", req.SubscriberID, req.KeyID) + if cached, ok := c.cached(ctx, tracer, cacheKey); ok { + log.Debugf(ctx, "OAN registry lookup cache hit for key: %s", cacheKey) + span.SetAttributes(telemetry.AttrErrorType.String(outcomeCacheHit)) + c.emitMetrics(ctx, start, operationLookup, outcomeCacheHit) + return cached, nil + } + + found, matched, searchOutcome, err := c.search(ctx, tracer, req.SubscriberID, req.KeyID) + if err != nil { + outcome := classify(err) + span.RecordError(err) + span.SetStatus(codes.Error, outcome) + span.SetAttributes(telemetry.AttrErrorType.String(outcome)) + c.emitMetrics(ctx, start, operationLookup, outcome) + return nil, err + } + span.SetAttributes(telemetry.AttrErrorType.String(searchOutcome)) + if searchOutcome != outcomeFound { + // Both of these give the caller an empty result, but they are very + // different facts and must not share a metric. "not_found" means this + // caller is not registered; "key_id_mismatch" means the participant is + // registered and the key identity model is wrong -- a sustained rate of + // the latter is a total outage that would otherwise hide inside routine + // misses. + c.emitMetrics(ctx, start, operationLookup, searchOutcome) + return nil, nil + } + + status, outcome := resolveStatus(found, matched) + results := []model.Subscription{toSubscription(found, matched, status)} + + // The header's algorithm was already validated upstream, so a disagreement + // here cannot let a bad signature through -- but it means the record and the + // caller disagree about the key, which is worth surfacing before it becomes a + // verification failure nobody can explain. + if matched.Algorithm != "" && !strings.EqualFold(matched.Algorithm, expectedAlgorithm) { + log.Warnf(ctx, "OAN registry participantId=%s osid=%s declares algorithm %q, expected %q", + req.SubscriberID, req.KeyID, matched.Algorithm, expectedAlgorithm) + } + + if outcome == outcomeFound { + log.Debugf(ctx, "OAN registry resolved participantId=%s osid=%s", req.SubscriberID, req.KeyID) + c.cacheResult(ctx, cacheKey, results) + } else { + // Not an error -- the plugin looked, and correctly declined. Logged at + // Info so a refused signature is traceable without reading as a fault. + log.Infof(ctx, "OAN registry participantId=%s osid=%s is not usable: %s", req.SubscriberID, req.KeyID, outcome) + } + + span.SetAttributes(telemetry.AttrErrorType.String(outcome)) + c.emitMetrics(ctx, start, operationLookup, outcome) + return results, nil +} + +// search asks the registry for the participant holding this business id, and +// returns it only if it carries the key the caller asked about. +// +// Only participantId is filtered on, deliberately. It is the schema's +// uniqueIndexFields, so the registry already guarantees at most one match. osid +// is a system-generated field and is not indexed at all, so adding it as a +// second filter does not narrow anything -- on an Elasticsearch-backed registry +// it matches nothing, which would turn every lookup into a not-found. The key +// identity is therefore checked below, client-side, where it works on any +// backend and enforces exactly the same property. +// +// status is deliberately not filtered on either. Excluding suspended +// participants server-side would return an empty result, making "suspended" +// indistinguishable from "unknown" and losing the reason the caller reports. +func (c *Client) search(ctx context.Context, tracer trace.Tracer, participantID, keyID string) (participant, key, string, error) { + records, err := searchRecords[participant](ctx, c, tracer, c.searchURL, map[string]eqFilter{ + fieldParticipantID: {Eq: participantID}, + }) + if err != nil { + return participant{}, key{}, "", err + } + + if len(records) == 0 { + log.Infof(ctx, "OAN registry has no record for participantId=%s", participantID) + return participant{}, key{}, outcomeNotFound, nil + } + if len(records) > 1 { + // participantId is the schema's unique index, so this is a registry + // integrity fault rather than something to resolve. Carry on -- the key + // check below still decides -- but say so loudly. + log.Errorf(ctx, nil, "OAN registry returned %d records for participantId=%s, expected at most 1", + len(records), participantID) + } + + // The key identity check the filter cannot do. Keys hang off the node, so this + // walks both levels. Scanning records rather than taking records[0] also covers + // the case the second filter was originally meant to guard: a stale or + // soft-deleted record sharing the participantId. + for _, record := range records { + for _, k := range record.Node.Keys { + if k.OSID != keyID { + continue + } + if !k.isSigning() { + // The osid resolved -- to a key that may not sign. Kept apart from a + // miss because it is a different fact: the caller is registered and + // sent a real key id, just one scoped to another purpose. + log.Errorf(ctx, nil, "OAN registry participantId=%s osid=%s is a %q key, not a signing key", + participantID, keyID, k.Use) + return participant{}, key{}, outcomeKeyNotSigning, nil + } + return record, k, outcomeFound, nil + } + } + + // Reported separately from not-found on purpose: the participant exists, so + // this says the key identity model is wrong rather than that the caller is + // unregistered. Logged at Error because a sustained rate of it is an outage. + log.Errorf(ctx, nil, "OAN registry has %d record(s) for participantId=%s but none carrying key osid %s", + len(records), participantID, keyID) + return participant{}, key{}, outcomeKeyIDMismatch, nil +} + +// resolveStatus maps the registry's own vocabulary onto the Beckn status the +// caller checks, and reports which outcome was reached. +// +// Only `status` is consulted -- at both levels. The key validity window +// (validFrom / validUntil) is deliberately NOT enforced: the Network Operator controls +// participation entirely through `status`, so an expired key is taken off the +// network by setting status rather than by this plugin timing it out. Those two +// fields are mapped onto the result for a caller to read, and nothing acts on +// them. Decided 20 Aug 2026; flagged as provisional. +// +// This is a security control, not a formatting step. model.IsKeyStatusUsable is +// a deny-list, so any status it does not recognise counts as usable -- passing +// the registry's "inactive" through unchanged would let a suspended +// participant's signature verify. Everything therefore denies unless explicitly +// allowed. +func resolveStatus(p participant, k key) (status, outcome string) { + if !strings.EqualFold(p.Status, statusActive) { + return statusUnsubscribed, outcomeInactive + } + // Checked separately from the participant's: a participant stays active while a + // single key is retired, and a retired key has to stop verifying on its own. + if !strings.EqualFold(k.Status, statusActive) { + return statusUnsubscribed, outcomeKeyInactive + } + if k.publicKey() == "" { + // Active but unusable. Denying here gives the caller a clear reason + // instead of an empty key that fails opaquely further down. + return statusUnsubscribed, outcomeNoKey + } + return statusSubscribed, outcomeFound +} + +// parseTime reads an RFC3339 timestamp, reporting whether it was present and +// well formed. An absent or unparseable value yields the zero time rather than +// an error: these timestamps are informational, so a malformed one must not +// fail a lookup. +func parseTime(value string) (time.Time, bool) { + if value == "" { + return time.Time{}, false + } + parsed, err := time.Parse(time.RFC3339, value) + if err != nil { + return time.Time{}, false + } + return parsed, true +} + +// toSubscription builds the value the sign-validation step consumes. +// +// This is the only place a model.Subscription is constructed, which is what +// guarantees Status is always set: its zero value "" is absent from +// IsKeyStatusUsable's deny-list and would therefore authorise the caller. +func toSubscription(p participant, k key, status string) model.Subscription { + // Informational only: nothing acts on these. The validity window is not + // enforced -- participation is controlled entirely through `status`. + validFrom, _ := parseTime(k.ValidFrom) + validUntil, _ := parseTime(k.ValidUntil) + + // Domain is absent from the OAN record and so is left unset. Nothing on the + // signature-validation path reads it. + return model.Subscription{ + Subscriber: model.Subscriber{ + SubscriberID: p.ParticipantID, + URL: p.Node.SubscriberURL, + Type: p.Node.Type, + }, + KeyID: k.OSID, + SigningPublicKey: k.publicKey(), + EncrPublicKey: encryptionKey(p.Node), + ValidFrom: validFrom, + ValidUntil: validUntil, + Status: status, + } +} + +// encryptionKey returns the node's active encryption key, or "" when it publishes +// none. It is resolved by use rather than by id: the request header names the +// signing key only, so there is nothing to match an encryption key against. +func encryptionKey(n node) string { + for _, k := range n.Keys { + if strings.EqualFold(k.Use, useEncr) && strings.EqualFold(k.Status, statusActive) { + return k.publicKey() + } + } + return "" +} + +// cachingEnabled reports whether the cache should be consulted at all. +// +// cacheTTL is 0 by default, which disables caching entirely rather than writing +// entries with a zero TTL. A cached entry keeps a suspended participant +// verifying until it expires, so the TTL is exactly the suspension-propagation +// window and is left to the operator to opt into. +func (c *Client) cachingEnabled() bool { + return c.cache != nil && c.cacheTTL > 0 +} + +func (c *Client) cached(ctx context.Context, tracer trace.Tracer, key string) ([]model.Subscription, bool) { + if !c.cachingEnabled() { + return nil, false + } + + cacheCtx, span := tracer.Start(ctx, "cache lookup") + defer span.End() + + raw, err := c.cache.Get(cacheCtx, key) + if err != nil { + return nil, false + } + var results []model.Subscription + if err := json.Unmarshal([]byte(raw), &results); err != nil { + log.Warnf(ctx, "Discarding unreadable cache entry for key %s: %v", key, err) + return nil, false + } + + // toSubscription guarantees Status is always set, but that invariant only + // covers values this process wrote. A cache is shared, outlives a deploy and + // can hold entries written by another version -- and an empty Status is + // absent from IsKeyStatusUsable's deny-list, so it would read as usable. + // Re-check on the way in rather than trusting the entry. + if len(results) != 1 || results[0].Status == "" { + log.Warnf(ctx, "Discarding malformed cache entry for key %s", key) + return nil, false + } + return results, true +} + +// cacheResult caches a usable result. Callers must not pass a not-found or unusable +// participant: caching those would extend an outage and delay a reinstatement. +// +// The TTL comes only from configuration, never from the record's own validity +// window -- that window is typically a year, which would keep a suspended +// participant verifying for a year. +func (c *Client) cacheResult(ctx context.Context, key string, results []model.Subscription) { + if !c.cachingEnabled() { + return + } + data, err := json.Marshal(results) + if err != nil { + log.Warnf(ctx, "Failed to encode OAN registry lookup for caching, key %s: %v", key, err) + return + } + if err := c.cache.Set(ctx, key, string(data), c.cacheTTL); err != nil { + log.Warnf(ctx, "Failed to cache OAN registry lookup for key %s: %v", key, err) + } +} + +// successOutcomes is an allow-list, deliberately: a new outcome counts as a +// failure until someone says otherwise. +// +// The inverse -- listing the failures and letting anything unlisted fall through +// as success -- is the same shape as model.IsKeyStatusUsable, which is the bug +// resolveStatus exists to work around. Here the blast radius is a dashboard +// rather than an auth decision, but the failure is just as silent: add a +// success-like outcome, forget to list it, and the error rate quietly stops +// being true. +var successOutcomes = map[string]bool{ + outcomeFound: true, + outcomeCacheHit: true, +} + +// emitMetrics emits the duration of every lookup, and the shared plugin error counter +// for anything that did not resolve a key. +// +// Note that "not a success" includes outcomes that are the plugin working +// correctly: refusing a suspended participant is a successful denial. Split on +// error_type when alerting, or a routine suspension reads as an incident. +func (c *Client) emitMetrics(ctx context.Context, start time.Time, operation, outcome string) { + m, err := telemetry.GetMetrics(ctx) + if err != nil { + return + } + + attrs := metric.WithAttributes( + telemetry.AttrPluginID.String(pluginID), + telemetry.AttrPluginType.String(pluginType), + telemetry.AttrOperation.String(operation), + telemetry.AttrErrorType.String(outcome), + ) + + m.PluginExecutionDuration.Record(ctx, time.Since(start).Seconds(), attrs) + if !successOutcomes[outcome] { + m.PluginErrorsTotal.Add(ctx, 1, attrs) + } +} diff --git a/pkg/plugin/implementation/oanregistry/oanregistry_test.go b/pkg/plugin/implementation/oanregistry/oanregistry_test.go new file mode 100644 index 00000000..684d5508 --- /dev/null +++ b/pkg/plugin/implementation/oanregistry/oanregistry_test.go @@ -0,0 +1,1577 @@ +package oanregistry + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/telemetry" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +const ( + testParticipantID = "provider-a-001" + testOSID = "1-d0442000-a677-4cfc-bd8f-02696c6088b3" + testPublicKey = "MCowBQYDK2VwAyEA3fS8bYhWEfmM7Zjk9x0EhAmvQKp3fMHXqTiA5xL1Qmw=" +) + +// mockCache is a test double for definition.Cache that records what it was asked +// to do, so tests can assert the cache was (or was not) used. +type mockCache struct { + getFunc func(ctx context.Context, key string) (string, error) + + getCalls int + setCalls int + setKey string + setVal string + setTTL time.Duration + setErr error +} + +func (m *mockCache) Get(ctx context.Context, key string) (string, error) { + m.getCalls++ + if m.getFunc != nil { + return m.getFunc(ctx, key) + } + return "", errors.New("cache miss") +} + +func (m *mockCache) Set(ctx context.Context, key, value string, ttl time.Duration) error { + m.setCalls++ + m.setKey = key + m.setVal = value + m.setTTL = ttl + return m.setErr +} + +func (m *mockCache) Delete(ctx context.Context, key string) error { return nil } +func (m *mockCache) Clear(ctx context.Context) error { return nil } + +// recordJSON renders a registry search response containing the given records. +func recordJSON(t *testing.T, records ...participant) string { + t.Helper() + if records == nil { + records = []participant{} + } + data, err := json.Marshal(records) + if err != nil { + t.Fatalf("failed to marshal test records: %v", err) + } + return string(data) +} + +// signingKey is a healthy signing key with an open validity window. Its value +// carries the registry's encoding label so every test that reaches +// toSubscription also exercises the prefix being stripped. +func signingKey() key { + return key{ + OSID: testOSID, + KeyID: "k1", + Use: useSign, + Algorithm: expectedAlgorithm, + Value: keyEncodingPrefix + testPublicKey, + Status: "active", + ValidFrom: time.Now().Add(-24 * time.Hour).UTC().Format(time.RFC3339), + ValidUntil: time.Now().Add(24 * time.Hour).UTC().Format(time.RFC3339), + } +} + +// activeRecord is a healthy participant publishing one active signing key. +func activeRecord() participant { + return participant{ + ParticipantID: testParticipantID, + Status: "active", + Node: node{ + SubscriberURL: "https://providera.example.com/onix", + Type: "BPP", + Keys: []key{signingKey()}, + }, + } +} + +// newTestClient builds a client pointed at srvURL, with retries effectively off +// and sub-millisecond backoff so tests stay fast. +func newTestClient(t *testing.T, srvURL string, cache definition.Cache, tweak ...func(*Config)) *Client { + t.Helper() + + cfg := &Config{ + URL: srvURL, + Timeout: DefaultTimeoutSeconds, + RetryMax: 0, + RetryWaitMin: time.Millisecond, + RetryWaitMax: 2 * time.Millisecond, + } + for _, apply := range tweak { + apply(cfg) + } + + client, closer, err := New(context.Background(), cache, cfg) + if err != nil { + t.Fatalf("New() returned an unexpected error: %v", err) + } + t.Cleanup(func() { _ = closer() }) + return client +} + +func lookup(t *testing.T, c *Client) ([]model.Subscription, error) { + t.Helper() + return c.Lookup(context.Background(), &model.Subscription{ + Subscriber: model.Subscriber{SubscriberID: testParticipantID}, + KeyID: testOSID, + }) +} + +// --- configuration ------------------------------------------------------- + +func TestValidate(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + config *Config + expectedErr string + }{ + { + name: "should return error for nil config", + config: nil, + expectedErr: "oan registry config cannot be nil", + }, + { + name: "should return error for empty URL", + config: &Config{URL: ""}, + expectedErr: "oan registry URL cannot be empty", + }, + { + name: "should succeed for valid config", + config: &Config{URL: "http://localhost:8081/api/v1"}, + expectedErr: "", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + err := validate(tc.config) + switch { + case tc.expectedErr == "" && err != nil: + t.Fatalf("expected no error, but got: %v", err) + case tc.expectedErr != "" && err == nil: + t.Fatalf("expected an error but got none") + case tc.expectedErr != "" && err.Error() != tc.expectedErr: + t.Errorf("expected error message %q, but got %q", tc.expectedErr, err.Error()) + } + }) + } +} + +// TestNewAlwaysBoundsTheTimeout guards the one deliberate difference from the +// sibling registry plugins. They apply the timeout only when it is configured, +// which leaves it infinite when it is not. Re-adding that guard here would look +// like harmless tidying, so it is asserted directly. +func TestNewAlwaysBoundsTheTimeout(t *testing.T) { + t.Parallel() + + client, closer, err := New(context.Background(), nil, &Config{URL: "http://localhost:8081"}) + if err != nil { + t.Fatalf("New() returned an unexpected error: %v", err) + } + defer func() { _ = closer() }() + + if got := client.client.HTTPClient.Timeout; got <= 0 { + t.Fatalf("expected a bounded timeout when none is configured, got %v", got) + } +} + +// TestNewRejectsAnInvalidConfig: a bad URL must stop the adapter at startup +// rather than failing every lookup once traffic arrives. +func TestNewRejectsAnInvalidConfig(t *testing.T) { + t.Parallel() + + for _, cfg := range []*Config{ + nil, + {URL: ""}, + {URL: "registry:8081"}, + } { + if _, _, err := New(context.Background(), nil, cfg); err == nil { + t.Errorf("expected New to reject config %+v", cfg) + } + } +} + +func TestNewBuildsSearchURL(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + config *Config + expected string + }{ + { + name: "defaults to the Participant entity", + config: &Config{URL: "http://registry:8081/api/v1"}, + expected: "http://registry:8081/api/v1/Participant/search", + }, + { + name: "honours a configured entity", + config: &Config{URL: "http://registry:8081/api/v1", Entity: "Subscriber"}, + expected: "http://registry:8081/api/v1/Subscriber/search", + }, + { + name: "tolerates a trailing slash on the base URL", + config: &Config{URL: "http://registry:8081/api/v1/"}, + expected: "http://registry:8081/api/v1/Participant/search", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + client, closer, err := New(context.Background(), nil, tc.config) + if err != nil { + t.Fatalf("New() returned an unexpected error: %v", err) + } + defer func() { _ = closer() }() + + if client.searchURL != tc.expected { + t.Errorf("expected search URL %q, got %q", tc.expected, client.searchURL) + } + }) + } +} + +// --- status mapping ------------------------------------------------------ + +// TestResolveStatus is the security regression test for this plugin. +// +// model.IsKeyStatusUsable is a deny-list, so a status it does not recognise +// counts as usable. Passing the registry's own "inactive" through unchanged +// would let a suspended participant's signature verify, which is why every case +// below asserts usability rather than just the mapped string. +func TestResolveStatus(t *testing.T) { + t.Parallel() + + now := time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC) + rfc := func(d time.Duration) string { return now.Add(d).Format(time.RFC3339) } + + // active builds a usable signing key, so each case below varies only the one + // thing it is about. + active := func(mutate ...func(*key)) key { + k := key{OSID: testOSID, Use: useSign, Value: testPublicKey, Status: "active"} + for _, apply := range mutate { + apply(&k) + } + return k + } + + testCases := []struct { + name string + participantStatus string + key key + expectedStatus string + expectedReason string + expectUsable bool + }{ + { + name: "active within window is usable", + participantStatus: "active", + key: active(func(k *key) { k.ValidFrom, k.ValidUntil = rfc(-time.Hour), rfc(time.Hour) }), + expectedStatus: statusSubscribed, + expectedReason: outcomeFound, + expectUsable: true, + }, + { + name: "active with no window bounds is usable", + participantStatus: "active", + key: active(), + expectedStatus: statusSubscribed, + expectedReason: outcomeFound, + expectUsable: true, + }, + { + name: "participant status is matched case insensitively", + participantStatus: "ACTIVE", + key: active(), + expectedStatus: statusSubscribed, + expectedReason: outcomeFound, + expectUsable: true, + }, + { + name: "key status is matched case insensitively", + participantStatus: "active", + key: active(func(k *key) { k.Status = "ACTIVE" }), + expectedStatus: statusSubscribed, + expectedReason: outcomeFound, + expectUsable: true, + }, + { + name: "the encoding label is not mistaken for key material", + participantStatus: "active", + key: active(func(k *key) { k.Value = keyEncodingPrefix + testPublicKey }), + expectedStatus: statusSubscribed, + expectedReason: outcomeFound, + expectUsable: true, + }, + { + name: "an inactive participant is denied", + participantStatus: "inactive", + key: active(), + expectedStatus: statusUnsubscribed, + expectedReason: outcomeInactive, + expectUsable: false, + }, + { + name: "an unrecognised participant status is denied", + participantStatus: "approved", + key: active(), + expectedStatus: statusUnsubscribed, + expectedReason: outcomeInactive, + expectUsable: false, + }, + { + name: "an empty participant status is denied", + participantStatus: "", + key: active(), + expectedStatus: statusUnsubscribed, + expectedReason: outcomeInactive, + expectUsable: false, + }, + { + // The reason per-key status exists: the participant is trading normally, + // one of its keys has been retired, and that key alone must stop verifying. + name: "a retired key under an active participant is denied", + participantStatus: "active", + key: active(func(k *key) { k.Status = "inactive" }), + expectedStatus: statusUnsubscribed, + expectedReason: outcomeKeyInactive, + expectUsable: false, + }, + { + name: "an unrecognised key status is denied", + participantStatus: "active", + key: active(func(k *key) { k.Status = "rotating" }), + expectedStatus: statusUnsubscribed, + expectedReason: outcomeKeyInactive, + expectUsable: false, + }, + { + name: "an empty key status is denied", + participantStatus: "active", + key: active(func(k *key) { k.Status = "" }), + expectedStatus: statusUnsubscribed, + expectedReason: outcomeKeyInactive, + expectUsable: false, + }, + { + name: "an active key with no material is denied", + participantStatus: "active", + key: active(func(k *key) { k.Value = "" }), + expectedStatus: statusUnsubscribed, + expectedReason: outcomeNoKey, + expectUsable: false, + }, + { + // A value that is nothing but the encoding label carries no material. + name: "a key that is only an encoding label is denied", + participantStatus: "active", + key: active(func(k *key) { k.Value = keyEncodingPrefix }), + expectedStatus: statusUnsubscribed, + expectedReason: outcomeNoKey, + expectUsable: false, + }, + { + // The validity window is not enforced: participation is controlled + // through `status` alone, so an expired key still verifies until the + // Network Operator deactivates it. + name: "a window that has not opened yet is NOT enforced", + participantStatus: "active", + key: active(func(k *key) { k.ValidFrom = rfc(time.Hour) }), + expectedStatus: statusSubscribed, + expectedReason: outcomeFound, + expectUsable: true, + }, + { + name: "a window that has closed is NOT enforced", + participantStatus: "active", + key: active(func(k *key) { k.ValidUntil = rfc(-time.Hour) }), + expectedStatus: statusSubscribed, + expectedReason: outcomeFound, + expectUsable: true, + }, + { + name: "an unparseable window bound is treated as unbounded", + participantStatus: "active", + key: active(func(k *key) { k.ValidUntil = "not-a-timestamp" }), + expectedStatus: statusSubscribed, + expectedReason: outcomeFound, + expectUsable: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + status, reason := resolveStatus(participant{Status: tc.participantStatus}, tc.key) + if status != tc.expectedStatus { + t.Errorf("expected status %q, got %q", tc.expectedStatus, status) + } + if reason != tc.expectedReason { + t.Errorf("expected outcome %q, got %q", tc.expectedReason, reason) + } + if usable := model.IsKeyStatusUsable(status); usable != tc.expectUsable { + t.Errorf("expected IsKeyStatusUsable to be %v for status %q, got %v", tc.expectUsable, status, usable) + } + }) + } +} + +// TestToSubscriptionMapsOptionalFields covers the registry carrying, and not +// carrying, the fields it may or may not populate. +func TestToSubscriptionMapsOptionalFields(t *testing.T) { + t.Parallel() + + t.Run("maps optional fields when present", func(t *testing.T) { + t.Parallel() + + record := activeRecord() + record.Node.Keys = append(record.Node.Keys, key{ + OSID: "1-abcdef00-0000-0000-0000-000000000000", + Use: useEncr, + Value: keyEncodingPrefix + "encryption-key", + Status: "active", + }) + + got := toSubscription(record, record.Node.Keys[0], statusSubscribed) + + if got.EncrPublicKey != "encryption-key" { + t.Errorf("expected encryption key to be mapped, got %q", got.EncrPublicKey) + } + if got.Type != "BPP" { + t.Errorf("expected type to be mapped, got %q", got.Type) + } + if got.SigningPublicKey != testPublicKey { + t.Errorf("expected the encoding label to be stripped, got %q", got.SigningPublicKey) + } + if got.ValidFrom.IsZero() || got.ValidUntil.IsZero() { + t.Error("expected the validity window to be parsed") + } + }) + + t.Run("leaves optional fields empty when absent", func(t *testing.T) { + t.Parallel() + + k := key{OSID: testOSID, Use: useSign, Value: testPublicKey, Status: "active"} + record := participant{ + ParticipantID: testParticipantID, + Node: node{Keys: []key{k}}, + } + got := toSubscription(record, k, statusSubscribed) + + // A node publishing only a signing key yields no encryption key, rather + // than falling back to the signing one. + if got.EncrPublicKey != "" { + t.Errorf("expected an empty encryption key, got %q", got.EncrPublicKey) + } + if got.Type != "" { + t.Errorf("expected an empty type, got %q", got.Type) + } + if got.SubscriberID != testParticipantID || got.KeyID != testOSID { + t.Errorf("expected identifiers to be mapped, got subscriber=%q key=%q", got.SubscriberID, got.KeyID) + } + }) + + t.Run("ignores a retired encryption key", func(t *testing.T) { + t.Parallel() + + record := activeRecord() + record.Node.Keys = append(record.Node.Keys, key{ + OSID: "1-abcdef00-0000-0000-0000-000000000000", + Use: useEncr, + Value: keyEncodingPrefix + "retired-encryption-key", + Status: "inactive", + }) + + got := toSubscription(record, record.Node.Keys[0], statusSubscribed) + + if got.EncrPublicKey != "" { + t.Errorf("a retired encryption key must not be published, got %q", got.EncrPublicKey) + } + }) +} + +// TestClassify pins the outcome vocabulary. It is a pure function, and the +// value of the split -- a dead registry and a malformed body landing in +// different series -- is entirely lost if a branch silently stops matching. +func TestClassify(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + err error + expected string + }{ + {name: "deadline exceeded", err: context.DeadlineExceeded, expected: outcomeTimeout}, + {name: "cancelled", err: context.Canceled, expected: outcomeTimeout}, + {name: "wrapped deadline", err: fmt.Errorf("sending: %w", context.DeadlineExceeded), expected: outcomeTimeout}, + {name: "registry status", err: fmt.Errorf("%w: 503", errRegistryStatus), expected: outcomeRegistryError}, + {name: "decode failure", err: fmt.Errorf("%w: bad json", errDecodeResponse), expected: outcomeDecodeError}, + {name: "anything else is transport", err: errors.New("connection refused"), expected: outcomeTransportError}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := classify(tc.err); got != tc.expected { + t.Errorf("expected outcome %q, got %q", tc.expected, got) + } + }) + } +} + +// --- lookup -------------------------------------------------------------- + +func TestLookupResolvesAnActiveParticipant(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, recordJSON(t, activeRecord())) + })) + defer srv.Close() + + results, err := lookup(t, newTestClient(t, srv.URL, nil)) + if err != nil { + t.Fatalf("Lookup() returned an unexpected error: %v", err) + } + if len(results) != 1 { + t.Fatalf("expected exactly 1 result, got %d", len(results)) + } + if results[0].SigningPublicKey != testPublicKey { + t.Errorf("expected the signing key to be returned, got %q", results[0].SigningPublicKey) + } + if !model.IsKeyStatusUsable(results[0].Status) { + t.Errorf("expected an active participant to be usable, got status %q", results[0].Status) + } +} + +// TestLookupSendsTheExpectedRequest pins the wire contract: both filters, and +// no Authorization header. The registry's search endpoint is public, and a +// malformed bearer is rejected before its permit rule is evaluated -- so +// accidentally sending one would break every lookup. +func TestLookupSendsTheExpectedRequest(t *testing.T) { + t.Parallel() + + var gotPath, gotMethod, gotAuth string + var gotBody searchRequest + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath, gotMethod, gotAuth = r.URL.Path, r.Method, r.Header.Get("Authorization") + _ = json.NewDecoder(r.Body).Decode(&gotBody) + fmt.Fprint(w, recordJSON(t, activeRecord())) + })) + defer srv.Close() + + if _, err := lookup(t, newTestClient(t, srv.URL, nil)); err != nil { + t.Fatalf("Lookup() returned an unexpected error: %v", err) + } + + if gotMethod != http.MethodPost { + t.Errorf("expected a POST, got %s", gotMethod) + } + if gotPath != "/Participant/search" { + t.Errorf("expected path /Participant/search, got %s", gotPath) + } + if gotAuth != "" { + t.Errorf("expected no Authorization header, got %q", gotAuth) + } + if got := gotBody.Filters[fieldParticipantID].Eq; got != testParticipantID { + t.Errorf("expected participant_id filter %q, got %q", testParticipantID, got) + } + // osid must NOT be filtered on: it is not an indexed field, so an + // Elasticsearch-backed registry matches nothing and every lookup becomes a + // not-found. The key identity is checked client-side instead. + if _, present := gotBody.Filters["osid"]; present { + t.Error("expected osid NOT to be sent as a filter; it is not an indexed field") + } + if len(gotBody.Filters) != 1 { + t.Errorf("expected exactly 1 filter, got %d: %v", len(gotBody.Filters), gotBody.Filters) + } +} + +// TestLookupRejectsEmptyIdentifiers: an empty key id would match any record +// whose OSID is absent, so it is refused before the registry is called. +func TestLookupRejectsEmptyIdentifiers(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + subscriberID string + keyID string + }{ + {name: "no subscriber id", subscriberID: "", keyID: testOSID}, + {name: "no key id", subscriberID: testParticipantID, keyID: ""}, + {name: "neither", subscriberID: "", keyID: ""}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + fmt.Fprint(w, recordJSON(t, activeRecord())) + })) + defer srv.Close() + + client := newTestClient(t, srv.URL, nil) + results, err := client.Lookup(context.Background(), &model.Subscription{ + Subscriber: model.Subscriber{SubscriberID: tc.subscriberID}, + KeyID: tc.keyID, + }) + if err != nil { + t.Fatalf("Lookup() returned an unexpected error: %v", err) + } + if len(results) != 0 { + t.Errorf("expected no results, got %d", len(results)) + } + if requests.Load() != 0 { + t.Error("expected the registry not to be called for an empty identifier") + } + }) + } +} + +// TestLookupWarnsOnAlgorithmMismatch: a record declaring an unexpected algorithm +// still resolves. The header's algorithm is validated upstream, so this cannot +// admit a bad signature -- it is surfaced as a warning, not a refusal. +func TestLookupWarnsOnAlgorithmMismatch(t *testing.T) { + t.Parallel() + + record := activeRecord() + record.Node.Keys[0].Algorithm = "rsa-2048" + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, recordJSON(t, record)) + })) + defer srv.Close() + + results, err := lookup(t, newTestClient(t, srv.URL, nil)) + if err != nil { + t.Fatalf("Lookup() returned an unexpected error: %v", err) + } + if len(results) != 1 || !model.IsKeyStatusUsable(results[0].Status) { + t.Fatal("expected an algorithm mismatch to warn, not to refuse the key") + } +} + +// TestLookupOnEmptyResult covers the registry answering "no such record". That +// is a legitimate answer, not a failure: the caller turns an empty slice into +// its own not-found error. +func TestLookupOnEmptyResult(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "[]") + })) + defer srv.Close() + + results, err := lookup(t, newTestClient(t, srv.URL, nil)) + if err != nil { + t.Fatalf("expected no error for an empty result, got: %v", err) + } + if len(results) != 0 { + t.Fatalf("expected no results, got %d", len(results)) + } +} + +// TestLookupAcceptsEitherResponseEnvelope: the registry answers with a bare +// array on some search backends and a {"data":[...]} envelope on others. Which +// one a deployment gets depends on its configured search provider, so both have +// to decode. +func TestLookupAcceptsEitherResponseEnvelope(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + body string + expectResults int + }{ + {name: "bare array", body: recordJSON(t, activeRecord()), expectResults: 1}, + {name: "bare empty array", body: `[]`, expectResults: 0}, + { + name: "data envelope", + body: fmt.Sprintf(`{"totalCount":1,"data":%s}`, recordJSON(t, activeRecord())), + expectResults: 1, + }, + {name: "empty data envelope", body: `{"totalCount":0,"data":[]}`, expectResults: 0}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, tc.body) + })) + defer srv.Close() + + results, err := lookup(t, newTestClient(t, srv.URL, nil)) + if err != nil { + t.Fatalf("Lookup() returned an unexpected error: %v", err) + } + if len(results) != tc.expectResults { + t.Fatalf("expected %d results, got %d", tc.expectResults, len(results)) + } + if tc.expectResults == 1 && results[0].SigningPublicKey != testPublicKey { + t.Errorf("expected the signing key to be returned, got %q", results[0].SigningPublicKey) + } + }) + } +} + +// TestLookupOnSuspendedParticipant is the end-to-end counterpart to +// TestResolveStatus: a suspended participant must come back as a refusal the +// caller can distinguish from "unknown", not as an empty result. +func TestLookupOnSuspendedParticipant(t *testing.T) { + t.Parallel() + + record := activeRecord() + record.Status = "inactive" + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, recordJSON(t, record)) + })) + defer srv.Close() + + results, err := lookup(t, newTestClient(t, srv.URL, nil)) + if err != nil { + t.Fatalf("Lookup() returned an unexpected error: %v", err) + } + if len(results) != 1 { + t.Fatalf("expected the record to be returned so the reason is reportable, got %d results", len(results)) + } + if model.IsKeyStatusUsable(results[0].Status) { + t.Fatalf("a suspended participant must not be usable, got status %q", results[0].Status) + } +} + +// TestLookupRejectsAKeyIdMismatch is the client-side replacement for the osid +// filter. osid is not an indexed field, so it cannot be filtered on server-side; +// the identity check has to happen here or a caller could present a valid +// participant id with someone else's key id. +func TestLookupRejectsAKeyIdMismatch(t *testing.T) { + t.Parallel() + + record := activeRecord() + record.Node.Keys[0].OSID = "1-99999999-0000-0000-0000-000000000000" + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, recordJSON(t, record)) + })) + defer srv.Close() + + results, err := lookup(t, newTestClient(t, srv.URL, nil)) + if err != nil { + t.Fatalf("Lookup() returned an unexpected error: %v", err) + } + if len(results) != 0 { + t.Fatalf("expected a key id mismatch to resolve to not-found, got %d results", len(results)) + } +} + +// TestSearchDistinguishesMismatchFromNotFound: both give the caller an empty +// result, but they are different facts. If the deployed key identity model is +// ever wrong, every lookup takes the mismatch path -- a total outage that would +// be invisible if it shared a metric with routine misses. +func TestSearchDistinguishesMismatchFromNotFound(t *testing.T) { + t.Parallel() + + otherKey := activeRecord() + otherKey.Node.Keys[0].OSID = "1-99999999-0000-0000-0000-000000000000" + + testCases := []struct { + name string + body string + expected string + }{ + {name: "no such participant", body: `[]`, expected: outcomeNotFound}, + {name: "participant exists, key id does not match", body: recordJSON(t, otherKey), expected: outcomeKeyIDMismatch}, + {name: "participant and key id both match", body: recordJSON(t, activeRecord()), expected: outcomeFound}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, tc.body) + })) + defer srv.Close() + + client := newTestClient(t, srv.URL, nil) + tracer := otel.Tracer("test") + + _, _, outcome, err := client.search(context.Background(), tracer, testParticipantID, testOSID) + if err != nil { + t.Fatalf("search() returned an unexpected error: %v", err) + } + if outcome != tc.expected { + t.Errorf("expected outcome %q, got %q", tc.expected, outcome) + } + }) + } +} + +// TestLookupSelectsTheRecordCarryingTheKey covers the case the osid filter was +// originally meant to guard: more than one record sharing a participant_id, e.g. +// a soft-deleted one alongside the live record. +func TestLookupSelectsTheRecordCarryingTheKey(t *testing.T) { + t.Parallel() + + stale := activeRecord() + stale.Node.Keys[0].OSID = "1-00000000-0000-0000-0000-000000000000" + stale.Node.Keys[0].Value = "stale-key" + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, recordJSON(t, stale, activeRecord())) + })) + defer srv.Close() + + results, err := lookup(t, newTestClient(t, srv.URL, nil)) + if err != nil { + t.Fatalf("Lookup() returned an unexpected error: %v", err) + } + if len(results) != 1 || results[0].SigningPublicKey != testPublicKey { + t.Fatalf("expected the record matching the requested key id to be chosen, got %+v", results) + } +} + +// TestLookupOnDuplicateRecords covers a registry integrity fault. osid is +// unique, so this cannot happen against a healthy registry -- but returning +// traffic-stopping errors on it would be worse than carrying on with the first +// record and logging loudly. +func TestLookupOnDuplicateRecords(t *testing.T) { + t.Parallel() + + first, second := activeRecord(), activeRecord() + second.Node.Keys[0].Value = "a-different-key" + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, recordJSON(t, first, second)) + })) + defer srv.Close() + + results, err := lookup(t, newTestClient(t, srv.URL, nil)) + if err != nil { + t.Fatalf("Lookup() returned an unexpected error: %v", err) + } + if len(results) != 1 { + t.Fatalf("expected exactly 1 result, got %d", len(results)) + } + if results[0].SigningPublicKey != testPublicKey { + t.Errorf("expected the first record to be used, got key %q", results[0].SigningPublicKey) + } +} + +// --- transport failures -------------------------------------------------- + +func TestLookupOnMalformedResponses(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + body string + }{ + {name: "not JSON at all", body: "this is not json"}, + {name: "an object with no data field", body: `{"participant_id":"provider-a-001"}`}, + {name: "a truncated array", body: `[{"participant_id":`}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, tc.body) + })) + defer srv.Close() + + if _, err := lookup(t, newTestClient(t, srv.URL, nil)); err == nil { + t.Fatal("expected an error for a malformed response, got none") + } + }) + } +} + +// TestLookupRetryBehaviour pins which status codes are retried. +// +// A 4xx means the request itself was wrong, so retrying it just wastes the +// caller's budget. The exception is 429, which means "too fast, try later" -- +// retryablehttp's default policy already draws exactly this line, which is why +// this plugin sets no custom retry policy. +func TestLookupRetryBehaviour(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + status int + retryMax int + expectedAttempts int32 + }{ + {name: "400 is not retried", status: http.StatusBadRequest, retryMax: 2, expectedAttempts: 1}, + {name: "404 is not retried", status: http.StatusNotFound, retryMax: 2, expectedAttempts: 1}, + {name: "429 is retried", status: http.StatusTooManyRequests, retryMax: 2, expectedAttempts: 3}, + {name: "503 is retried", status: http.StatusServiceUnavailable, retryMax: 2, expectedAttempts: 3}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + var attempts atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts.Add(1) + w.WriteHeader(tc.status) + })) + defer srv.Close() + + client := newTestClient(t, srv.URL, nil, func(c *Config) { c.RetryMax = tc.retryMax }) + if _, err := lookup(t, client); err == nil { + t.Fatal("expected an error for a failing registry, got none") + } + if got := attempts.Load(); got != tc.expectedAttempts { + t.Errorf("expected %d attempts, got %d", tc.expectedAttempts, got) + } + }) + } +} + +// TestLookupClampsRetryAfter covers a hostile or misconfigured registry. +// +// retryablehttp's DefaultBackoff honours Retry-After on 429/503 and returns it +// without applying its own ceiling, so an hour-long header would park the +// request for an hour inside signature validation. +func TestLookupClampsRetryAfter(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "3600") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer srv.Close() + + client := newTestClient(t, srv.URL, nil, func(c *Config) { + c.RetryMax = 1 + c.RetryWaitMax = 50 * time.Millisecond + }) + + start := time.Now() + if _, err := lookup(t, client); err == nil { + t.Fatal("expected an error after retries were exhausted, got none") + } + + // Generous bound: the point is that it is not honouring 3600s, not the + // precise backoff. + if elapsed := time.Since(start); elapsed > 5*time.Second { + t.Fatalf("Retry-After was not clamped: lookup took %v", elapsed) + } +} + +// TestLookupRejectsCacheEntryWithoutStatus: an empty Status is absent from +// IsKeyStatusUsable's deny-list, so a cache entry carrying one would be treated +// as verifiable. The cache is shared and outlives a deploy, so the construction +// invariant has to be re-checked on read. +func TestLookupRejectsCacheEntryWithoutStatus(t *testing.T) { + t.Parallel() + + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + fmt.Fprint(w, recordJSON(t, activeRecord())) + })) + defer srv.Close() + + poisoned, err := json.Marshal([]model.Subscription{{ + Subscriber: model.Subscriber{SubscriberID: testParticipantID}, + KeyID: testOSID, + SigningPublicKey: "attacker-supplied-key", + }}) + if err != nil { + t.Fatalf("failed to build the test cache entry: %v", err) + } + + cache := &mockCache{ + getFunc: func(ctx context.Context, key string) (string, error) { return string(poisoned), nil }, + } + client := newTestClient(t, srv.URL, cache, func(c *Config) { c.CacheTTL = 30 * time.Second }) + + results, err := lookup(t, client) + if err != nil { + t.Fatalf("Lookup() returned an unexpected error: %v", err) + } + if requests.Load() != 1 { + t.Error("expected the statusless cache entry to be discarded and the registry consulted") + } + if len(results) != 1 || results[0].SigningPublicKey != testPublicKey { + t.Fatal("expected the registry's key to be returned, not the cached one") + } +} + +// TestLookupDoesNotHangOnAStalledRegistry is the dead-peer case: the registry +// accepts the connection and then never answers. Without a bounded client +// timeout this would hang the calling request, and with it the adapter. +func TestLookupDoesNotHangOnAStalledRegistry(t *testing.T) { + t.Parallel() + + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-release + })) + defer srv.Close() + defer close(release) + + client := newTestClient(t, srv.URL, nil, func(c *Config) { c.Timeout = 1 }) + + done := make(chan error, 1) + go func() { + _, err := lookup(t, client) + done <- err + }() + + select { + case err := <-done: + if err == nil { + t.Fatal("expected a timeout error, got none") + } + // classify() must see this as a timeout, not fall through to + // transport_error. It depends on errors.Is holding through + // http.Client.Timeout -> *url.Error -> retryablehttp's wrapper, which is + // exactly the sort of chain that regresses quietly on a dependency bump. + if got := classify(err); got != outcomeTimeout { + t.Errorf("expected a stalled registry to classify as %q, got %q (%v)", outcomeTimeout, got, err) + } + case <-time.After(10 * time.Second): + t.Fatal("Lookup() did not return; the client timeout is not being applied") + } +} + +func TestLookupHonoursContextCancellation(t *testing.T) { + t.Parallel() + + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-release + })) + defer srv.Close() + defer close(release) + + // Timeout deliberately far longer than the assertion window: with the default + // 2s the client timeout fires first and this test passes even if context + // propagation is deleted, which makes it a green test protecting nothing. + client := newTestClient(t, srv.URL, nil, func(c *Config) { c.Timeout = 30 }) + ctx, cancel := context.WithCancel(context.Background()) + + done := make(chan error, 1) + go func() { + _, err := client.Lookup(ctx, &model.Subscription{ + Subscriber: model.Subscriber{SubscriberID: testParticipantID}, + KeyID: testOSID, + }) + done <- err + }() + + cancel() + + select { + case err := <-done: + if err == nil { + t.Fatal("expected an error after cancellation, got none") + } + if !errors.Is(err, context.Canceled) { + t.Errorf("expected a context.Canceled error, got: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Lookup() ignored context cancellation") + } +} + +func TestLookupOnUnreachableRegistry(t *testing.T) { + t.Parallel() + + // A server that is closed immediately, so the port refuses connections. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + url := srv.URL + srv.Close() + + if _, err := lookup(t, newTestClient(t, url, nil)); err == nil { + t.Fatal("expected an error for an unreachable registry, got none") + } +} + +// --- caching ------------------------------------------------------------- + +// TestLookupCachingDisabledByDefault matters because the TTL is exactly the +// window in which a suspended participant keeps verifying. Caching is therefore +// opt-in, and "off" must mean the cache is not touched at all. +func TestLookupCachingDisabledByDefault(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, recordJSON(t, activeRecord())) + })) + defer srv.Close() + + cache := &mockCache{} + if _, err := lookup(t, newTestClient(t, srv.URL, cache)); err != nil { + t.Fatalf("Lookup() returned an unexpected error: %v", err) + } + + if cache.getCalls != 0 || cache.setCalls != 0 { + t.Errorf("expected the cache to be untouched by default, got %d reads and %d writes", cache.getCalls, cache.setCalls) + } +} + +func TestLookupCachesUsableResults(t *testing.T) { + t.Parallel() + + const ttl = 30 * time.Second + var requests atomic.Int32 + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + fmt.Fprint(w, recordJSON(t, activeRecord())) + })) + defer srv.Close() + + cache := &mockCache{} + client := newTestClient(t, srv.URL, cache, func(c *Config) { c.CacheTTL = ttl }) + + if _, err := lookup(t, client); err != nil { + t.Fatalf("Lookup() returned an unexpected error: %v", err) + } + + if cache.setCalls != 1 { + t.Fatalf("expected exactly 1 cache write, got %d", cache.setCalls) + } + // The TTL must come from configuration, never from the record's own validity + // window -- that window is typically a year, which would keep a suspended + // participant verifying for a year. + if cache.setTTL != ttl { + t.Errorf("expected the configured TTL %v, got %v", ttl, cache.setTTL) + } + if expected := fmt.Sprintf("oan_lookup_%s_%s", testParticipantID, testOSID); cache.setKey != expected { + t.Errorf("expected cache key %q, got %q", expected, cache.setKey) + } + + // A second lookup should be served from the cache without another round trip. + cached := cache.setVal + cache.getFunc = func(ctx context.Context, key string) (string, error) { return cached, nil } + + if _, err := lookup(t, client); err != nil { + t.Fatalf("Lookup() returned an unexpected error on the cached path: %v", err) + } + if got := requests.Load(); got != 1 { + t.Errorf("expected the second lookup to be served from cache, but the registry saw %d requests", got) + } +} + +// TestLookupDoesNotCacheUnusableResults: caching a refusal would delay a +// reinstatement, and caching a miss would extend an outage. +func TestLookupDoesNotCacheUnusableResults(t *testing.T) { + t.Parallel() + + suspended := activeRecord() + suspended.Status = "inactive" + + testCases := []struct { + name string + body string + }{ + {name: "a suspended participant", body: recordJSON(t, suspended)}, + {name: "a participant that does not exist", body: "[]"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, tc.body) + })) + defer srv.Close() + + cache := &mockCache{} + client := newTestClient(t, srv.URL, cache, func(c *Config) { c.CacheTTL = 30 * time.Second }) + + if _, err := lookup(t, client); err != nil { + t.Fatalf("Lookup() returned an unexpected error: %v", err) + } + if cache.setCalls != 0 { + t.Errorf("expected nothing to be cached, got %d writes", cache.setCalls) + } + }) + } +} + +// TestLookupSurvivesCacheFailures: the cache is a performance aid, so neither an +// unreadable entry nor a failing write may break verification. +func TestLookupSurvivesCacheFailures(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, recordJSON(t, activeRecord())) + })) + defer srv.Close() + + cache := &mockCache{ + getFunc: func(ctx context.Context, key string) (string, error) { return "{not-json", nil }, + setErr: errors.New("cache is down"), + } + client := newTestClient(t, srv.URL, cache, func(c *Config) { c.CacheTTL = 30 * time.Second }) + + results, err := lookup(t, client) + if err != nil { + t.Fatalf("expected cache failures to be survivable, got: %v", err) + } + if len(results) != 1 || results[0].SigningPublicKey != testPublicKey { + t.Error("expected the lookup to fall back to the registry and return the key") + } +} + +// TestEmitMetricsSuccessPartition pins which outcomes count against +// onix_plugin_errors_total. +// +// The partition is an allow-list on purpose (see successOutcomes): an unlisted +// outcome must count as a failure, because the alternative -- a new success-like +// outcome silently falling through as success -- makes the error rate quietly +// untrue, and nothing else would catch it. +// +// This test must NOT call t.Parallel(). otel.SetMeterProvider is global and +// telemetry.GetMetrics caches instruments against the provider pointer, so +// running alongside the parallel tests would mix their measurements into this +// reader. Go never runs a non-parallel test concurrently with a parallel one, so +// the sequential phase gives this exclusive use of the global provider -- but +// that safety is invisible, hence this comment. +func TestEmitMetricsSuccessPartition(t *testing.T) { + ctx := context.Background() + + for _, tc := range []struct { + name string + outcome string + expectErrors bool + }{ + {name: "found is a success", outcome: outcomeFound, expectErrors: false}, + {name: "cache hit is a success", outcome: outcomeCacheHit, expectErrors: false}, + {name: "inactive counts as a failure", outcome: outcomeInactive, expectErrors: true}, + {name: "key id mismatch counts as a failure", outcome: outcomeKeyIDMismatch, expectErrors: true}, + {name: "an unlisted outcome counts as a failure", outcome: "some_future_outcome", expectErrors: true}, + } { + t.Run(tc.name, func(t *testing.T) { + previous := otel.GetMeterProvider() + reader := sdkmetric.NewManualReader() + mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + otel.SetMeterProvider(mp) + t.Cleanup(func() { + otel.SetMeterProvider(previous) + _ = mp.Shutdown(ctx) + }) + + (&Client{}).emitMetrics(ctx, time.Now(), operationLookup, tc.outcome) + + var rm metricdata.ResourceMetrics + if err := reader.Collect(ctx, &rm); err != nil { + t.Fatalf("failed to collect metrics: %v", err) + } + + var sawDuration, sawErrors bool + for _, scope := range rm.ScopeMetrics { + for _, m := range scope.Metrics { + switch m.Name { + case "onix_plugin_execution_duration_seconds": + sawDuration = true + assertOutcomeAttribute(t, m, tc.outcome) + case "onix_plugin_errors_total": + sawErrors = true + assertOutcomeAttribute(t, m, tc.outcome) + } + } + } + + if !sawDuration { + t.Error("expected the duration histogram to be recorded for every outcome") + } + if sawErrors != tc.expectErrors { + t.Errorf("expected errors-counter recorded=%v for outcome %q, got %v", tc.expectErrors, tc.outcome, sawErrors) + } + }) + } +} + +// assertOutcomeAttribute checks the four attributes every measurement carries. +func assertOutcomeAttribute(t *testing.T, m metricdata.Metrics, outcome string) { + t.Helper() + + want := map[string]string{ + string(telemetry.AttrPluginID): pluginID, + string(telemetry.AttrPluginType): pluginType, + string(telemetry.AttrOperation): operationLookup, + string(telemetry.AttrErrorType): outcome, + } + + var attrSets []attribute.Set + switch data := m.Data.(type) { + case metricdata.Histogram[float64]: + for _, dp := range data.DataPoints { + attrSets = append(attrSets, dp.Attributes) + } + case metricdata.Sum[int64]: + for _, dp := range data.DataPoints { + attrSets = append(attrSets, dp.Attributes) + } + default: + t.Fatalf("unexpected metric data type for %s: %T", m.Name, m.Data) + } + + if len(attrSets) == 0 { + t.Fatalf("expected at least one data point for %s", m.Name) + } + for key, expected := range want { + value, ok := attrSets[0].Value(attribute.Key(key)) + if !ok { + t.Errorf("%s: missing attribute %q", m.Name, key) + continue + } + if value.AsString() != expected { + t.Errorf("%s: attribute %q = %q, want %q", m.Name, key, value.AsString(), expected) + } + } +} + +// TestLookupAgainstCapturedRegistryResponse runs the plugin against a verbatim +// response captured from the real OAN registry on 29 Aug 2026, reformatted for +// readability with field order and values untouched. +// +// It pins the deployed shape: the data envelope, keys nested under node as an +// array, the camelCase field names, the "base64:" encoding label, and the +// "active" status vocabulary at both levels. The capture it replaces described a +// flat record with snake_case fields, and this is the test that said so. +func TestLookupAgainstCapturedRegistryResponse(t *testing.T) { + t.Parallel() + + const ( + capturedParticipantID = "oan-provider" + capturedKeyOSID = "1-e73cd04b-d992-4ecb-81e3-003f28ea36ea" + capturedNodeOSID = "1-bc829800-6acb-48ec-86cd-0f52de25abb9" + capturedParticipantOSID = "1-e1072144-938e-4ab1-87c0-efd5cc45f6e6" + capturedKey = "xq4+2oQ6MgSZdHHBMtNd1TmnPTmzY5UoZlqzf0yn6ZA=" + capturedURL = "https://provider-network-vistaar.da.gov.in/beckn" + ) + + const captured = `{ + "totalCount": 1, + "data": [ + { + "participantId": "oan-provider", + "osUpdatedAt": "2026-08-29T06:47:56.019Z", + "osCreatedAt": "2026-08-29T06:47:56.019Z", + "osUpdatedBy": "89bf9fcb-c6f7-4f08-80f9-18f47ce7667d", + "name": "OpenAgriNet provider adapter", + "osCreatedBy": "89bf9fcb-c6f7-4f08-80f9-18f47ce7667d", + "osid": "1-e1072144-938e-4ab1-87c0-efd5cc45f6e6", + "osOwner": ["89bf9fcb-c6f7-4f08-80f9-18f47ce7667d"], + "node": { + "osid": "1-bc829800-6acb-48ec-86cd-0f52de25abb9", + "osUpdatedAt": "2026-08-29T06:47:56.019Z", + "osCreatedAt": "2026-08-29T06:47:56.019Z", + "osUpdatedBy": "89bf9fcb-c6f7-4f08-80f9-18f47ce7667d", + "subscriberUrl": "https://provider-network-vistaar.da.gov.in/beckn", + "osCreatedBy": "89bf9fcb-c6f7-4f08-80f9-18f47ce7667d", + "type": "BPP", + "keys": [ + { + "osUpdatedAt": "2026-08-29T06:47:56.019Z", + "osUpdatedBy": "89bf9fcb-c6f7-4f08-80f9-18f47ce7667d", + "use": "sign", + "keyId": "k1", + "osid": "1-e73cd04b-d992-4ecb-81e3-003f28ea36ea", + "validFrom": "2026-08-01T00:00:00Z", + "osCreatedAt": "2026-08-29T06:47:56.019Z", + "osCreatedBy": "89bf9fcb-c6f7-4f08-80f9-18f47ce7667d", + "validUntil": "2026-11-01T00:00:00Z", + "alg": "ed25519", + "key": "base64:xq4+2oQ6MgSZdHHBMtNd1TmnPTmzY5UoZlqzf0yn6ZA=", + "status": "active" + } + ] + }, + "status": "active" + } + ] +}` + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, captured) + })) + defer srv.Close() + + client := newTestClient(t, srv.URL, nil) + resolve := func(keyID string) ([]model.Subscription, error) { + return client.Lookup(context.Background(), &model.Subscription{ + Subscriber: model.Subscriber{SubscriberID: capturedParticipantID}, + KeyID: keyID, + }) + } + + results, err := resolve(capturedKeyOSID) + if err != nil { + t.Fatalf("Lookup() returned an unexpected error: %v", err) + } + if len(results) != 1 { + t.Fatalf("expected exactly 1 result, got %d", len(results)) + } + + got := results[0] + if got.SigningPublicKey != capturedKey { + t.Errorf("signing key = %q, want %q (the encoding label must be stripped)", got.SigningPublicKey, capturedKey) + } + if !model.IsKeyStatusUsable(got.Status) { + t.Errorf("an active participant with an active key must be usable, got status %q", got.Status) + } + if got.SubscriberID != capturedParticipantID { + t.Errorf("subscriber id = %q, want %q", got.SubscriberID, capturedParticipantID) + } + if got.KeyID != capturedKeyOSID { + t.Errorf("key id = %q, want %q", got.KeyID, capturedKeyOSID) + } + if got.URL != capturedURL { + t.Errorf("endpoint url = %q, want the captured subscriberUrl %q", got.URL, capturedURL) + } + if got.Type != "BPP" { + t.Errorf("type = %q, want %q", got.Type, "BPP") + } + if got.EncrPublicKey != "" { + t.Errorf("this record publishes no encryption key, got %q", got.EncrPublicKey) + } + if got.ValidFrom.IsZero() || got.ValidUntil.IsZero() { + t.Error("expected the validity window to be parsed from the key's validFrom/validUntil") + } + + // The record carries three osids -- participant, node and key -- and only the + // key's identifies a signing key. Matching either of the other two would + // resolve the wrong thing, and would keep resolving it as soon as a second key + // were published. + for _, tc := range []struct{ name, keyID string }{ + {"participant osid", capturedParticipantOSID}, + {"node osid", capturedNodeOSID}, + {"an unrelated osid", "1-00000000-0000-0000-0000-000000000000"}, + } { + mismatched, err := resolve(tc.keyID) + if err != nil { + t.Fatalf("Lookup() with the %s returned an unexpected error: %v", tc.name, err) + } + if len(mismatched) != 0 { + t.Errorf("expected the %s not to resolve a signing key, got %d results", tc.name, len(mismatched)) + } + } +} + +// --- cache write and metrics edge cases ----------------------------------- + +// TestCacheResultSkipsWhenDisabled: cacheTTL of 0 means the cache is not +// touched at all, rather than written with a zero TTL whose meaning would +// depend on the cache implementation. +func TestCacheResultSkipsWhenDisabled(t *testing.T) { + t.Parallel() + + cache := &mockCache{} + c := &Client{cache: cache, cacheTTL: 0} + c.cacheResult(context.Background(), "some-key", []model.Subscription{{Status: statusSubscribed}}) + + if cache.setCalls != 0 { + t.Errorf("expected no cache write when disabled, got %d", cache.setCalls) + } +} + +// TestCacheResultSkipsWithoutACache covers the cache plugin being absent +// entirely, which is legal -- the adapter may run without one. +func TestCacheResultSkipsWithoutACache(t *testing.T) { + t.Parallel() + + // Nil cache with a positive TTL: must not panic. + c := &Client{cache: nil, cacheTTL: 30 * time.Second} + c.cacheResult(context.Background(), "some-key", []model.Subscription{{Status: statusSubscribed}}) +} + +// TestCacheResultSurvivesAFailingWrite: the cache is a performance aid, so a +// write failure is logged and swallowed rather than propagated. +func TestCacheResultSurvivesAFailingWrite(t *testing.T) { + t.Parallel() + + cache := &mockCache{setErr: errors.New("cache is down")} + c := &Client{cache: cache, cacheTTL: 30 * time.Second} + c.cacheResult(context.Background(), "some-key", []model.Subscription{{Status: statusSubscribed}}) + + if cache.setCalls != 1 { + t.Errorf("expected the write to be attempted once, got %d", cache.setCalls) + } +} + +// TestCachedSkipsWhenDisabled: with caching off the cache must not even be +// read, so a stale entry from a previous run cannot be served. +func TestCachedSkipsWhenDisabled(t *testing.T) { + t.Parallel() + + cache := &mockCache{ + getFunc: func(ctx context.Context, key string) (string, error) { + t.Error("cache was read despite being disabled") + return "", nil + }, + } + c := &Client{cache: cache, cacheTTL: 0} + + if _, ok := c.cached(context.Background(), otel.Tracer("test"), "some-key"); ok { + t.Error("expected no cache hit when caching is disabled") + } +} + +// TestParseTimeRejectsMalformedValues: a bad timestamp yields "absent" rather +// than an error, since these values are informational and must not fail a +// lookup. +func TestParseTimeRejectsMalformedValues(t *testing.T) { + t.Parallel() + + for _, value := range []string{"", "not-a-timestamp", "2026-08-19", "19/08/2026"} { + if _, ok := parseTime(value); ok { + t.Errorf("expected %q to be rejected as a timestamp", value) + } + } + if _, ok := parseTime("2026-08-19T00:00:00Z"); !ok { + t.Error("expected a valid RFC3339 timestamp to parse") + } +} + +// TestValidateRejectsAMalformedURL covers url.Parse itself failing, which it +// only does for genuinely broken input such as a control character. +func TestValidateRejectsAMalformedURL(t *testing.T) { + t.Parallel() + + if err := validate(&Config{URL: "http://registry:8081/\x7f"}); err == nil { + t.Error("expected a malformed URL to be rejected") + } + for _, u := range []string{"registry:8081", "/api/v1", "registry.example.com"} { + if err := validate(&Config{URL: u}); err == nil { + t.Errorf("expected %q to be rejected for missing scheme or host", u) + } + } +} diff --git a/pkg/plugin/implementation/oanregistry/providerrecord.go b/pkg/plugin/implementation/oanregistry/providerrecord.go new file mode 100644 index 00000000..dd237f65 --- /dev/null +++ b/pkg/plugin/implementation/oanregistry/providerrecord.go @@ -0,0 +1,419 @@ +package oanregistry + +// providerrecord.go resolves a capability binding into a call plan: what to +// call, how to call it, and which mappings translate in and out. +// +// This is the second thing the OAN registry is asked for, and it is a different +// question from the signing-key lookup in oanregistry.go. That one asks "who +// sent this", keyed by an inbound Authorization header. This one asks "who do I +// call next", keyed by a binding taken from the request body. Different subject, +// different cache, different meaning of failure -- so they share transport and +// nothing else. + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "sort" + "strings" + "time" + + "github.com/beckn-one/beckn-onix/pkg/log" + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/telemetry" + "github.com/hashicorp/go-retryablehttp" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" +) + +// Provider-record outcomes, used as the error_type metric dimension and in logs. +// Each refusal is kept distinct: they all deny the call, but "this capability was +// withdrawn" and "this provider was suspended" are different operational events +// and must not collapse into one series. +const ( + outcomeBindingNotFound = "binding_not_found" + outcomeBindingInactive = "binding_inactive" + outcomeBindingUnowned = "binding_unowned" + outcomeBindingNoActions = "binding_no_actions" + outcomeParticipantNotFound = "participant_not_found" + outcomeParticipantInactive = "participant_inactive" + outcomeNoUpstreamURL = "no_upstream_url" + outcomeNoBindingKey = "no_binding_key" +) + +// providerBinding is the subset of a capability-binding record this plugin +// reads. As with participant, the registry carries more -- Sunbird audit fields, +// the enricher name -- and none of it is modelled: the enricher is resolved by +// the provider plugin from its own code, not from the registry. +type providerBinding struct { + BindingKey string `json:"bindingKey"` + ParticipantID string `json:"participantId"` + CapabilityCode string `json:"capabilityCode"` + Actions []actionPlan `json:"actions"` + RequestMapping string `json:"requestMapping"` + ResponseMapping string `json:"responseMapping"` + Status string `json:"status"` +} + +// actionPlan is one action's upstream call, as the registry publishes it. +// +// A list rather than a map keyed by action, because the registry models nested +// collections as arrays and injects its own osid/osCreatedAt fields into every +// object it stores. A map would have to hold those alongside real actions; a +// list of structs ignores them, the same way the key list on a participant +// already does. +type actionPlan struct { + Action string `json:"action"` + Method string `json:"method"` + Path string `json:"path"` + TimeoutMs int `json:"timeoutMs"` + RetryMax int `json:"retryMax"` +} + +var ( + _ definition.RegistryLookup = (*Client)(nil) + _ definition.ProviderRecordLookup = (*Client)(nil) +) + +// searchURLFor builds the search endpoint for one registry entity. +func searchURLFor(baseURL, entity string) string { + return fmt.Sprintf("%s/%s/%s", strings.TrimSuffix(baseURL, "/"), entity, searchPath) +} + +// ProviderRecord resolves bindingKey into everything needed to call the +// provider, reading the capability binding and then the participant that owns +// it. +// +// Every way of saying "this capability cannot be served" -- absent, withdrawn, +// suspended, unroutable -- returns ErrProviderRecordNotFound, because a caller +// does the same thing with all of them. A registry that could not be consulted +// returns its own error instead: that is an outage, not an answer. +func (c *Client) ProviderRecord(ctx context.Context, bindingKey string) (*model.ProviderRecord, error) { + start := time.Now() + tracer := otel.Tracer(telemetry.ScopeName, trace.WithInstrumentationVersion(telemetry.ScopeVersion)) + ctx, span := tracer.Start(ctx, "oan registry provider record") + defer span.End() + + if bindingKey == "" { + // A caller bug rather than a registry miss: logged at Error so it is not + // mistaken for routine traffic, and refused without a round trip. + log.Errorf(ctx, nil, "OAN registry provider record requested with an empty binding key") + return nil, c.refuse(ctx, span, start, outcomeNoBindingKey) + } + + cacheKey := providerRecordCacheKey(bindingKey) + if plan, found := c.cachedProviderRecord(ctx, tracer, cacheKey); found { + log.Debugf(ctx, "OAN registry provider record cache hit for key: %s", cacheKey) + span.SetAttributes(telemetry.AttrErrorType.String(outcomeCacheHit)) + c.emitMetrics(ctx, start, operationProviderRecord, outcomeCacheHit) + return plan, nil + } + + binding, outcome, err := c.activeBinding(ctx, tracer, bindingKey) + if err != nil { + return nil, c.fail(ctx, span, start, err) + } + if outcome != outcomeFound { + return nil, c.refuse(ctx, span, start, outcome) + } + + owner, outcome, err := c.activeUpstream(ctx, tracer, binding.ParticipantID) + if err != nil { + return nil, c.fail(ctx, span, start, err) + } + if outcome != outcomeFound { + return nil, c.refuse(ctx, span, start, outcome) + } + + plan := toProviderRecord(binding, owner) + log.Debugf(ctx, "OAN registry resolved bindingKey=%s to %s serving %s", bindingKey, plan.BaseURL, strings.Join(servedActions(plan), ", ")) + c.cacheProviderRecord(ctx, cacheKey, plan) + + span.SetAttributes(telemetry.AttrErrorType.String(outcomeFound)) + c.emitMetrics(ctx, start, operationProviderRecord, outcomeFound) + return plan, nil +} + +// servedActions lists the actions a plan covers, sorted so the same record logs +// the same way twice. +func servedActions(plan *model.ProviderRecord) []string { + names := make([]string, 0, len(plan.Actions)) + for action := range plan.Actions { + names = append(names, action) + } + sort.Strings(names) + return names +} + +// refuse records a deliberate denial and returns the caller's sentinel. The +// registry answered; the answer was no. +func (c *Client) refuse(ctx context.Context, span trace.Span, start time.Time, outcome string) error { + span.SetAttributes(telemetry.AttrErrorType.String(outcome)) + c.emitMetrics(ctx, start, operationProviderRecord, outcome) + return definition.ErrProviderRecordNotFound +} + +// fail records a registry that could not be consulted at all, which is not an +// answer and must never read as one. +func (c *Client) fail(ctx context.Context, span trace.Span, start time.Time, err error) error { + outcome := classify(err) + span.RecordError(err) + span.SetStatus(codes.Error, outcome) + span.SetAttributes(telemetry.AttrErrorType.String(outcome)) + c.emitMetrics(ctx, start, operationProviderRecord, outcome) + return err +} + +// activeBinding reads the capability binding and reports whether it may be used. +func (c *Client) activeBinding(ctx context.Context, tracer trace.Tracer, bindingKey string) (providerBinding, string, error) { + bindings, err := searchRecords[providerBinding](ctx, c, tracer, c.providerSearchURL, map[string]eqFilter{ + fieldBindingKey: {Eq: bindingKey}, + }) + if err != nil { + return providerBinding{}, "", err + } + + if len(bindings) == 0 { + log.Infof(ctx, "OAN registry has no capability binding for bindingKey=%s", bindingKey) + return providerBinding{}, outcomeBindingNotFound, nil + } + if len(bindings) > 1 { + // bindingKey is the schema's unique index, so this is a registry + // integrity fault. The first row is used rather than refusing outright: + // duplicates are near-always identical, and denying would turn a registry + // hiccup into a total outage for the capability. Said loudly either way. + log.Errorf(ctx, nil, "OAN registry returned %d bindings for bindingKey=%s, expected at most 1", + len(bindings), bindingKey) + } + + binding := bindings[0] + if !isActive(binding.Status) { + log.Infof(ctx, "OAN registry capability binding bindingKey=%s is not usable: status=%q", bindingKey, binding.Status) + return providerBinding{}, outcomeBindingInactive, nil + } + if binding.ParticipantID == "" { + // Nothing to look up next, so the plan can never be completed. + log.Errorf(ctx, nil, "OAN registry capability binding bindingKey=%s names no participant", bindingKey) + return providerBinding{}, outcomeBindingUnowned, nil + } + if len(namedActions(binding)) == 0 { + // Active, owned, and callable for nothing. Refusing here says so, rather + // than letting every action fail one at a time further down. + log.Errorf(ctx, nil, "OAN registry capability binding bindingKey=%s serves no actions", bindingKey) + return providerBinding{}, outcomeBindingNoActions, nil + } + return binding, outcomeFound, nil +} + +// namedActions returns the actions a binding can actually be reached by. +func namedActions(binding providerBinding) []actionPlan { + named := make([]actionPlan, 0, len(binding.Actions)) + for _, plan := range binding.Actions { + if plan.Action != "" { + named = append(named, plan) + } + } + return named +} + +// activeUpstream reads the participant that owns a binding and reports whether +// its upstream may be called. +func (c *Client) activeUpstream(ctx context.Context, tracer trace.Tracer, participantID string) (participant, string, error) { + participants, err := searchRecords[participant](ctx, c, tracer, c.searchURL, map[string]eqFilter{ + fieldParticipantID: {Eq: participantID}, + }) + if err != nil { + return participant{}, "", err + } + + if len(participants) == 0 { + log.Errorf(ctx, nil, "OAN registry has no participant %s, named by a live capability binding", participantID) + return participant{}, outcomeParticipantNotFound, nil + } + if len(participants) > 1 { + log.Errorf(ctx, nil, "OAN registry returned %d records for participantId=%s, expected at most 1", + len(participants), participantID) + } + + owner := participants[0] + if !isActive(owner.Status) { + log.Infof(ctx, "OAN registry participantId=%s is not usable: status=%q", participantID, owner.Status) + return participant{}, outcomeParticipantInactive, nil + } + if owner.Upstream.BaseURL == "" { + // Active but unroutable. Denying here gives a clear reason rather than a + // request sent to an empty host further down. + log.Errorf(ctx, nil, "OAN registry participantId=%s publishes no upstream base url", participantID) + return participant{}, outcomeNoUpstreamURL, nil + } + return owner, outcomeFound, nil +} + +// isActive reports whether a registry status permits use. +// +// An allow-list, deliberately, and for the same reason resolveStatus is one: a +// deny-list lets every status nobody thought of through, so "withdrawn" or +// "draft" would read as callable. +func isActive(status string) bool { + return strings.EqualFold(status, statusActive) +} + +// toProviderRecord joins the two records into the plan a caller consumes. +// Mapping references are carried verbatim: they are URLs the mapper resolves, +// and this plugin does not interpret them. +func toProviderRecord(binding providerBinding, owner participant) *model.ProviderRecord { + // Keyed by action for the caller, which looks one up rather than scanning. + // An entry naming no action is skipped: it cannot be reached, and refusing + // the whole record over one malformed row would take down the actions that + // are fine. + actions := make(map[string]model.ActionPlan, len(binding.Actions)) + for _, plan := range binding.Actions { + if plan.Action == "" { + continue + } + actions[plan.Action] = model.ActionPlan{ + Method: plan.Method, + Path: plan.Path, + TimeoutMs: plan.TimeoutMs, + RetryMax: plan.RetryMax, + } + } + + return &model.ProviderRecord{ + BindingKey: binding.BindingKey, + ParticipantID: binding.ParticipantID, + CapabilityCode: binding.CapabilityCode, + BaseURL: owner.Upstream.BaseURL, + Actions: actions, + RequestMapping: binding.RequestMapping, + ResponseMapping: binding.ResponseMapping, + } +} + +// searchRecords posts a filter to one registry entity and decodes the matching +// records. It is the single transport path for both entities: they differ only +// in URL, filter and record type. +func searchRecords[T any](ctx context.Context, c *Client, tracer trace.Tracer, url string, filters map[string]eqFilter) ([]T, error) { + body, err := json.Marshal(searchRequest{Filters: filters}) + if err != nil { + return nil, fmt.Errorf("failed to marshal search request: %w", err) + } + + // No Authorization header: the registry's search endpoint is public, and + // sending a malformed or empty bearer is rejected before the endpoint's own + // permit rule is reached. + req, err := retryablehttp.NewRequest(http.MethodPost, url, bytes.NewBuffer(body)) + if err != nil { + return nil, fmt.Errorf("failed to create search request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + httpCtx, httpSpan := tracer.Start(ctx, "http search") + defer httpSpan.End() + req = req.WithContext(httpCtx) + + log.Debugf(ctx, "Making OAN registry search request to: %s", url) + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send search request: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read search response: %w", err) + } + if resp.StatusCode != http.StatusOK { + // The body can carry registry internals, so it is logged but never + // returned in the error. + log.Errorf(ctx, nil, "OAN registry search failed with status: %s, response: %s", resp.Status, string(respBody)) + return nil, fmt.Errorf("%w: %s", errRegistryStatus, resp.Status) + } + return decodeRecords[T](respBody) +} + +// decodeRecords accepts either shape the registry answers with: a bare array on +// some search backends, a data envelope on others. Depending on which one a +// deployment happens to run would be a needless coupling. +func decodeRecords[T any](body []byte) ([]T, error) { + var records []T + if err := json.Unmarshal(body, &records); err == nil { + return records, nil + } else { + // Data is a pointer so an absent "data" key is distinguishable from an + // empty one. Without that, any unrecognised JSON object -- an error body + // returned with a 200, say -- would decode to zero records and be + // reported as "no such record", hiding a real failure as a benign miss. + var envelope struct { + Data *[]T `json:"data"` + } + if envelopeErr := json.Unmarshal(body, &envelope); envelopeErr != nil || envelope.Data == nil { + // Both attempts are reported. The envelope error is the one that + // usually matters -- a registry answering with an envelope whose + // records do not fit is a schema mismatch, and reporting only the + // array failure ("cannot unmarshal object into []T") sends a reader + // looking at the wrong level entirely. + if envelopeErr != nil { + return nil, fmt.Errorf("%w: as an array: %v; as a data envelope: %v", errDecodeResponse, err, envelopeErr) + } + return nil, fmt.Errorf("%w: %v", errDecodeResponse, err) + } + return *envelope.Data, nil + } +} + +// providerRecordCacheKey namespaces plans away from signing keys. The two share +// one cache but have different subjects and lifetimes, and a collision would +// serve one as the other. +func providerRecordCacheKey(bindingKey string) string { + return "oan_provider_" + bindingKey +} + +func (c *Client) cachedProviderRecord(ctx context.Context, tracer trace.Tracer, key string) (*model.ProviderRecord, bool) { + if !c.cachingEnabled() { + return nil, false + } + + cacheCtx, span := tracer.Start(ctx, "cache lookup") + defer span.End() + + raw, err := c.cache.Get(cacheCtx, key) + if err != nil || raw == "" { + return nil, false + } + var plan model.ProviderRecord + if err := json.Unmarshal([]byte(raw), &plan); err != nil { + log.Warnf(ctx, "Discarding unreadable cache entry for key %s: %v", key, err) + return nil, false + } + // A plan with nothing to call is unusable however it got here. The cache is + // shared and outlives a deploy, so entries written by another version are + // re-checked rather than trusted. + if plan.BaseURL == "" { + log.Warnf(ctx, "Discarding malformed cache entry for key %s", key) + return nil, false + } + return &plan, true +} + +// cacheProviderRecord caches a usable plan. Refusals are never passed here: +// caching one would keep a capability dark for the whole TTL after it is +// reinstated. +func (c *Client) cacheProviderRecord(ctx context.Context, key string, plan *model.ProviderRecord) { + if !c.cachingEnabled() { + return + } + data, err := json.Marshal(plan) + if err != nil { + log.Warnf(ctx, "Failed to encode OAN registry provider record for caching, key %s: %v", key, err) + return + } + if err := c.cache.Set(ctx, key, string(data), c.cacheTTL); err != nil { + log.Warnf(ctx, "Failed to cache OAN registry provider record for key %s: %v", key, err) + } +} diff --git a/pkg/plugin/implementation/oanregistry/providerrecord_test.go b/pkg/plugin/implementation/oanregistry/providerrecord_test.go new file mode 100644 index 00000000..695513ae --- /dev/null +++ b/pkg/plugin/implementation/oanregistry/providerrecord_test.go @@ -0,0 +1,590 @@ +package oanregistry + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" +) + +const ( + testBindingKey = "mausamgram|openagrinet:WeatherObservation" + testCapabilityCode = "openagrinet:WeatherObservation" + testProviderID = "mausamgram" + testBaseURL = "https://mausamgram.imd.gov.in/nwpapi" + testRequestMapping = "https://mappings.example.com/mausamgram/select.request.yaml" + testResponseMapping = "https://mappings.example.com/mausamgram/select.response.yaml" +) + +// envelopeJSON renders a registry search response in the data-envelope form. +func envelopeJSON[T any](t *testing.T, records ...T) string { + t.Helper() + if records == nil { + records = []T{} + } + data, err := json.Marshal(struct { + Data []T `json:"data"` + }{Data: records}) + if err != nil { + t.Fatalf("failed to marshal test records: %v", err) + } + return string(data) +} + +// arrayJSON renders the same records as a bare array, the shape some search +// backends return instead of an envelope. +func arrayJSON[T any](t *testing.T, records ...T) string { + t.Helper() + if records == nil { + records = []T{} + } + data, err := json.Marshal(records) + if err != nil { + t.Fatalf("failed to marshal test records: %v", err) + } + return string(data) +} + +// upstreamRecord is an active participant publishing a callable upstream. It is +// the provider half of a Participant record: no node, and so no keys. +func upstreamRecord() participant { + return participant{ + ParticipantID: testProviderID, + Status: "active", + Upstream: upstream{BaseURL: testBaseURL}, + } +} + +// bindingRecord is a healthy ProviderSchema row for testBindingKey. +func bindingRecord() providerBinding { + return providerBinding{ + BindingKey: testBindingKey, + ParticipantID: testProviderID, + CapabilityCode: testCapabilityCode, + Actions: []actionPlan{ + {Action: "select", Method: "GET", Path: "/get-daily", TimeoutMs: 30000, RetryMax: 3}, + }, + RequestMapping: testRequestMapping, + ResponseMapping: testResponseMapping, + Status: "active", + } +} + +// bodyForPath picks the response body for an entity search path. +func bodyForPath(t *testing.T, path, bindings, participants string) string { + t.Helper() + switch { + case strings.Contains(path, "/"+DefaultProviderEntity+"/"): + return bindings + case strings.Contains(path, "/"+DefaultEntity+"/"): + return participants + default: + t.Errorf("unexpected request path %q", path) + return "" + } +} + +// newRegistryServer serves both entities from one server, routing on the path +// the client builds for each. +func newRegistryServer(t *testing.T, bindings, participants string) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, bodyForPath(t, r.URL.Path, bindings, participants)) + })) +} + +func resolvePlan(t *testing.T, c *Client) (*model.ProviderRecord, error) { + t.Helper() + return c.ProviderRecord(context.Background(), testBindingKey) +} + +// --- happy path ------------------------------------------------------------ + +func TestProviderRecordResolvesACallPlan(t *testing.T) { + t.Parallel() + + srv := newRegistryServer(t, envelopeJSON(t, bindingRecord()), envelopeJSON(t, upstreamRecord())) + defer srv.Close() + + got, err := resolvePlan(t, newTestClient(t, srv.URL, nil)) + if err != nil { + t.Fatalf("ProviderRecord() returned an unexpected error: %v", err) + } + if got == nil { + t.Fatal("expected a call plan, got nil") + } + + for _, field := range []struct{ name, got, want string }{ + {"binding key", got.BindingKey, testBindingKey}, + {"participant id", got.ParticipantID, testProviderID}, + {"capability code", got.CapabilityCode, testCapabilityCode}, + {"base url", got.BaseURL, testBaseURL}, + {"request mapping", got.RequestMapping, testRequestMapping}, + {"response mapping", got.ResponseMapping, testResponseMapping}, + } { + if field.got != field.want { + t.Errorf("%s = %q, want %q", field.name, field.got, field.want) + } + } + + call, served := got.Actions["select"] + if !served { + t.Fatalf("no call plan for select, got actions %v", got.Actions) + } + if call.Method != "GET" || call.Path != "/get-daily" { + t.Errorf("select call = %s %s, want GET /get-daily", call.Method, call.Path) + } + if call.TimeoutMs != 30000 || call.RetryMax != 3 { + t.Errorf("select budget = timeout %d retry %d, want 30000 and 3", call.TimeoutMs, call.RetryMax) + } +} + +// One capability, several actions, each with its own endpoint. This is what the +// per-action plan exists for: a confirm posting somewhere a select does not. +func TestProviderRecordResolvesAnEndpointPerAction(t *testing.T) { + t.Parallel() + + binding := bindingRecord() + binding.Actions = append(binding.Actions, + actionPlan{Action: "confirm", Method: "POST", Path: "/book", TimeoutMs: 60000, RetryMax: 1}) + + srv := newRegistryServer(t, envelopeJSON(t, binding), envelopeJSON(t, upstreamRecord())) + defer srv.Close() + + got, err := resolvePlan(t, newTestClient(t, srv.URL, nil)) + if err != nil { + t.Fatalf("ProviderRecord() returned an unexpected error: %v", err) + } + + for _, want := range []struct { + action, method, path string + }{ + {"select", "GET", "/get-daily"}, + {"confirm", "POST", "/book"}, + } { + call, served := got.Actions[want.action] + if !served { + t.Errorf("no call plan for %s", want.action) + continue + } + if call.Method != want.method || call.Path != want.path { + t.Errorf("%s call = %s %s, want %s %s", want.action, call.Method, call.Path, want.method, want.path) + } + } +} + +// The registry may omit the call budget. Zero means "the caller applies its own +// default", and must not be mistaken for "no timeout and no retries". +func TestProviderRecordLeavesAnAbsentBudgetAtZero(t *testing.T) { + t.Parallel() + + binding := bindingRecord() + binding.Actions = []actionPlan{{Action: "select", Method: "GET", Path: "/get-daily"}} + + srv := newRegistryServer(t, envelopeJSON(t, binding), envelopeJSON(t, upstreamRecord())) + defer srv.Close() + + got, err := resolvePlan(t, newTestClient(t, srv.URL, nil)) + if err != nil { + t.Fatalf("ProviderRecord() returned an unexpected error: %v", err) + } + call := got.Actions["select"] + if call.TimeoutMs != 0 || call.RetryMax != 0 { + t.Errorf("expected an absent budget to stay zero, got timeout=%d retry=%d", call.TimeoutMs, call.RetryMax) + } +} + +// The participant read is the one the binding names, not one parsed out of the +// binding key -- the registry owns that relationship, not the key format. +func TestProviderRecordReadsTheParticipantNamedByTheBinding(t *testing.T) { + t.Parallel() + + var askedFor string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/"+DefaultEntity+"/") { + var req searchRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Errorf("failed to decode participant search: %v", err) + } + askedFor = req.Filters[fieldParticipantID].Eq + fmt.Fprint(w, envelopeJSON(t, upstreamRecord())) + return + } + fmt.Fprint(w, envelopeJSON(t, bindingRecord())) + })) + defer srv.Close() + + if _, err := resolvePlan(t, newTestClient(t, srv.URL, nil)); err != nil { + t.Fatalf("ProviderRecord() returned an unexpected error: %v", err) + } + if askedFor != testProviderID { + t.Errorf("participant searched for = %q, want %q", askedFor, testProviderID) + } +} + +// --- refusals -------------------------------------------------------------- + +// Every refusal means the same thing to a caller -- this capability cannot be +// served -- so all report ErrProviderRecordNotFound rather than an error a +// caller would have to string-match. +func TestProviderRecordRefusals(t *testing.T) { + t.Parallel() + + inactiveBinding := bindingRecord() + inactiveBinding.Status = "inactive" + + unknownStatusBinding := bindingRecord() + unknownStatusBinding.Status = "draft" + + emptyStatusBinding := bindingRecord() + emptyStatusBinding.Status = "" + + noParticipantBinding := bindingRecord() + noParticipantBinding.ParticipantID = "" + + noActionsBinding := bindingRecord() + noActionsBinding.Actions = nil + + unnamedActionBinding := bindingRecord() + unnamedActionBinding.Actions = []actionPlan{{Method: "GET", Path: "/get-daily"}} + + inactiveUpstream := upstreamRecord() + inactiveUpstream.Status = "inactive" + + emptyStatusUpstream := upstreamRecord() + emptyStatusUpstream.Status = "" + + noBaseURL := upstreamRecord() + noBaseURL.Upstream.BaseURL = "" + + testCases := []struct { + name string + bindings string + participants string + }{ + {"no binding for the key", envelopeJSON[providerBinding](t), envelopeJSON(t, upstreamRecord())}, + {"an inactive binding", envelopeJSON(t, inactiveBinding), envelopeJSON(t, upstreamRecord())}, + {"a binding with an unrecognised status", envelopeJSON(t, unknownStatusBinding), envelopeJSON(t, upstreamRecord())}, + {"a binding with an empty status", envelopeJSON(t, emptyStatusBinding), envelopeJSON(t, upstreamRecord())}, + {"a binding naming no participant", envelopeJSON(t, noParticipantBinding), envelopeJSON(t, upstreamRecord())}, + {"a binding serving no action", envelopeJSON(t, noActionsBinding), envelopeJSON(t, upstreamRecord())}, + {"a binding whose only action is unnamed", envelopeJSON(t, unnamedActionBinding), envelopeJSON(t, upstreamRecord())}, + {"no participant owning the binding", envelopeJSON(t, bindingRecord()), envelopeJSON[participant](t)}, + {"an inactive participant", envelopeJSON(t, bindingRecord()), envelopeJSON(t, inactiveUpstream)}, + {"a participant with an empty status", envelopeJSON(t, bindingRecord()), envelopeJSON(t, emptyStatusUpstream)}, + {"a participant with no upstream url", envelopeJSON(t, bindingRecord()), envelopeJSON(t, noBaseURL)}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + srv := newRegistryServer(t, tc.bindings, tc.participants) + defer srv.Close() + + got, err := resolvePlan(t, newTestClient(t, srv.URL, nil)) + if !errors.Is(err, definition.ErrProviderRecordNotFound) { + t.Errorf("expected ErrProviderRecordNotFound, got %v", err) + } + if got != nil { + t.Errorf("expected no call plan alongside a refusal, got %+v", got) + } + }) + } +} + +// An empty key cannot match anything, so it is refused without troubling the +// registry -- one fewer round trip on what is a caller bug. +func TestProviderRecordRefusesAnEmptyKeyWithoutCallingTheRegistry(t *testing.T) { + t.Parallel() + + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + })) + defer srv.Close() + + got, err := newTestClient(t, srv.URL, nil).ProviderRecord(context.Background(), "") + if !errors.Is(err, definition.ErrProviderRecordNotFound) { + t.Errorf("expected ErrProviderRecordNotFound, got %v", err) + } + if got != nil { + t.Errorf("expected no call plan, got %+v", got) + } + if requests.Load() != 0 { + t.Errorf("expected the registry not to be called, got %d request(s)", requests.Load()) + } +} + +// --- failures that are not refusals ---------------------------------------- + +// A registry that could not be consulted is not a registry that answered "no". +// Collapsing the two would report an outage as a routine miss. +func TestProviderRecordDistinguishesFailureFromRefusal(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + body string + status int + }{ + {"a registry error status", "", http.StatusInternalServerError}, + {"a not-found status", "", http.StatusNotFound}, + {"an undecodable body", `{"data":`, http.StatusOK}, + {"a body that is neither array nor envelope", `"not-a-record-set"`, http.StatusOK}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if tc.status != http.StatusOK { + w.WriteHeader(tc.status) + return + } + fmt.Fprint(w, tc.body) + })) + defer srv.Close() + + got, err := resolvePlan(t, newTestClient(t, srv.URL, nil)) + if err == nil { + t.Fatal("expected an error") + } + if errors.Is(err, definition.ErrProviderRecordNotFound) { + t.Error("a registry that could not be consulted must not report not-found") + } + if got != nil { + t.Errorf("expected no call plan, got %+v", got) + } + }) + } +} + +// The binding resolves but the participant read fails: still a failure, not a +// refusal. The capability may well be fine. +func TestProviderRecordReportsAFailingParticipantRead(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/"+DefaultEntity+"/") { + w.WriteHeader(http.StatusInternalServerError) + return + } + fmt.Fprint(w, envelopeJSON(t, bindingRecord())) + })) + defer srv.Close() + + _, err := resolvePlan(t, newTestClient(t, srv.URL, nil)) + if err == nil { + t.Fatal("expected an error") + } + if errors.Is(err, definition.ErrProviderRecordNotFound) { + t.Error("a failing participant read must not report not-found") + } +} + +// --- both response envelopes ------------------------------------------------ + +func TestProviderRecordAcceptsEitherEnvelope(t *testing.T) { + t.Parallel() + + testCases := []struct{ name, bindings, participants string }{ + {"data envelope", envelopeJSON(t, bindingRecord()), envelopeJSON(t, upstreamRecord())}, + {"bare array", arrayJSON(t, bindingRecord()), arrayJSON(t, upstreamRecord())}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + srv := newRegistryServer(t, tc.bindings, tc.participants) + defer srv.Close() + + if _, err := resolvePlan(t, newTestClient(t, srv.URL, nil)); err != nil { + t.Errorf("ProviderRecord() returned an unexpected error: %v", err) + } + }) + } +} + +// --- caching ---------------------------------------------------------------- + +// storingCache round-trips what it is given, so a test can prove a second +// resolve is served without touching the registry. mockCache cannot: it is a +// spy whose Get always misses, which is right for asserting what was written +// but cannot demonstrate a hit. +type storingCache struct { + mu sync.Mutex + entries map[string]string + getCalls int + setCalls int + setKey string +} + +func newStoringCache() *storingCache { + return &storingCache{entries: make(map[string]string)} +} + +func (c *storingCache) Get(ctx context.Context, key string) (string, error) { + c.mu.Lock() + defer c.mu.Unlock() + c.getCalls++ + value, found := c.entries[key] + if !found { + return "", errors.New("cache miss") + } + return value, nil +} + +func (c *storingCache) Set(ctx context.Context, key, value string, ttl time.Duration) error { + c.mu.Lock() + defer c.mu.Unlock() + c.setCalls++ + c.setKey = key + c.entries[key] = value + return nil +} + +func (c *storingCache) Delete(ctx context.Context, key string) error { return nil } +func (c *storingCache) Clear(ctx context.Context) error { return nil } + +// countingRegistry serves both entities and counts every request, so a test can +// tell a cache hit from a second round trip. +func countingRegistry(t *testing.T, requests *atomic.Int32, bindings, participants string) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + fmt.Fprint(w, bodyForPath(t, r.URL.Path, bindings, participants)) + })) +} + +func TestProviderRecordCachesAResolvedPlan(t *testing.T) { + t.Parallel() + + var requests atomic.Int32 + srv := countingRegistry(t, &requests, envelopeJSON(t, bindingRecord()), envelopeJSON(t, upstreamRecord())) + defer srv.Close() + + client := newTestClient(t, srv.URL, newStoringCache(), func(c *Config) { c.CacheTTL = 30 * time.Second }) + + if _, err := resolvePlan(t, client); err != nil { + t.Fatalf("first ProviderRecord() returned an unexpected error: %v", err) + } + afterFirst := requests.Load() + if afterFirst == 0 { + t.Fatal("expected the first resolve to consult the registry") + } + + if _, err := resolvePlan(t, client); err != nil { + t.Fatalf("second ProviderRecord() returned an unexpected error: %v", err) + } + if requests.Load() != afterFirst { + t.Errorf("expected the second resolve to be served from cache, registry was called %d more time(s)", requests.Load()-afterFirst) + } +} + +// A refusal must never be cached: it would keep a capability dark for the whole +// TTL after the operator reinstates it. +func TestProviderRecordDoesNotCacheARefusal(t *testing.T) { + t.Parallel() + + cache := &mockCache{} + srv := newRegistryServer(t, envelopeJSON[providerBinding](t), envelopeJSON(t, upstreamRecord())) + defer srv.Close() + + client := newTestClient(t, srv.URL, cache, func(c *Config) { c.CacheTTL = 30 * time.Second }) + if _, err := resolvePlan(t, client); !errors.Is(err, definition.ErrProviderRecordNotFound) { + t.Fatalf("expected ErrProviderRecordNotFound, got %v", err) + } + if cache.setCalls != 0 { + t.Errorf("expected a refusal not to be cached, got %d write(s)", cache.setCalls) + } +} + +func TestProviderRecordSkipsTheCacheWhenDisabled(t *testing.T) { + t.Parallel() + + cache := &mockCache{} + srv := newRegistryServer(t, envelopeJSON(t, bindingRecord()), envelopeJSON(t, upstreamRecord())) + defer srv.Close() + + // No CacheTTL is set: caching is opt-in, because the TTL is exactly how long + // a withdrawn capability keeps being called. + if _, err := resolvePlan(t, newTestClient(t, srv.URL, cache)); err != nil { + t.Fatalf("ProviderRecord() returned an unexpected error: %v", err) + } + if cache.getCalls != 0 || cache.setCalls != 0 { + t.Errorf("expected the cache to be untouched, got %d read(s) and %d write(s)", cache.getCalls, cache.setCalls) + } +} + +func TestProviderRecordDiscardsAnUnreadableCacheEntry(t *testing.T) { + t.Parallel() + + var requests atomic.Int32 + srv := countingRegistry(t, &requests, envelopeJSON(t, bindingRecord()), envelopeJSON(t, upstreamRecord())) + defer srv.Close() + + cache := &mockCache{ + getFunc: func(ctx context.Context, key string) (string, error) { return "not-json", nil }, + } + client := newTestClient(t, srv.URL, cache, func(c *Config) { c.CacheTTL = 30 * time.Second }) + + got, err := resolvePlan(t, client) + if err != nil { + t.Fatalf("ProviderRecord() returned an unexpected error: %v", err) + } + if got.BaseURL != testBaseURL { + t.Errorf("expected the registry's plan to be used, got base url %q", got.BaseURL) + } + if requests.Load() == 0 { + t.Error("expected the unreadable entry to be discarded and the registry consulted") + } +} + +// Two capabilities of one provider must not share a cache entry, so the binding +// key has to appear in the key. +func TestProviderRecordCachesPerBindingKey(t *testing.T) { + t.Parallel() + + cache := &mockCache{} + srv := newRegistryServer(t, envelopeJSON(t, bindingRecord()), envelopeJSON(t, upstreamRecord())) + defer srv.Close() + + client := newTestClient(t, srv.URL, cache, func(c *Config) { c.CacheTTL = 30 * time.Second }) + if _, err := resolvePlan(t, client); err != nil { + t.Fatalf("ProviderRecord() returned an unexpected error: %v", err) + } + if !strings.Contains(cache.setKey, testBindingKey) { + t.Errorf("cache key %q does not identify the binding", cache.setKey) + } +} + +// A cached plan must not collide with a cached signing key: different subjects, +// different lifetimes, one shared cache. +func TestProviderRecordCacheKeyIsDistinctFromTheKeyLookupCacheKey(t *testing.T) { + t.Parallel() + + cache := &mockCache{} + srv := newRegistryServer(t, envelopeJSON(t, bindingRecord()), envelopeJSON(t, upstreamRecord())) + defer srv.Close() + + client := newTestClient(t, srv.URL, cache, func(c *Config) { c.CacheTTL = 30 * time.Second }) + if _, err := resolvePlan(t, client); err != nil { + t.Fatalf("ProviderRecord() returned an unexpected error: %v", err) + } + if strings.HasPrefix(cache.setKey, "oan_lookup_") { + t.Errorf("provider plan cache key %q shares the signing-key namespace", cache.setKey) + } +} From b29da0f7719ae80d5e506e6da6ae88f5391a9127 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 31 Aug 2026 11:47:50 +0530 Subject: [PATCH 027/130] feat: add the JSONata mapper plugin [OpenAgriNet/engineering-tracker#66] Providers do not speak Beckn. The old provider backend answered that with one hand-written service per provider -- around 6,900 lines across eight of them, most of it building catalog JSON field by field. This makes the translation configuration instead: a new provider ships mapping files, not another transformation routine. The plugin is domain-free by design. It knows nothing about who is calling, nothing about what a mapping says, and nothing about the payloads passing through: it is handed a reference and an input, and it fetches, compiles, caches and runs whatever is there. Anything specific to a network or a provider belongs in the caller, which is what lets one mapper serve all of them. A mapping file carries every action one capability serves, keyed by action name. Request files are keyed by the action they translate, response files by the one they produce -- so a select mapping sits under "select" and its answer under "on_select", and each file names the Beckn actions it actually deals in. One file per direction rather than per action means a transaction walking select then confirm pays one fetch, not one per step. An action may be declared with an empty value. That is a statement rather than an omission: this action needs no document built, because the caller supplies the request itself -- a provider taking two query parameters is the ordinary case, and passing already-resolved values through a fetch and a compile to arrive at the same two fields buys nothing. Declared-but-empty and absent are deliberately different, and reported differently: the first says "I serve this, build it yourself", the second says "I do not serve this at all". Collapsing them would send an empty request where a refusal was owed, answered with a 200 and the wrong data. Two things the race detector settled rather than the design: - jsonata.Expression.Evaluate MUTATES the expression it is called on, binding into its own frame, so a cached compiled expression cannot serve two requests at once. Evaluation takes a per-mapping lock rather than recompiling: measured, evaluation is ~22us against ~184us to compile, and both are dwarfed by the upstream call that follows - a compile failure is held against its own action, so a typo in confirm is no reason for select to stop being served References arrive from the registry, which makes them external input: anything that is not an http(s) URL with a host is refused, and reads are capped in both time and size. --- install/build-plugins.sh | 1 + pkg/plugin/definition/mapper.go | 45 ++ .../implementation/jsonmapper/README.md | 127 ++++ .../implementation/jsonmapper/cmd/plugin.go | 114 ++++ .../jsonmapper/cmd/plugin_test.go | 168 ++++++ .../implementation/jsonmapper/jsonmapper.go | 415 +++++++++++++ .../jsonmapper/jsonmapper_test.go | 554 ++++++++++++++++++ 7 files changed, 1424 insertions(+) create mode 100644 pkg/plugin/definition/mapper.go create mode 100644 pkg/plugin/implementation/jsonmapper/README.md create mode 100644 pkg/plugin/implementation/jsonmapper/cmd/plugin.go create mode 100644 pkg/plugin/implementation/jsonmapper/cmd/plugin_test.go create mode 100644 pkg/plugin/implementation/jsonmapper/jsonmapper.go create mode 100644 pkg/plugin/implementation/jsonmapper/jsonmapper_test.go diff --git a/install/build-plugins.sh b/install/build-plugins.sh index d205616d..14820ef9 100755 --- a/install/build-plugins.sh +++ b/install/build-plugins.sh @@ -31,6 +31,7 @@ plugins=( "registry" "dediregistry" "oanregistry" + "jsonmapper" "manifestloader" "reqpreprocessor" "otelsetup" diff --git a/pkg/plugin/definition/mapper.go b/pkg/plugin/definition/mapper.go new file mode 100644 index 00000000..e2850e96 --- /dev/null +++ b/pkg/plugin/definition/mapper.go @@ -0,0 +1,45 @@ +package definition + +import ( + "context" + "errors" +) + +// Mapper transforms a document with a mapping fetched from a reference. +// +// It exists so that translating between OAN's Beckn payloads and a provider's +// own shape is configuration rather than code: a new provider ships mapping +// files, not a new transformation routine. The mapper itself knows nothing +// about any provider, and nothing about what a mapping says -- it fetches, +// compiles and runs whatever the reference points at. +type Mapper interface { + // Transform runs the mapping at mappingRef over input and returns the + // result. + // + // action is the Beckn action of the request being served. The mapping + // reference must identify itself as being for that action, and Transform + // refuses if it does not: running a select mapping over a confirm payload + // would otherwise succeed quietly and produce nonsense. + Transform(ctx context.Context, mappingRef, action string, input any) ([]byte, error) +} + +// MapperProvider initializes a new Mapper. +type MapperProvider interface { + New(ctx context.Context, config map[string]string) (Mapper, func() error, error) +} + +// ErrNoTransform reports an action a mapping file declares but leaves empty. +// +// An empty mapping is a statement, not an omission: this action needs no +// document built for it, because the caller supplies the request itself. A +// provider taking two query parameters is the ordinary case -- the values are +// already resolved, and passing them through a fetch and a compile to arrive at +// the same two fields buys nothing. +// +// It is a sentinel rather than an empty result so that a caller which does not +// handle it fails loudly. Returning (nil, nil) would let one send an empty +// request instead, which a provider answers with a 200 and the wrong data. +// +// An action absent from the file is a different thing entirely: that capability +// does not serve it, and Transform refuses. +var ErrNoTransform = errors.New("mapping declares this action but supplies no transform") diff --git a/pkg/plugin/implementation/jsonmapper/README.md b/pkg/plugin/implementation/jsonmapper/README.md new file mode 100644 index 00000000..ccbdc1d2 --- /dev/null +++ b/pkg/plugin/implementation/jsonmapper/README.md @@ -0,0 +1,127 @@ +# JSON Mapper Plugin + +A **mapper plugin** for Beckn-ONIX that transforms one JSON document into another +using a JSONata mapping fetched at runtime. + +## Overview + +Implements `definition.Mapper`. Given a mapping reference and an input, it +fetches, compiles, caches and runs whatever is there. + +It is domain-free by design: it knows nothing about who is calling, nothing about +what a mapping says, and nothing about the payloads passing through. Anything +specific to a network or a provider belongs in the caller, which is what lets one +mapper serve all of them. + +Its first caller is the OAN provider flow, where it translates between Beckn +payloads and each provider's own request and response shapes -- so adding a +provider is two mapping files and a registry row rather than another +transformation routine. + +It is **not** a pipeline step. A provider plugin holds it and calls it twice -- +once to build the upstream request, once to turn the answer back into Beckn. +That is what lets one plugin own the whole exchange while the translation stays +generic. + +## Mapping files + +A mapping file carries every action one capability serves, keyed by action name: + +```yaml +actions: + select: | + { + "lat": _local.lat, + "lon": _local.lon + } + confirm: | + { + "booking_id": beckn.message.contract.commitments[0].id + } +``` + +**Request files are keyed by the action they translate** (`select`); **response +files by the action they produce** (`on_select`). Each file therefore names the +Beckn actions it actually deals in, and the filename carries no meaning — naming +a file after one action while it serves several would be worse than not naming +it. + +One file per direction rather than per action means a transaction walking +`select` then `confirm` pays one fetch, not one per step. An action the file does +not declare is refused, and the error names the ones it does serve. + +A mapping that fails to compile takes down only its own action: a typo in +`confirm` is no reason for `select` to stop being served. + +References come from the registry (`requestMapping` / `responseMapping` on a +capability binding) and are fully-qualified `http`/`https` URLs. Anything else -- +a bare path, a `file://`, a URL with no host -- is refused: references are +external input, and an unchecked one would let a registry record name a local +file and have the adapter read it. + +## What a mapping can read + +| key | request leg | response leg | +|---|---|---| +| `beckn` | the inbound Beckn payload | the inbound Beckn payload | +| `_local` | values the provider plugin resolved | the same values | +| `response` | — | the provider's raw answer | + +`_local` stays in scope on the response leg on purpose. A provider's answer +rarely repeats what it was asked, so values resolved before the call are often +the only source for them in the output — the coordinates of a forecast, say. + +## Why the action is a key, not a convention + +Nothing else in the pipeline knows the action. A binding key is +`participantId|capabilityCode` and carries none, so without this a capability +publishing one mapping would run it for every action that reached it — a +`confirm` served by a `select` mapping, succeeding quietly and producing +nonsense. + +Making the action a key in the file rather than a part of its name means the file +states which actions it serves, instead of a convention someone has to remember. + +## Configuration + +```yaml +mapper: + id: jsonmapper + config: + fetchTimeout: 5s + cacheTTL: 1h + negativeTTL: 1m + maxMappingBytes: "262144" + maxCacheEntries: "200" +``` + +Every setting is optional; the defaults above are what the plugin applies. + +`negativeTTL` is how long a failed fetch is remembered. Without it a broken +reference turns every inbound request into an outbound one. + +`maxMappingBytes` caps what is read from a mapping host. References come from the +registry, so an unbounded read is an unbounded allocation driven by whoever can +write a registry record. + +## Caching, and why evaluation takes a lock + +A compiled expression is code, not data, so it cannot live in the shared Redis +cache. It is held in memory, keyed by reference, bounded by `maxCacheEntries`. + +`jsonata.Expression.Evaluate` **mutates the expression it is called on** — it +binds into the expression's own frame — so one compiled mapping cannot serve two +requests at once. Confirmed with the race detector, not assumed. + +Evaluation therefore takes a per-mapping lock. That is the cheaper trade by a +wide margin: evaluation is ~22µs against ~184µs to compile, and both are dwarfed +by the upstream call the mapped request goes on to make. Different mappings still +run in parallel. A pool of compiled expressions would remove even that, and is +the upgrade if one mapping ever becomes hot enough to matter. + +## Failure + +A mapping that cannot be fetched, parsed or compiled is an operator or registry +fault and surfaces as a plain error. A mapping that ran but could not be applied +is the payload's shape being wrong, and surfaces as a `SCH_SCHEMA_ADAPTATION_FAILED` +bad request. diff --git a/pkg/plugin/implementation/jsonmapper/cmd/plugin.go b/pkg/plugin/implementation/jsonmapper/cmd/plugin.go new file mode 100644 index 00000000..dda9de19 --- /dev/null +++ b/pkg/plugin/implementation/jsonmapper/cmd/plugin.go @@ -0,0 +1,114 @@ +package main + +import ( + "context" + "errors" + "fmt" + "strconv" + "time" + + "github.com/beckn-one/beckn-onix/pkg/log" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" +) + +// jsonMapperProvider implements definition.MapperProvider. +type jsonMapperProvider struct{} + +// newMapperFunc creates a new mapper. Indirected for tests. +var newMapperFunc = jsonmapper.New + +// parseConfig turns the plugin config map into a typed Config. Anything absent +// is left zero: jsonmapper.New applies the defaults, so they live in one place. +func (o jsonMapperProvider) parseConfig(config map[string]string) (*jsonmapper.Config, error) { + cfg := &jsonmapper.Config{} + + if err := parseDuration(config, "fetchTimeout", &cfg.FetchTimeout); err != nil { + return nil, err + } + if err := parseDuration(config, "cacheTTL", &cfg.CacheTTL); err != nil { + return nil, err + } + if err := parseDuration(config, "negativeTTL", &cfg.NegativeTTL); err != nil { + return nil, err + } + if err := parseInt64(config, "maxMappingBytes", &cfg.MaxMappingBytes); err != nil { + return nil, err + } + + if raw, exists := config["maxCacheEntries"]; exists && raw != "" { + value, err := strconv.Atoi(raw) + if err != nil { + return nil, fmt.Errorf("invalid maxCacheEntries value '%s': %w", raw, err) + } + if value <= 0 { + return nil, fmt.Errorf("maxCacheEntries must be positive, got %d", value) + } + cfg.MaxCacheEntries = value + } + + return cfg, nil +} + +// parseDuration reads an optional duration setting into target. +func parseDuration(config map[string]string, key string, target *time.Duration) error { + raw, exists := config[key] + if !exists || raw == "" { + return nil + } + value, err := time.ParseDuration(raw) + if err != nil { + return fmt.Errorf("invalid %s value '%s': %w", key, raw, err) + } + if value <= 0 { + return fmt.Errorf("%s must be positive, got %v", key, value) + } + *target = value + return nil +} + +// parseInt64 reads an optional byte-count setting into target. +func parseInt64(config map[string]string, key string, target *int64) error { + raw, exists := config[key] + if !exists || raw == "" { + return nil + } + value, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + return fmt.Errorf("invalid %s value '%s': %w", key, raw, err) + } + if value <= 0 { + return fmt.Errorf("%s must be positive, got %d", key, value) + } + *target = value + return nil +} + +// New creates a new JSON mapper plugin instance. +func (o jsonMapperProvider) New(ctx context.Context, config map[string]string) (definition.Mapper, func() error, error) { + if ctx == nil { + return nil, nil, errors.New("context cannot be nil") + } + + cfg, err := o.parseConfig(config) + if err != nil { + log.Errorf(ctx, err, "Failed to parse JSON mapper configuration") + return nil, nil, fmt.Errorf("failed to parse oan mapper configuration: %w", err) + } + + mapper, closer, err := newMapperFunc(ctx, cfg) + if err != nil { + log.Errorf(ctx, err, "Failed to create JSON mapper instance") + return nil, nil, err + } + + log.Infof(ctx, "JSON mapper instance created successfully") + return mapper, closer, nil +} + +// Provider is the exported plugin instance. +var Provider = jsonMapperProvider{} + +// Compile-time proof the provider satisfies the interface the manager asserts +// against. A mismatch is otherwise a runtime cast failure at startup. +var _ definition.MapperProvider = Provider diff --git a/pkg/plugin/implementation/jsonmapper/cmd/plugin_test.go b/pkg/plugin/implementation/jsonmapper/cmd/plugin_test.go new file mode 100644 index 00000000..3b979202 --- /dev/null +++ b/pkg/plugin/implementation/jsonmapper/cmd/plugin_test.go @@ -0,0 +1,168 @@ +package main + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + "time" + + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" +) + +func TestParseConfig(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + config map[string]string + expected *jsonmapper.Config + expectedErr string + }{ + { + // Everything absent is left zero on purpose: jsonmapper.New applies + // the defaults, so they are defined in exactly one place. + name: "leaves everything unset for New to default", + config: map[string]string{}, + expected: &jsonmapper.Config{}, + }, + { + name: "reads every supported setting", + config: map[string]string{ + "fetchTimeout": "3s", + "cacheTTL": "30m", + "negativeTTL": "45s", + "maxMappingBytes": "1024", + "maxCacheEntries": "50", + }, + expected: &jsonmapper.Config{ + FetchTimeout: 3 * time.Second, + CacheTTL: 30 * time.Minute, + NegativeTTL: 45 * time.Second, + MaxMappingBytes: 1024, + MaxCacheEntries: 50, + }, + }, + { + name: "ignores empty values", + config: map[string]string{"fetchTimeout": "", "maxCacheEntries": ""}, + expected: &jsonmapper.Config{}, + }, + { + name: "rejects a malformed duration", + config: map[string]string{"fetchTimeout": "soon"}, + expectedErr: "invalid fetchTimeout value 'soon'", + }, + { + // Zero would mean "no timeout", which is the opposite of what an + // operator writing 0 expects. + name: "rejects a non-positive duration", + config: map[string]string{"fetchTimeout": "0s"}, + expectedErr: "fetchTimeout must be positive", + }, + { + name: "rejects a malformed byte cap", + config: map[string]string{"maxMappingBytes": "lots"}, + expectedErr: "invalid maxMappingBytes value 'lots'", + }, + { + name: "rejects a non-positive byte cap", + config: map[string]string{"maxMappingBytes": "0"}, + expectedErr: "maxMappingBytes must be positive", + }, + { + name: "rejects a malformed cache size", + config: map[string]string{"maxCacheEntries": "many"}, + expectedErr: "invalid maxCacheEntries value 'many'", + }, + { + name: "rejects a non-positive cache size", + config: map[string]string{"maxCacheEntries": "0"}, + expectedErr: "maxCacheEntries must be positive", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got, err := jsonMapperProvider{}.parseConfig(tc.config) + + if tc.expectedErr != "" { + if err == nil { + t.Fatalf("expected error %q but got none", tc.expectedErr) + } + if !strings.Contains(err.Error(), tc.expectedErr) { + t.Errorf("expected error containing %q, got %q", tc.expectedErr, err.Error()) + } + return + } + + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !reflect.DeepEqual(got, tc.expected) { + t.Errorf("expected config %+v, got %+v", tc.expected, got) + } + }) + } +} + +func TestNew(t *testing.T) { + t.Parallel() + + t.Run("rejects a nil context", func(t *testing.T) { + t.Parallel() + + //nolint:staticcheck // deliberately passing a nil context to assert the guard. + _, _, err := jsonMapperProvider{}.New(nil, map[string]string{}) + if err == nil { + t.Fatal("expected an error for a nil context, got none") + } + }) + + t.Run("rejects an unparseable config", func(t *testing.T) { + t.Parallel() + + _, _, err := jsonMapperProvider{}.New(context.Background(), map[string]string{"cacheTTL": "soon"}) + if err == nil { + t.Fatal("expected an error for an invalid cacheTTL, got none") + } + }) + + t.Run("builds a mapper from an empty config", func(t *testing.T) { + t.Parallel() + + mapper, closer, err := jsonMapperProvider{}.New(context.Background(), map[string]string{}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if mapper == nil { + t.Fatal("expected a mapper, got nil") + } + if closer == nil { + t.Fatal("expected a closer, got nil") + } + if err := closer(); err != nil { + t.Errorf("expected the closer to succeed, got: %v", err) + } + }) + + // Deliberately NOT parallel: this swaps the package-level newMapperFunc, so + // running it alongside its parallel siblings would race on that variable. + t.Run("propagates a construction failure", func(t *testing.T) { + original := newMapperFunc + t.Cleanup(func() { newMapperFunc = original }) + + wantErr := errors.New("boom") + newMapperFunc = func(context.Context, *jsonmapper.Config) (*jsonmapper.Mapper, func() error, error) { + return nil, nil, wantErr + } + + _, _, err := jsonMapperProvider{}.New(context.Background(), map[string]string{}) + if !errors.Is(err, wantErr) { + t.Errorf("expected the construction error to propagate, got %v", err) + } + }) +} diff --git a/pkg/plugin/implementation/jsonmapper/jsonmapper.go b/pkg/plugin/implementation/jsonmapper/jsonmapper.go new file mode 100644 index 00000000..dfe159ac --- /dev/null +++ b/pkg/plugin/implementation/jsonmapper/jsonmapper.go @@ -0,0 +1,415 @@ +// Package jsonmapper transforms one JSON document into another using a JSONata +// mapping fetched at runtime, so that translating between two parties' shapes is +// configuration rather than code. +// +// It is domain-free by design: it knows nothing about who is calling, nothing +// about what a mapping says, and nothing about the payloads passing through. It +// is handed a reference and an input, and it fetches, compiles, caches and runs +// whatever is there. Anything specific to a network or a provider belongs in the +// caller, which is what lets one mapper serve all of them. +package jsonmapper + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "sync" + "time" + + "github.com/beckn-one/beckn-onix/pkg/log" + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/jsonata-go/jsonata" + "gopkg.in/yaml.v2" +) + +// Defaults applied when an operator leaves a setting out. +const ( + DefaultFetchTimeout = 5 * time.Second + DefaultMaxMappingBytes = 256 << 10 // 256 KiB, far above any realistic mapping + DefaultCacheTTL = time.Hour + DefaultNegativeTTL = time.Minute + DefaultMaxCacheEntries = 200 +) + +// codeAdaptationFailed reports a mapping that ran but could not produce a +// result. It is the payload's shape that is wrong, so it is a bad request +// rather than a fault of this adapter. +const codeAdaptationFailed = "SCH_SCHEMA_ADAPTATION_FAILED" + +// mappingFile is the published form of a mapping: every action one capability +// serves, in one file, keyed by action name. +// +// Request files are keyed by the action they translate ("select"); response +// files by the action they produce ("on_select"). Each file therefore names the +// Beckn actions it deals in, and the filename says nothing -- naming a file +// after one action while it serves several would be worse than not naming it. +// +// One file per direction rather than per action means a transaction walking +// select then confirm pays one fetch, not one per step. +type mappingFile struct { + Actions map[string]string `yaml:"actions"` +} + +// Config holds configuration parameters for the mapper. +type Config struct { + // FetchTimeout bounds a single mapping fetch. A mapping host that accepts + // the connection and then goes quiet must not hold a request open. + FetchTimeout time.Duration `yaml:"fetchTimeout" json:"fetchTimeout"` + + // MaxMappingBytes caps what is read from a mapping host. References come + // from the registry, so an unbounded read is an unbounded allocation driven + // by whoever can write a registry record. + MaxMappingBytes int64 `yaml:"maxMappingBytes" json:"maxMappingBytes"` + + // CacheTTL is how long a compiled mapping is reused. It is also how long a + // corrected mapping takes to take effect. + CacheTTL time.Duration `yaml:"cacheTTL" json:"cacheTTL"` + + // NegativeTTL is how long a failed fetch is remembered. Without it a broken + // reference turns every inbound request into an outbound one. + NegativeTTL time.Duration `yaml:"negativeTTL" json:"negativeTTL"` + + // MaxCacheEntries bounds the cache, which would otherwise grow with the + // number of capabilities ever seen. + MaxCacheEntries int `yaml:"maxCacheEntries" json:"maxCacheEntries"` +} + +// cacheEntry is one compiled mapping, or the failure that stopped it compiling. +// Failures are cached too, which is the whole point of the negative TTL. +// +// The mutex guards evaluation, not the entry: jsonata.Expression.Evaluate +// mutates the expression it is called on -- it binds into the expression's own +// frame -- so one compiled mapping cannot serve two requests at once. Confirmed +// with the race detector, not assumed. +// +// Evaluating under a lock rather than compiling per request is the cheaper +// trade by a wide margin: evaluation is ~22us against ~184us to compile, and +// both are dwarfed by the upstream call the mapped request goes on to make. The +// lock is per mapping, so different mappings still run in parallel. A pool of +// compiled expressions would remove even that, and is the upgrade if one +// mapping ever becomes hot enough to matter. +type cacheEntry struct { + // actions holds one compiled mapping per action the file serves. A file is + // fetched and compiled as a whole, so every action it declares is ready + // after the first request for any of them. + actions map[string]*compiledAction + // err is a failure that applies to the whole file -- it could not be + // fetched, or not parsed -- as opposed to one action failing to compile. + err error + expiresAt time.Time +} + +// compiledAction is one action's mapping, or the failure that stopped it +// compiling. Failures are held per action deliberately: a typo in confirm is no +// reason for select to stop being served. +type compiledAction struct { + expression jsonata.Expression + evaluating *sync.Mutex + err error +} + +// Mapper fetches, compiles and runs mappings. It is safe for concurrent use: +// one mapper serves every inbound request. +type Mapper struct { + config *Config + httpClient *http.Client + instance jsonata.JSONataInstance + + mu sync.RWMutex + entries map[string]cacheEntry +} + +// New creates a Mapper, applying defaults for anything left unset. +func New(ctx context.Context, cfg *Config) (*Mapper, func() error, error) { + if cfg == nil { + return nil, nil, errors.New("jsonmapper: config cannot be nil") + } + applyDefaults(cfg) + + instance, err := jsonata.OpenLatest() + if err != nil { + return nil, nil, fmt.Errorf("jsonmapper: failed to open jsonata: %w", err) + } + + mapper := &Mapper{ + config: cfg, + httpClient: &http.Client{Timeout: cfg.FetchTimeout}, + instance: instance, + entries: make(map[string]cacheEntry), + } + + closer := func() error { + log.Debugf(ctx, "Cleaning up JSON mapper resources") + mapper.httpClient.CloseIdleConnections() + return nil + } + + log.Infof(ctx, "JSON mapper created successfully") + return mapper, closer, nil +} + +// applyDefaults fills in every setting an operator left out. +func applyDefaults(cfg *Config) { + if cfg.FetchTimeout <= 0 { + cfg.FetchTimeout = DefaultFetchTimeout + } + if cfg.MaxMappingBytes <= 0 { + cfg.MaxMappingBytes = DefaultMaxMappingBytes + } + if cfg.CacheTTL <= 0 { + cfg.CacheTTL = DefaultCacheTTL + } + if cfg.NegativeTTL <= 0 { + cfg.NegativeTTL = DefaultNegativeTTL + } + if cfg.MaxCacheEntries <= 0 { + cfg.MaxCacheEntries = DefaultMaxCacheEntries + } +} + +// Transform runs the mapping at mappingRef over input. +func (m *Mapper) Transform(ctx context.Context, mappingRef, action string, input any) ([]byte, error) { + if action == "" { + return nil, fmt.Errorf("jsonmapper: cannot resolve a mapping in %q without an action", mappingRef) + } + + entry, err := m.compiled(ctx, mappingRef) + if err != nil { + return nil, err + } + + mapping, served := entry.actions[action] + if !served { + // Naming what the file does serve turns a deploy mistake into a one-line + // fix, rather than a hunt through registry rows. + return nil, fmt.Errorf("jsonmapper: mapping %q does not serve action %q; it serves %s", + mappingRef, action, strings.Join(servedActions(entry), ", ")) + } + if mapping.err != nil { + return nil, mapping.err + } + return m.evaluate(ctx, mapping, mappingRef, action, input) +} + +// servedActions lists the actions a file serves, in a stable order so the same +// mistake reads the same way twice. +func servedActions(entry cacheEntry) []string { + names := make([]string, 0, len(entry.actions)) + for name := range entry.actions { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// compiled returns the compiled mapping for a reference, fetching and compiling +// it on first use. A failure is cached too, for a shorter time. +func (m *Mapper) compiled(ctx context.Context, mappingRef string) (cacheEntry, error) { + if entry, found := m.cached(mappingRef); found { + return entry, entry.err + } + + actions, err := m.fetchAndCompile(ctx, mappingRef) + return m.remember(mappingRef, actions, err), err +} + +// cached returns a live cache entry, if there is one. +func (m *Mapper) cached(mappingRef string) (cacheEntry, bool) { + m.mu.RLock() + defer m.mu.RUnlock() + + entry, found := m.entries[mappingRef] + if !found || time.Now().After(entry.expiresAt) { + return cacheEntry{}, false + } + return entry, true +} + +// remember caches a compiled mapping, or the failure that stopped it compiling. +// A failure gets the shorter TTL: it should stop hammering a broken reference +// without outlasting the fix. +func (m *Mapper) remember(mappingRef string, actions map[string]*compiledAction, err error) cacheEntry { + ttl := m.config.CacheTTL + if err != nil { + ttl = m.config.NegativeTTL + } + entry := cacheEntry{ + actions: actions, + err: err, + expiresAt: time.Now().Add(ttl), + } + + m.mu.Lock() + defer m.mu.Unlock() + + // Bounded rather than evicting: references come from the registry, and a + // deployment serving more capabilities than the cap wants a bigger cap, not + // a cache that silently thrashes. The entry is still returned to its caller + // when it is not stored, so a request over the cap is served rather than + // refused -- it just pays to compile again next time. + if len(m.entries) >= m.config.MaxCacheEntries { + if _, replacing := m.entries[mappingRef]; !replacing { + return entry + } + } + m.entries[mappingRef] = entry + return entry +} + +// cachedCount reports how many mappings are held. Used by tests to assert the +// cache stays bounded. +func (m *Mapper) cachedCount() int { + m.mu.RLock() + defer m.mu.RUnlock() + return len(m.entries) +} + +// fetchAndCompile retrieves a mapping and turns it into a runnable expression. +func (m *Mapper) fetchAndCompile(ctx context.Context, mappingRef string) (map[string]*compiledAction, error) { + body, err := m.fetch(ctx, mappingRef) + if err != nil { + return nil, err + } + sources, err := parseActions(body) + if err != nil { + return nil, fmt.Errorf("jsonmapper: mapping %q: %w", mappingRef, err) + } + + // Every action is compiled now rather than on first use, so one fetch + // leaves the whole file ready. A compile failure is recorded against its own + // action and goes no further than that action. + actions := make(map[string]*compiledAction, len(sources)) + for action, source := range sources { + actions[action] = m.compileAction(ctx, mappingRef, action, source) + } + log.Debugf(ctx, "JSON mapper compiled %d action(s) from mapping: %s", len(actions), mappingRef) + return actions, nil +} + +// compileAction compiles one action's mapping, keeping any failure local to it. +func (m *Mapper) compileAction(ctx context.Context, mappingRef, action, source string) *compiledAction { + if strings.TrimSpace(source) == "" { + // Declared, but with nothing to build. That is a statement rather than an + // omission -- see definition.ErrNoTransform -- so it is held as this + // action's outcome and reported to whoever asks for it, while every other + // action in the file is unaffected. + return &compiledAction{err: fmt.Errorf("jsonmapper: mapping %q action %q: %w", + mappingRef, action, definition.ErrNoTransform)} + } + expression, err := m.instance.Compile(source, false) + if err != nil { + log.Errorf(ctx, err, "JSON mapper could not compile action %s of %s: %v", action, mappingRef, err) + return &compiledAction{err: fmt.Errorf("jsonmapper: mapping %q action %q failed to compile: %w", mappingRef, action, err)} + } + return &compiledAction{expression: expression, evaluating: &sync.Mutex{}} +} + +// fetch retrieves a mapping's bytes, bounded in both time and size. +func (m *Mapper) fetch(ctx context.Context, mappingRef string) ([]byte, error) { + if err := verifyFetchable(mappingRef); err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, mappingRef, nil) + if err != nil { + return nil, fmt.Errorf("jsonmapper: failed to build request for mapping %q: %w", mappingRef, err) + } + + log.Debugf(ctx, "Fetching mapping: %s", mappingRef) + resp, err := m.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("jsonmapper: failed to fetch mapping %q: %w", mappingRef, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + log.Errorf(ctx, nil, "Mapping fetch failed with status: %s, mapping: %s", resp.Status, mappingRef) + return nil, fmt.Errorf("jsonmapper: mapping %q returned %s", mappingRef, resp.Status) + } + + // LimitReader with one spare byte, so an oversized body is refused rather + // than silently truncated into a mapping that compiles to something else. + body, err := io.ReadAll(io.LimitReader(resp.Body, m.config.MaxMappingBytes+1)) + if err != nil { + return nil, fmt.Errorf("jsonmapper: failed to read mapping %q: %w", mappingRef, err) + } + if int64(len(body)) > m.config.MaxMappingBytes { + return nil, fmt.Errorf("jsonmapper: mapping %q exceeds the %d byte limit", mappingRef, m.config.MaxMappingBytes) + } + return body, nil +} + +// verifyFetchable rejects a reference this mapper will not retrieve. +// +// References come from the registry, which makes them external input: an +// unchecked one would let a registry record name a file path or an internal +// scheme and have the adapter read it. +func verifyFetchable(mappingRef string) error { + if mappingRef == "" { + return errors.New("jsonmapper: mapping reference is empty") + } + parsed, err := url.Parse(mappingRef) + if err != nil { + return fmt.Errorf("jsonmapper: invalid mapping reference %q: %w", mappingRef, err) + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return fmt.Errorf("jsonmapper: mapping reference %q must be http or https", mappingRef) + } + if parsed.Host == "" { + return fmt.Errorf("jsonmapper: mapping reference %q names no host", mappingRef) + } + return nil +} + +// parseActions reads the actions a published mapping serves. +func parseActions(body []byte) (map[string]string, error) { + var file mappingFile + if err := yaml.Unmarshal(body, &file); err != nil { + return nil, fmt.Errorf("could not be parsed: %w", err) + } + if len(file.Actions) == 0 { + return nil, errors.New("serves no actions") + } + return file.Actions, nil +} + +// marshalInput renders the named inputs a mapping reads -- beckn, _local and, +// on the response leg, response -- as the single JSON document JSONata +// evaluates against. +func marshalInput(input any) ([]byte, error) { + document, err := json.Marshal(input) + if err != nil { + return nil, fmt.Errorf("input could not be encoded: %w", err) + } + return document, nil +} + +// evaluate runs a compiled mapping over the input document. +func (m *Mapper) evaluate(ctx context.Context, mapping *compiledAction, mappingRef, action string, input any) ([]byte, error) { + document, err := marshalInput(input) + if err != nil { + return nil, fmt.Errorf("jsonmapper: mapping %q: %w", mappingRef, err) + } + + // See compiledAction: Evaluate mutates the expression, so one action's + // mapping serves one request at a time. Other actions in the same file are + // unaffected, and marshalling above is deliberately outside the lock. + mapping.evaluating.Lock() + result, err := mapping.expression.Evaluate(document, nil) + mapping.evaluating.Unlock() + if err != nil { + // The mapping is valid and the payload is not what it expected, so this + // is the caller's request being wrong rather than this adapter failing. + log.Errorf(ctx, err, "JSON mapping %s action %s failed to evaluate: %v", mappingRef, action, err) + return nil, model.NewBadReqErr(codeAdaptationFailed, + fmt.Errorf("mapping %q action %q could not be applied: %w", mappingRef, action, err)) + } + return result, nil +} diff --git a/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go b/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go new file mode 100644 index 00000000..6aaac5e6 --- /dev/null +++ b/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go @@ -0,0 +1,554 @@ +package jsonmapper + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" +) + +// twoActionMapping is the published form: one file, every action it serves. +// Request files are keyed by the action they translate; response files by the +// action they produce, which is why on_select rather than select appears there. +const twoActionMapping = `actions: + select: | + { "lat": _local.lat, "txn": beckn.context.transactionId } + confirm: | + { "booking": beckn.context.messageId } +` + +const responseMapping = `actions: + on_select: | + { "lat": _local.lat, "txn": beckn.context.transactionId, "rain": response.fcstday1.rain } +` + +func requestInput() map[string]any { + return map[string]any{ + "beckn": map[string]any{"context": map[string]any{"transactionId": "txn-123", "messageId": "msg-1"}}, + "_local": map[string]any{"lat": 19.9975, "lon": 73.7898}, + } +} + +// newMappingServer serves body at every path and counts what was asked for. +func newMappingServer(t *testing.T, body string, fetches *atomic.Int32) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if fetches != nil { + fetches.Add(1) + } + fmt.Fprint(w, body) + })) +} + +func newTestMapper(t *testing.T, tweak ...func(*Config)) *Mapper { + t.Helper() + + cfg := &Config{ + FetchTimeout: 2 * time.Second, + MaxMappingBytes: DefaultMaxMappingBytes, + CacheTTL: time.Minute, + NegativeTTL: time.Minute, + MaxCacheEntries: DefaultMaxCacheEntries, + } + for _, apply := range tweak { + apply(cfg) + } + + mapper, closer, err := New(context.Background(), cfg) + if err != nil { + t.Fatalf("New() returned an unexpected error: %v", err) + } + t.Cleanup(func() { _ = closer() }) + return mapper +} + +// ref builds a mapping reference. The filename carries no meaning -- the file's +// own keys say which actions it serves. +func ref(base string) string { return base + "/mappings/anything.yaml" } + +// --- transformation -------------------------------------------------------- + +func TestTransformRunsTheMappingForTheRequestedAction(t *testing.T) { + t.Parallel() + + srv := newMappingServer(t, twoActionMapping, nil) + defer srv.Close() + + got, err := newTestMapper(t).Transform(context.Background(), ref(srv.URL), "select", requestInput()) + if err != nil { + t.Fatalf("Transform() returned an unexpected error: %v", err) + } + + var result map[string]any + if err := json.Unmarshal(got, &result); err != nil { + t.Fatalf("failed to decode the result: %v", err) + } + if result["lat"] != 19.9975 { + t.Errorf("lat = %v, want 19.9975 -- _local was not reachable", result["lat"]) + } + if result["txn"] != "txn-123" { + t.Errorf("txn = %v, want txn-123 -- beckn was not reachable", result["txn"]) + } +} + +// One file, several actions, each reached by name. This is what the format +// exists for: adding an action is a new key, not a new file. +func TestTransformPicksTheRightActionFromOneFile(t *testing.T) { + t.Parallel() + + srv := newMappingServer(t, twoActionMapping, nil) + defer srv.Close() + mapper := newTestMapper(t) + + selected, err := mapper.Transform(context.Background(), ref(srv.URL), "select", requestInput()) + if err != nil { + t.Fatalf("select: %v", err) + } + confirmed, err := mapper.Transform(context.Background(), ref(srv.URL), "confirm", requestInput()) + if err != nil { + t.Fatalf("confirm: %v", err) + } + + if !strings.Contains(string(selected), `"lat"`) { + t.Errorf("select produced %s, want the select mapping's output", selected) + } + if !strings.Contains(string(confirmed), `"booking"`) { + t.Errorf("confirm produced %s, want the confirm mapping's output", confirmed) + } +} + +// The response leg is keyed by the action it produces, so a caller asks for +// on_select rather than select. +func TestTransformExposesTheResponseAlongsideTheRequest(t *testing.T) { + t.Parallel() + + srv := newMappingServer(t, responseMapping, nil) + defer srv.Close() + + input := requestInput() + input["response"] = map[string]any{"fcstday1": map[string]any{"rain": 12.4}} + + got, err := newTestMapper(t).Transform(context.Background(), ref(srv.URL), "on_select", input) + if err != nil { + t.Fatalf("Transform() returned an unexpected error: %v", err) + } + + var result map[string]any + if err := json.Unmarshal(got, &result); err != nil { + t.Fatalf("failed to decode the result: %v", err) + } + for _, field := range []struct { + key string + want any + }{{"lat", 19.9975}, {"txn", "txn-123"}, {"rain", 12.4}} { + if result[field.key] != field.want { + t.Errorf("%s = %v, want %v", field.key, result[field.key], field.want) + } + } +} + +// --- declared but empty ----------------------------------------------------- + +// The ordinary case for a provider taking query parameters: the action is +// declared so the file still says what the capability serves, but there is no +// document to build. +func TestTransformReportsADeclaredButEmptyAction(t *testing.T) { + t.Parallel() + + mapping := `actions: + select: "" + confirm: | + { "booking": beckn.context.messageId } +` + srv := newMappingServer(t, mapping, nil) + defer srv.Close() + mapper := newTestMapper(t) + + _, err := mapper.Transform(context.Background(), ref(srv.URL), "select", requestInput()) + if !errors.Is(err, definition.ErrNoTransform) { + t.Errorf("expected ErrNoTransform, got %v", err) + } + + // Its neighbours are unaffected: one action needing no transform says + // nothing about the rest of the file. + if _, err := mapper.Transform(context.Background(), ref(srv.URL), "confirm", requestInput()); err != nil { + t.Errorf("a sibling action must still be served: %v", err) + } +} + +// Declared-but-empty and absent are different facts and must not collapse: the +// first says "I serve this, build it yourself", the second says "I do not serve +// this at all". Confusing them would send an empty request where the answer +// should have been a refusal. +func TestTransformSeparatesAnEmptyActionFromAnAbsentOne(t *testing.T) { + t.Parallel() + + srv := newMappingServer(t, "actions:\n select: \"\"\n", nil) + defer srv.Close() + mapper := newTestMapper(t) + + if _, err := mapper.Transform(context.Background(), ref(srv.URL), "select", requestInput()); !errors.Is(err, definition.ErrNoTransform) { + t.Errorf("declared-but-empty should report ErrNoTransform, got %v", err) + } + err := func() error { + _, err := mapper.Transform(context.Background(), ref(srv.URL), "confirm", requestInput()) + return err + }() + if errors.Is(err, definition.ErrNoTransform) { + t.Error("an absent action must not report ErrNoTransform -- it is not served at all") + } + if err == nil { + t.Error("expected an absent action to be refused") + } +} + +// --- an action the file does not serve -------------------------------------- + +// A capability that publishes no mapping for an action does not serve it. The +// refusal has to be clear, because the alternative -- running whichever mapping +// happened to be there -- succeeds quietly and produces nonsense. +func TestTransformRefusesAnActionTheFileDoesNotServe(t *testing.T) { + t.Parallel() + + srv := newMappingServer(t, twoActionMapping, nil) + defer srv.Close() + + _, err := newTestMapper(t).Transform(context.Background(), ref(srv.URL), "init", requestInput()) + if err == nil { + t.Fatal("expected an unserved action to be refused") + } + if !strings.Contains(err.Error(), "init") { + t.Errorf("error %q should name the action that was asked for", err) + } + // Naming what it does serve turns a deploy mistake into a one-line fix. + if !strings.Contains(err.Error(), "select") { + t.Errorf("error %q should say which actions the mapping does serve", err) + } +} + +func TestTransformRefusesAnEmptyAction(t *testing.T) { + t.Parallel() + + srv := newMappingServer(t, twoActionMapping, nil) + defer srv.Close() + + if _, err := newTestMapper(t).Transform(context.Background(), ref(srv.URL), "", requestInput()); err == nil { + t.Fatal("expected an empty action to be refused") + } +} + +// The filename says nothing. Naming a file after one action while it serves +// several would be worse than naming it nothing at all. +func TestTransformIgnoresTheFilename(t *testing.T) { + t.Parallel() + + srv := newMappingServer(t, twoActionMapping, nil) + defer srv.Close() + mapper := newTestMapper(t) + + for _, name := range []string{"/anything.yaml", "/confirm.yaml", "/x/y/z"} { + if _, err := mapper.Transform(context.Background(), srv.URL+name, "select", requestInput()); err != nil { + t.Errorf("Transform(%q) returned an unexpected error: %v", name, err) + } + } +} + +// --- reference validation --------------------------------------------------- + +func TestTransformRefusesAnUnusableReference(t *testing.T) { + t.Parallel() + + testCases := []struct{ name, ref string }{ + {"empty", ""}, + {"a bare path with no scheme", "/mappings/select.yaml"}, + {"a file url", "file:///etc/passwd"}, + {"a scheme that is not http", "ftp://example.com/select.yaml"}, + {"no host", "http:///select.yaml"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + if _, err := newTestMapper(t).Transform(context.Background(), tc.ref, "select", requestInput()); err == nil { + t.Errorf("expected reference %q to be refused", tc.ref) + } + }) + } +} + +// --- fetch and parse failures ----------------------------------------------- + +func TestTransformReportsAFailedFetch(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + status int + body string + }{ + {name: "a not-found status", status: http.StatusNotFound}, + {name: "a server error status", status: http.StatusInternalServerError}, + {name: "malformed yaml", status: http.StatusOK, body: "actions: [unclosed"}, + {name: "no actions key", status: http.StatusOK, body: "other: value\n"}, + {name: "an empty actions map", status: http.StatusOK, body: "actions: {}\n"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if tc.status != http.StatusOK { + w.WriteHeader(tc.status) + return + } + fmt.Fprint(w, tc.body) + })) + defer srv.Close() + + if _, err := newTestMapper(t).Transform(context.Background(), ref(srv.URL), "select", requestInput()); err == nil { + t.Error("expected an error") + } + }) + } +} + +// One unusable action must not take the rest of the file down with it: a typo +// in confirm is no reason for select to stop being served. +func TestTransformIsolatesABrokenAction(t *testing.T) { + t.Parallel() + + mapping := `actions: + select: | + { "lat": _local.lat } + confirm: | + {{{ + init: "" +` + srv := newMappingServer(t, mapping, nil) + defer srv.Close() + mapper := newTestMapper(t) + + if _, err := mapper.Transform(context.Background(), ref(srv.URL), "select", requestInput()); err != nil { + t.Errorf("a healthy action must still be served: %v", err) + } + if _, err := mapper.Transform(context.Background(), ref(srv.URL), "confirm", requestInput()); err == nil { + t.Error("expected an uncompilable action to be refused") + } + // An action declared with no mapping is a statement, not a fault: the caller + // builds that request itself. It is reported as its own sentinel so a caller + // that does not handle it fails loudly rather than sending nothing. + if _, err := mapper.Transform(context.Background(), ref(srv.URL), "init", requestInput()); !errors.Is(err, definition.ErrNoTransform) { + t.Errorf("expected ErrNoTransform for a declared-but-empty action, got %v", err) + } +} + +// A mapping is fetched into memory and compiled, so an unbounded one is an +// unbounded allocation driven by whoever can write the registry record. +func TestTransformEnforcesASizeCap(t *testing.T) { + t.Parallel() + + oversized := "actions:\n select: |\n " + strings.Repeat("x", 2048) + "\n" + srv := newMappingServer(t, oversized, nil) + defer srv.Close() + + mapper := newTestMapper(t, func(c *Config) { c.MaxMappingBytes = 512 }) + if _, err := mapper.Transform(context.Background(), ref(srv.URL), "select", requestInput()); err == nil { + t.Fatal("expected an oversized mapping to be refused") + } +} + +// A mapping host that accepts the connection and then goes quiet must not hold +// a request open indefinitely. +func TestTransformBoundsTheFetch(t *testing.T) { + t.Parallel() + + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-release + })) + defer srv.Close() + defer close(release) + + mapper := newTestMapper(t, func(c *Config) { c.FetchTimeout = 50 * time.Millisecond }) + + done := make(chan error, 1) + go func() { + _, err := mapper.Transform(context.Background(), ref(srv.URL), "select", requestInput()) + done <- err + }() + + select { + case err := <-done: + if err == nil { + t.Error("expected a stalled fetch to fail") + } + case <-time.After(3 * time.Second): + t.Fatal("Transform() did not return: the fetch is unbounded") + } +} + +// --- caching ---------------------------------------------------------------- + +// Compiling is the expensive half, and it cannot be cached anywhere but in +// memory: a compiled expression is code, not data. +func TestTransformCompilesEachMappingOnce(t *testing.T) { + t.Parallel() + + var fetches atomic.Int32 + srv := newMappingServer(t, twoActionMapping, &fetches) + defer srv.Close() + + mapper := newTestMapper(t) + for i := 0; i < 3; i++ { + if _, err := mapper.Transform(context.Background(), ref(srv.URL), "select", requestInput()); err != nil { + t.Fatalf("Transform() returned an unexpected error: %v", err) + } + } + if got := fetches.Load(); got != 1 { + t.Errorf("fetched %d times, want 1 -- the mapping is being refetched per request", got) + } +} + +// One fetch serves every action in the file. This is the whole reason a file +// holds several: a transaction walking select then confirm pays one round trip, +// not one per action. +func TestTransformFetchesOnceForEveryActionInAFile(t *testing.T) { + t.Parallel() + + var fetches atomic.Int32 + srv := newMappingServer(t, twoActionMapping, &fetches) + defer srv.Close() + + mapper := newTestMapper(t) + for _, action := range []string{"select", "confirm", "select"} { + if _, err := mapper.Transform(context.Background(), ref(srv.URL), action, requestInput()); err != nil { + t.Fatalf("%s: %v", action, err) + } + } + if got := fetches.Load(); got != 1 { + t.Errorf("fetched %d times, want 1 -- a second action refetched the file", got) + } +} + +// Two references are two mappings even when they compile to the same thing. +func TestTransformCachesPerReference(t *testing.T) { + t.Parallel() + + var fetches atomic.Int32 + srv := newMappingServer(t, twoActionMapping, &fetches) + defer srv.Close() + + mapper := newTestMapper(t) + for _, r := range []string{srv.URL + "/a.yaml", srv.URL + "/b.yaml"} { + if _, err := mapper.Transform(context.Background(), r, "select", requestInput()); err != nil { + t.Fatalf("Transform() returned an unexpected error: %v", err) + } + } + if got := fetches.Load(); got != 2 { + t.Errorf("fetched %d times, want 2 -- distinct references shared a cache entry", got) + } +} + +// A reference that cannot be fetched must not be retried on every request: a +// broken mapping would otherwise turn each inbound message into an outbound one. +func TestTransformNegativeCachesAFailure(t *testing.T) { + t.Parallel() + + var fetches atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fetches.Add(1) + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + mapper := newTestMapper(t) + for i := 0; i < 3; i++ { + if _, err := mapper.Transform(context.Background(), ref(srv.URL), "select", requestInput()); err == nil { + t.Fatal("expected a missing mapping to fail") + } + } + if got := fetches.Load(); got != 1 { + t.Errorf("fetched %d times, want 1 -- a broken reference is being retried per request", got) + } +} + +// An expired entry is refetched, so a corrected mapping takes effect without a +// restart. +func TestTransformRefetchesAfterTheTTL(t *testing.T) { + t.Parallel() + + var fetches atomic.Int32 + srv := newMappingServer(t, twoActionMapping, &fetches) + defer srv.Close() + + mapper := newTestMapper(t, func(c *Config) { c.CacheTTL = 20 * time.Millisecond }) + if _, err := mapper.Transform(context.Background(), ref(srv.URL), "select", requestInput()); err != nil { + t.Fatalf("Transform() returned an unexpected error: %v", err) + } + time.Sleep(60 * time.Millisecond) + if _, err := mapper.Transform(context.Background(), ref(srv.URL), "select", requestInput()); err != nil { + t.Fatalf("Transform() returned an unexpected error: %v", err) + } + + if got := fetches.Load(); got != 2 { + t.Errorf("fetched %d times, want 2 -- an expired mapping was not refetched", got) + } +} + +// The cache is bounded: references come from the registry, so an unbounded one +// would grow with the number of capabilities ever seen. +func TestTransformBoundsTheCache(t *testing.T) { + t.Parallel() + + srv := newMappingServer(t, twoActionMapping, nil) + defer srv.Close() + + mapper := newTestMapper(t, func(c *Config) { c.MaxCacheEntries = 2 }) + for i := 0; i < 5; i++ { + if _, err := mapper.Transform(context.Background(), + fmt.Sprintf("%s/%d.yaml", srv.URL, i), "select", requestInput()); err != nil { + t.Fatalf("Transform() returned an unexpected error: %v", err) + } + } + if got := mapper.cachedCount(); got > 2 { + t.Errorf("cache holds %d entries, want at most 2", got) + } +} + +// --- concurrency ------------------------------------------------------------ + +// Every inbound request shares one mapper, so the cache is read and written +// concurrently, and jsonata.Expression.Evaluate mutates what it is called on. +// Run with -race. +func TestTransformIsSafeUnderConcurrentUse(t *testing.T) { + t.Parallel() + + srv := newMappingServer(t, twoActionMapping, nil) + defer srv.Close() + + mapper := newTestMapper(t) + actions := []string{"select", "confirm"} + errs := make(chan error, 20) + for i := 0; i < 20; i++ { + go func(i int) { + _, err := mapper.Transform(context.Background(), + fmt.Sprintf("%s/%d.yaml", srv.URL, i%3), actions[i%2], requestInput()) + errs <- err + }(i) + } + for i := 0; i < 20; i++ { + if err := <-errs; err != nil { + t.Errorf("concurrent Transform() failed: %v", err) + } + } +} From f4ef51a48534de5844b373063aa8a35a7361cb86 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 31 Aug 2026 11:48:20 +0530 Subject: [PATCH 028/130] feat: add the Mausamgram provider plugin [#1] The last piece: a step that recognises its own capability, resolves what the provider needs beyond the Beckn payload, calls it, and lets the mapper translate both ways. With the registry supplying the call plan and the mapper the translation, adding a provider is now a plugin with two short methods, two mapping files and a registry row. Dispatch turned out to need no mechanism at all. A provider step handed a request for a capability it does not serve does nothing and returns nil, so several sit in one pipeline and each recognises its own work. There is no routing table to keep in step with the registry, and no filename convention -- which matters because a binding key contains | and :, and a plugin id is its .so basename. Keying on the binding key rather than the participant is deliberate: one provider can serve several capabilities with different logic, as gfr-crop-registry and gfr-crop-recommendation did in the old backend. Three supporting pieces: - definition.ProviderStepProvider, because a provider step needs a registry and a mapper handed to it, which the plain StepProvider contract cannot do. Same shape as PolicyCheckerProvider taking a ManifestLoader - internal/oanbinding derives the binding from a payload. Shared, because the binding is a property of OAN's payloads and not of any provider. A payload naming more than one distinct provider or type is refused rather than resolved to its first: one binding key describes one upstream call, so guessing would silently serve part of the request - model.StepContext.ResponseBody, so a step that has already obtained an answer has somewhere to put it. Without it the no-route path writes a fixed ACK and ignores the body entirely, so the answer was discarded and the caller got an ACK for data it asked for synchronously That last one has four call sites, every one gated on the field being non-empty, so no existing module changes behaviour by a byte. The gate that matters least visibly is in the step instrumentor: it shallow-copies the context in but copies only named fields out, so without one line there an answer written by an instrumented step vanishes -- and instrumentation is the default path, meaning it would work unwrapped and fail wrapped. signAck signs whichever body will actually be written. Signing the generated ACK while sending an answer would put a valid signature over the wrong bytes, which is the one failure here that looks fine in testing and is rejected by every peer. Mausamgram itself is small: its prerequisite reads a point from the request, and coordinates are GeoJSON order -- [lon, lat] -- which read the other way round yields a valid request for the wrong hemisphere, so there is a test for exactly that. Auth is configured by scheme naming the ENVIRONMENT VARIABLE to read, never the credential: the secret reaches the process through its environment and nothing else, and never through the registry. A configured credential that is absent fails the request rather than calling the provider unauthenticated. Verified end to end against a live registry, mappings served over HTTP and a mock provider: a signed select in, a valid on_select out. --- config/local-beckn-one-bap.yaml | 54 ++ config/local-beckn-one-bpp.yaml | 81 +++ config/mappings/mausamgram/request.yaml | 19 + config/mappings/mausamgram/response.yaml | 92 +++ config/oan-provider-adapter.yaml | 88 +++ core/module/handler/config.go | 14 + core/module/handler/responsebody_test.go | 246 ++++++++ core/module/handler/responsestep.go | 37 +- core/module/handler/responsestep_test.go | 4 +- core/module/handler/stdHandler.go | 42 +- core/module/handler/stdHandler_test.go | 8 + core/module/handler/step_instrumentor.go | 1 + core/module/module_test.go | 8 + install/build-plugins.sh | 1 + pkg/model/model.go | 9 + pkg/plugin/definition/step.go | 18 + .../catalogpublisher/handler_test.go | 8 + .../internal/oanbinding/oanbinding.go | 109 ++++ .../internal/oanbinding/oanbinding_test.go | 159 +++++ .../implementation/mausamgram/cmd/plugin.go | 74 +++ .../mausamgram/cmd/plugin_test.go | 163 +++++ .../mausamgram/dispatch_test.go | 87 +++ .../mausamgram/mappings_test.go | 285 +++++++++ .../implementation/mausamgram/mausamgram.go | 527 ++++++++++++++++ .../mausamgram/mausamgram_test.go | 583 ++++++++++++++++++ pkg/plugin/manager.go | 41 ++ 26 files changed, 2753 insertions(+), 5 deletions(-) create mode 100644 config/mappings/mausamgram/request.yaml create mode 100644 config/mappings/mausamgram/response.yaml create mode 100644 config/oan-provider-adapter.yaml create mode 100644 core/module/handler/responsebody_test.go create mode 100644 pkg/plugin/implementation/internal/oanbinding/oanbinding.go create mode 100644 pkg/plugin/implementation/internal/oanbinding/oanbinding_test.go create mode 100644 pkg/plugin/implementation/mausamgram/cmd/plugin.go create mode 100644 pkg/plugin/implementation/mausamgram/cmd/plugin_test.go create mode 100644 pkg/plugin/implementation/mausamgram/dispatch_test.go create mode 100644 pkg/plugin/implementation/mausamgram/mappings_test.go create mode 100644 pkg/plugin/implementation/mausamgram/mausamgram.go create mode 100644 pkg/plugin/implementation/mausamgram/mausamgram_test.go diff --git a/config/local-beckn-one-bap.yaml b/config/local-beckn-one-bap.yaml index 7399cc5e..0cfcf533 100644 --- a/config/local-beckn-one-bap.yaml +++ b/config/local-beckn-one-bap.yaml @@ -55,6 +55,33 @@ modules: retry_max: 3 retry_wait_min: 100ms retry_wait_max: 500ms + + # To resolve signing keys from an OAN Registry (SunbirdRC) instead, + # swap the block above for this one. Both implement RegistryLookup, so + # nothing else in the module changes. See + # pkg/plugin/implementation/oanregistry/README.md. + # + # registry: + # id: oanregistry + # config: + # # Required, and the only setting with no default. Include the API + # # version prefix -- the plugin appends /{entity}/search. Use the + # # service name, not localhost: in a container localhost is the + # # adapter itself. + # url: http://registry:8081/api/v1 + # entity: Participant + # # Tighter than dediregistry's on purpose: this runs inside + # # signature validation on every inbound message, so + # # timeout x (retry_max + 1) is time a request waits before it can + # # even be rejected. + # timeout: 2 + # retry_max: 1 + # retry_wait_min: 100ms + # retry_wait_max: 500ms + # # Omitted means caching is off. The TTL is how long a suspended + # # participant keeps verifying, so it is opt-in. Needs a cache + # # plugin configured as well. + # # cacheTTL: 60s keyManager: id: simplekeymanager config: @@ -130,6 +157,33 @@ modules: retry_max: 3 retry_wait_min: 100ms retry_wait_max: 500ms + + # To resolve signing keys from an OAN Registry (SunbirdRC) instead, + # swap the block above for this one. Both implement RegistryLookup, so + # nothing else in the module changes. See + # pkg/plugin/implementation/oanregistry/README.md. + # + # registry: + # id: oanregistry + # config: + # # Required, and the only setting with no default. Include the API + # # version prefix -- the plugin appends /{entity}/search. Use the + # # service name, not localhost: in a container localhost is the + # # adapter itself. + # url: http://registry:8081/api/v1 + # entity: Participant + # # Tighter than dediregistry's on purpose: this runs inside + # # signature validation on every inbound message, so + # # timeout x (retry_max + 1) is time a request waits before it can + # # even be rejected. + # timeout: 2 + # retry_max: 1 + # retry_wait_min: 100ms + # retry_wait_max: 500ms + # # Omitted means caching is off. The TTL is how long a suspended + # # participant keeps verifying, so it is opt-in. Needs a cache + # # plugin configured as well. + # # cacheTTL: 60s keyManager: id: simplekeymanager config: diff --git a/config/local-beckn-one-bpp.yaml b/config/local-beckn-one-bpp.yaml index 64e94653..f7a849d2 100644 --- a/config/local-beckn-one-bpp.yaml +++ b/config/local-beckn-one-bpp.yaml @@ -79,6 +79,33 @@ modules: retry_max: 3 retry_wait_min: 100ms retry_wait_max: 500ms + + # To resolve signing keys from an OAN Registry (SunbirdRC) instead, + # swap the block above for this one. Both implement RegistryLookup, so + # nothing else in the module changes. See + # pkg/plugin/implementation/oanregistry/README.md. + # + # registry: + # id: oanregistry + # config: + # # Required, and the only setting with no default. Include the API + # # version prefix -- the plugin appends /{entity}/search. Use the + # # service name, not localhost: in a container localhost is the + # # adapter itself. + # url: http://registry:8081/api/v1 + # entity: Participant + # # Tighter than dediregistry's on purpose: this runs inside + # # signature validation on every inbound message, so + # # timeout x (retry_max + 1) is time a request waits before it can + # # even be rejected. + # timeout: 2 + # retry_max: 1 + # retry_wait_min: 100ms + # retry_wait_max: 500ms + # # Omitted means caching is off. The TTL is how long a suspended + # # participant keeps verifying, so it is opt-in. Needs a cache + # # plugin configured as well. + # # cacheTTL: 60s keyManager: id: simplekeymanager config: @@ -154,6 +181,33 @@ modules: retry_max: 3 retry_wait_min: 100ms retry_wait_max: 500ms + + # To resolve signing keys from an OAN Registry (SunbirdRC) instead, + # swap the block above for this one. Both implement RegistryLookup, so + # nothing else in the module changes. See + # pkg/plugin/implementation/oanregistry/README.md. + # + # registry: + # id: oanregistry + # config: + # # Required, and the only setting with no default. Include the API + # # version prefix -- the plugin appends /{entity}/search. Use the + # # service name, not localhost: in a container localhost is the + # # adapter itself. + # url: http://registry:8081/api/v1 + # entity: Participant + # # Tighter than dediregistry's on purpose: this runs inside + # # signature validation on every inbound message, so + # # timeout x (retry_max + 1) is time a request waits before it can + # # even be rejected. + # timeout: 2 + # retry_max: 1 + # retry_wait_min: 100ms + # retry_wait_max: 500ms + # # Omitted means caching is off. The TTL is how long a suspended + # # participant keeps verifying, so it is opt-in. Needs a cache + # # plugin configured as well. + # # cacheTTL: 60s keyManager: id: simplekeymanager config: @@ -238,6 +292,33 @@ modules: retry_max: 3 retry_wait_min: 100ms retry_wait_max: 500ms + + # To resolve signing keys from an OAN Registry (SunbirdRC) instead, + # swap the block above for this one. Both implement RegistryLookup, so + # nothing else in the module changes. See + # pkg/plugin/implementation/oanregistry/README.md. + # + # registry: + # id: oanregistry + # config: + # # Required, and the only setting with no default. Include the API + # # version prefix -- the plugin appends /{entity}/search. Use the + # # service name, not localhost: in a container localhost is the + # # adapter itself. + # url: http://registry:8081/api/v1 + # entity: Participant + # # Tighter than dediregistry's on purpose: this runs inside + # # signature validation on every inbound message, so + # # timeout x (retry_max + 1) is time a request waits before it can + # # even be rejected. + # timeout: 2 + # retry_max: 1 + # retry_wait_min: 100ms + # retry_wait_max: 500ms + # # Omitted means caching is off. The TTL is how long a suspended + # # participant keeps verifying, so it is opt-in. Needs a cache + # # plugin configured as well. + # # cacheTTL: 60s cache: id: cache config: diff --git a/config/mappings/mausamgram/request.yaml b/config/mappings/mausamgram/request.yaml new file mode 100644 index 00000000..4086949b --- /dev/null +++ b/config/mappings/mausamgram/request.yaml @@ -0,0 +1,19 @@ +# Mausamgram: OAN requests -> the provider's own. +# +# Keyed by the Beckn action each mapping translates. +# +# An action declared with no mapping needs no request document built: this +# provider takes its parameters in the query string, the step has already +# resolved them, and putting them through a transform to arrive at the same two +# fields would buy nothing. The values the step resolved become the parameters. +# +# It is still declared, so this file remains the statement of what the capability +# serves. An action absent from here is one it does not serve at all -- a +# different thing, and refused. +# +# An action that DOES need a document -- a provider wanting a body in its own +# shape -- carries its transform here, reading: +# beckn the inbound Beckn payload +# _local whatever the step resolved before the call +actions: + select: "" diff --git a/config/mappings/mausamgram/response.yaml b/config/mappings/mausamgram/response.yaml new file mode 100644 index 00000000..ef903204 --- /dev/null +++ b/config/mappings/mausamgram/response.yaml @@ -0,0 +1,92 @@ +# Mausamgram: the provider's own responses -> OAN. +# +# Keyed by the Beckn action each mapping PRODUCES, not the one that arrived: a +# select is answered by an on_select. Each file therefore names the actions it +# actually deals in. +# +# Every mapping reads: +# beckn the original request, for the context to echo and the offer to +# quote against +# _local what the provider step resolved before the call. The provider does +# not repeat it back, so the output's own coordinates have no other +# source +# response the provider's answer, in its own shape +actions: + on_select: | + ( + $lat := _local.lat; + $lon := _local.lon; + $days := [ + response.fcstday1, response.fcstday2, response.fcstday3, + response.fcstday4, response.fcstday5 + ]; + + $reading := function($name, $aggregation, $unit, $value) { + $exists($value) ? { + "parameter": $name, + "aggregation": $aggregation, + "unit": $unit, + "value": $value + } + }; + + { + "context": { + "version": beckn.context.version, + "action": "on_select", + "networkId": beckn.context.networkId, + "bapId": beckn.context.bapId, + "bapUri": beckn.context.bapUri, + "bppId": beckn.context.bppId, + "bppUri": beckn.context.bppUri, + "transactionId": beckn.context.transactionId, + "messageId": beckn.context.messageId, + "timestamp": $now() + }, + "message": { + "contract": { + "commitments": [ + { + "status": { + "descriptor": { "code": "QUOTED", "name": "Quoted" } + }, + "offer": beckn.message.contract.commitments[0].offer, + "resources": $map($days, function($day) { + { + "id": "res:mausamgram:forecast:" & $day.date, + "resourceAttributes": { + "@context": "https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld", + "@type": "openagrinet:WeatherObservation", + "informationMode": "Direct", + "observationType": "Forecast", + "subjectCategories": ["Weather"], + "source": { + "sourceId": "mausamgram", + "sourceName": "IMD Mausamgram NWP" + }, + "location": { + "type": "Point", + "coordinates": [$lon, $lat] + }, + "validity": { + "startsAt": $day.date, + "endsAt": $day.date + }, + "parameters": [ + $reading("Rainfall", "Total", "mm", $day.rain), + $reading("Temperature", "Minimum", "Cel", $day.tmin), + $reading("Temperature", "Maximum", "Cel", $day.tmax), + $reading("Humidity", "Minimum", "%", $day.rhmin), + $reading("Humidity", "Maximum", "%", $day.rhmax), + $reading("WindSpeed", "Average", "m/s", $day.wspd) + ], + "advisory": $day.weather_warning ? $day.weather_warning : $day.cloud_message + } + } + }) + } + ] + } + } + } + ) diff --git a/config/oan-provider-adapter.yaml b/config/oan-provider-adapter.yaml new file mode 100644 index 00000000..7c289742 --- /dev/null +++ b/config/oan-provider-adapter.yaml @@ -0,0 +1,88 @@ +# OAN provider adapter. +# +# Serves /select synchronously: verifies the sender, resolves the capability's +# call plan from the registry, calls the provider, and answers with the mapped +# result. There is no callback -- the answer is the HTTP response. +# +# Adding a provider is three things and no Go changes to this file's shape: +# 1. a registry row binding "|" to a call plan +# 2. two mapping files, published at the URLs that row names +# 3. one more entry under providerSteps +appName: "oan-provider-adapter" + +log: + level: debug + destinations: + - type: stdout + contextKeys: [transaction_id, message_id, subscriber_id, module_id] + +http: + port: 8080 + timeout: + read: 30 + write: 30 + idle: 30 + +pluginManager: + root: ./plugins + +modules: + - name: oanProvider + # A subtree, not one action. The trailing slash matters: Go's ServeMux + # treats a path without one as an exact match, so /beckn/select would mount + # that action and 404 every other. Every action lands here, and which one it + # is comes from the payload's context.action, never from the URL. + path: /beckn/ + handler: + type: std + role: bpp + subscriberId: provider-network-vistaar.da.gov.in + + plugins: + # Serves both halves: the sender's signing key for validateSign, and the + # capability call plans the provider steps resolve against. + registry: + id: oanregistry + config: + url: http://registry:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + # Opt-in. This is exactly how long a suspended participant keeps + # verifying, and a withdrawn capability keeps being called. + cacheTTL: 60s + + keyManager: + id: simplekeymanager + config: + subscriberId: provider-network-vistaar.da.gov.in + + signer: + id: signer + signValidator: + id: signvalidator + + # Generic. Knows nothing about any provider; fetches, compiles and + # caches whatever the registry's mapping URLs point at. + mapper: + id: jsonmapper + config: + fetchTimeout: 5s + cacheTTL: 1h + negativeTTL: 1m + + # One entry per provider capability. Each recognises its own binding key + # and passes through anything else, which is the whole dispatch + # mechanism. Credentials are named, never held: authScheme says how to + # present them, and the *Env keys name environment variables. + providerSteps: + - id: mausamgram + config: + bindingKey: "mausamgram|openagrinet:WeatherObservation" + authScheme: basic + usernameEnv: MAUSAMGRAM_USER + passwordEnv: MAUSAMGRAM_X_API_KEY + + steps: + - validateSign # the sender's key, from the registry + - mausamgram # resolve, map out, call, map back + - signAck # signs whatever the step answered with diff --git a/core/module/handler/config.go b/core/module/handler/config.go index e7560365..78e10b86 100644 --- a/core/module/handler/config.go +++ b/core/module/handler/config.go @@ -31,6 +31,8 @@ type PluginManager interface { PayloadStore(ctx context.Context, cache definition.Cache, namespace string, cfg *plugin.Config) (definition.PayloadStore, error) CatalogPublisher(ctx context.Context, km definition.KeyManager, blobStore definition.CatalogBlobStore, registry definition.RegistryLookup, cfg *plugin.Config) (definition.CatalogPublisher, error) CatalogBlobStore(ctx context.Context, cfg *plugin.Config) (definition.CatalogBlobStore, error) + Mapper(ctx context.Context, cfg *plugin.Config) (definition.Mapper, error) + ProviderStep(ctx context.Context, registry definition.ProviderRecordLookup, mapper definition.Mapper, cfg *plugin.Config) (definition.Step, error) } // Type defines different handler types for processing requests. @@ -85,8 +87,14 @@ type PluginCfg struct { PayloadStore *plugin.Config `yaml:"payloadStore,omitempty"` CatalogPublisher *plugin.Config `yaml:"catalogPublisher,omitempty"` CatalogBlobStore *plugin.Config `yaml:"catalogBlobStore,omitempty"` + Mapper *plugin.Config `yaml:"mapper,omitempty"` Middleware []plugin.Config `yaml:"middleware,omitempty"` Steps []plugin.Config + // ProviderSteps are steps that serve one provider capability end to end. + // Separate from Steps because they are handed a registry and a mapper, which + // the plain StepProvider contract cannot do. They resolve by id in a step + // list exactly as Steps entries do. + ProviderSteps []plugin.Config `yaml:"providerSteps,omitempty"` } // PluginEntries returns a flat list of all configured plugins in this PluginCfg. @@ -115,11 +123,17 @@ func (p *PluginCfg) PluginEntries() []telemetry.PluginEntry { add("payload_store", p.PayloadStore) add("catalog_publisher", p.CatalogPublisher) add("catalog_blob_store", p.CatalogBlobStore) + add("mapper", p.Mapper) for i := range p.Steps { if p.Steps[i].ID != "" { entries = append(entries, telemetry.PluginEntry{Type: "step", ID: p.Steps[i].ID}) } } + for i := range p.ProviderSteps { + if p.ProviderSteps[i].ID != "" { + entries = append(entries, telemetry.PluginEntry{Type: "provider_step", ID: p.ProviderSteps[i].ID}) + } + } for i := range p.Middleware { if p.Middleware[i].ID != "" { entries = append(entries, telemetry.PluginEntry{Type: "middleware", ID: p.Middleware[i].ID}) diff --git a/core/module/handler/responsebody_test.go b/core/module/handler/responsebody_test.go new file mode 100644 index 00000000..7ccc87f9 --- /dev/null +++ b/core/module/handler/responsebody_test.go @@ -0,0 +1,246 @@ +package handler + +import ( + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" +) + +// answeringStep is a step that produces a synchronous answer, the way a +// provider plugin does once it has called upstream itself. +type answeringStep struct { + answer []byte +} + +func (s *answeringStep) Run(ctx *model.StepContext) error { + ctx.ResponseBody = s.answer + return nil +} + +// routeSettingStep sets a route, putting the request on the proxy path. +type routeSettingStep struct{} + +func (s *routeSettingStep) Run(ctx *model.StepContext) error { + target, err := url.Parse("http://upstream.invalid/get-daily") + if err != nil { + return err + } + ctx.Route = &model.Route{TargetType: "url", URL: target} + return nil +} + +var errStepFailed = errors.New("step failed") + +const v2SelectBody = `{"context":{"version":"2.0.0","action":"select","messageId":"msg-1"}}` + +func serve(t *testing.T, h *stdHandler, body string) *httptest.ResponseRecorder { + t.Helper() + req, err := http.NewRequest(http.MethodPost, "/select", strings.NewReader(body)) + if err != nil { + t.Fatalf("failed to build request: %v", err) + } + recorder := httptest.NewRecorder() + h.ServeHTTP(recorder, req) + return recorder +} + +// The behaviour every existing module depends on: no ResponseBody means the +// generated ACK, unchanged. This is the regression guard for the whole change. +func TestServeHTTPWritesTheGeneratedAckWhenNoStepAnswers(t *testing.T) { + h := &stdHandler{ + SubscriberID: "test-sub", + role: model.RoleBPP, + moduleName: "test-module", + steps: []definition.Step{}, + } + + recorder := serve(t, h, v2SelectBody) + + if recorder.Code != http.StatusOK { + t.Errorf("status = %d, want 200", recorder.Code) + } + var got model.Response + if err := json.Unmarshal(recorder.Body.Bytes(), &got); err != nil { + t.Fatalf("failed to decode the response: %v", err) + } + if got.Message.Status != model.StatusACK { + t.Errorf("status = %q, want %q", got.Message.Status, model.StatusACK) + } + if got.Message.MessageID != "msg-1" { + t.Errorf("message id = %q, want %q", got.Message.MessageID, "msg-1") + } +} + +// A step that answered gets its answer written verbatim, in place of the ACK. +func TestServeHTTPWritesAStepsAnswerInPlaceOfTheAck(t *testing.T) { + answer := []byte(`{"context":{"action":"on_select"},"message":{"catalogs":[]}}`) + h := &stdHandler{ + SubscriberID: "test-sub", + role: model.RoleBPP, + moduleName: "test-module", + steps: []definition.Step{&answeringStep{answer: answer}}, + } + + recorder := serve(t, h, v2SelectBody) + + if recorder.Code != http.StatusOK { + t.Errorf("status = %d, want 200", recorder.Code) + } + if got := recorder.Body.String(); got != string(answer) { + t.Errorf("body = %q, want %q", got, string(answer)) + } + if contentType := recorder.Header().Get("Content-Type"); contentType != "application/json" { + t.Errorf("content type = %q, want application/json", contentType) + } +} + +// An answer is only for the no-route path. A step that both answers and routes +// is contradicting itself, and routing wins because the proxy owns the response +// from that point on -- silently discarding one or the other would be worse. +func TestServeHTTPPrefersRoutingOverAnAnswer(t *testing.T) { + h := &stdHandler{ + SubscriberID: "test-sub", + role: model.RoleBPP, + moduleName: "test-module", + // proxy() reaches straight for httpClient.Transport, so a routed handler + // without one panics rather than failing. + httpClient: http.DefaultClient, + steps: []definition.Step{&answeringStep{answer: []byte(`{"answered":true}`)}, &routeSettingStep{}}, + } + + recorder := serve(t, h, v2SelectBody) + + if strings.Contains(recorder.Body.String(), `"answered"`) { + t.Error("expected routing to own the response once a route is set") + } +} + +// A step that fails after answering must still NACK: a half-built answer is not +// an answer, and an error has to reach the caller as one. +func TestServeHTTPNacksWhenAStepFailsAfterAnswering(t *testing.T) { + h := &stdHandler{ + SubscriberID: "test-sub", + role: model.RoleBPP, + moduleName: "test-module", + steps: []definition.Step{ + &answeringStep{answer: []byte(`{"answered":true}`)}, + &mockFailStep{err: model.NewBadReqErr("", errStepFailed)}, + }, + } + + recorder := serve(t, h, v2SelectBody) + + if recorder.Code != http.StatusBadRequest { + t.Errorf("status = %d, want 400", recorder.Code) + } + if strings.Contains(recorder.Body.String(), `"answered"`) { + t.Error("expected a NACK, not the partial answer") + } +} + +// An empty answer is not an answer. A step that sets no body leaves the ACK +// exactly as it was. +func TestServeHTTPTreatsAnEmptyAnswerAsNoAnswer(t *testing.T) { + h := &stdHandler{ + SubscriberID: "test-sub", + role: model.RoleBPP, + moduleName: "test-module", + steps: []definition.Step{&answeringStep{answer: []byte{}}}, + } + + recorder := serve(t, h, v2SelectBody) + + var got model.Response + if err := json.Unmarshal(recorder.Body.Bytes(), &got); err != nil { + t.Fatalf("failed to decode the response: %v", err) + } + if got.Message.Status != model.StatusACK { + t.Errorf("expected the generated ACK, got status %q", got.Message.Status) + } +} + +// The instrumentor shallow-copies the context in but copies only named fields +// back out. An answer written by an instrumented step has to survive that, or +// it works unwrapped and vanishes wrapped -- and wrapped is the default. +func TestInstrumentedStepCarriesAnAnswerBack(t *testing.T) { + answer := []byte(`{"answered":true}`) + instrumented, err := NewInstrumentedStep(&answeringStep{answer: answer}, "answer", "test-module") + if err != nil { + t.Fatalf("failed to instrument the step: %v", err) + } + + ctx := &model.StepContext{Context: t.Context()} + if err := instrumented.Run(ctx); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + if string(ctx.ResponseBody) != string(answer) { + t.Errorf("response body = %q, want %q -- the instrumentor dropped it", ctx.ResponseBody, answer) + } +} + +// signAck must cover what is actually sent. Signing the generated ACK while +// sending something else would put a valid signature over the wrong bytes. +func TestAckSignerSignsTheAnswerThatWillBeSent(t *testing.T) { + signer := &mockSigner{returnSig: "sig-over-the-answer"} + step, err := newAckSignerStep(signer, &mockKM{keyset: &model.Keyset{UniqueKeyID: "k1", SigningPrivate: "priv"}}) + if err != nil { + t.Fatalf("failed to build the ack signer: %v", err) + } + + ctx := &model.StepContext{ + Context: t.Context(), + SubID: "test-sub", + ProtocolVersion: model.ProtocolVersionV2, + MessageID: "msg-1", + RespHeader: http.Header{}, + ResponseBody: []byte(`{"context":{"action":"on_select"}}`), + } + if err := step.RunOnResponse(ctx, nil); err != nil { + t.Fatalf("RunOnResponse() returned an unexpected error: %v", err) + } + + if !signer.signAckCalled { + t.Fatal("expected the answer to be signed") + } + if got := ctx.RespHeader.Get("Signature"); !strings.Contains(got, "sig-over-the-answer") { + t.Errorf("Signature header = %q, want it to carry the answer's signature", got) + } + if string(signer.signedBody) != string(ctx.ResponseBody) { + t.Errorf("signed %q, want the body that will be sent, %q", signer.signedBody, ctx.ResponseBody) + } +} + +// With no answer, signAck still covers the generated ACK, exactly as before. +func TestAckSignerStillSignsTheGeneratedAckWhenNoStepAnswered(t *testing.T) { + signer := &mockSigner{returnSig: "sig-over-the-ack"} + step, err := newAckSignerStep(signer, &mockKM{keyset: &model.Keyset{UniqueKeyID: "k1", SigningPrivate: "priv"}}) + if err != nil { + t.Fatalf("failed to build the ack signer: %v", err) + } + + ctx := &model.StepContext{ + Context: t.Context(), + SubID: "test-sub", + ProtocolVersion: model.ProtocolVersionV2, + MessageID: "msg-1", + RespHeader: http.Header{}, + } + if err := step.RunOnResponse(ctx, nil); err != nil { + t.Fatalf("RunOnResponse() returned an unexpected error: %v", err) + } + + wantAck, err := buildAckBody(model.ProtocolVersionV2, "msg-1") + if err != nil { + t.Fatalf("failed to build the expected ack: %v", err) + } + if string(signer.signedBody) != string(wantAck) { + t.Errorf("signed %q, want the generated ack %q", signer.signedBody, wantAck) + } +} diff --git a/core/module/handler/responsestep.go b/core/module/handler/responsestep.go index 714b4047..e5cce064 100644 --- a/core/module/handler/responsestep.go +++ b/core/module/handler/responsestep.go @@ -35,6 +35,29 @@ type preV2Response struct { Message preV2Message `json:"message"` } +// sendResponse writes the synchronous response for the no-route path: a step's +// own answer when it produced one, and the generated ACK otherwise. +// +// Kept separate from sendAck rather than folded into it, because sendAck is also +// reached from the routing path where a step's answer has no meaning -- the +// proxy owns the response there. +func sendResponse(ctx *model.StepContext, w http.ResponseWriter) []byte { + if len(ctx.ResponseBody) == 0 { + return sendAck(ctx, w) + } + return writeJSONResponse(ctx, w, ctx.ResponseBody) +} + +// writeJSONResponse writes body as a 200 JSON response, reporting what it wrote. +func writeJSONResponse(ctx context.Context, w http.ResponseWriter, body []byte) []byte { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + if _, err := w.Write(body); err != nil { + log.Errorf(ctx, err, "failed to write response body: %v", err) + } + return body +} + // sendAck sends a synchronous ACK response to the client. // For context.version "2.0.0" and later the response uses the v2 envelope: // @@ -271,9 +294,17 @@ func (a *ackSignerStep) RunOnResponse(ctx *model.StepContext, rctx *model.Respon // Publisher / no-route path: ONIX writes the ACK — build the deterministic // body that sendAck will write so the digest matches. - ackBody, err := buildAckBody(ctx.ProtocolVersion, ctx.MessageID) - if err != nil { - return fmt.Errorf("ackSigner: failed to build ack body: %w", err) + // A step that answered supplies the body; otherwise rebuild the deterministic + // ACK. Either way this signs exactly what sendResponse will write -- signing + // the ACK while sending an answer would put a valid signature over the wrong + // bytes. + ackBody := ctx.ResponseBody + if len(ackBody) == 0 { + built, err := buildAckBody(ctx.ProtocolVersion, ctx.MessageID) + if err != nil { + return fmt.Errorf("ackSigner: failed to build ack body: %w", err) + } + ackBody = built } // signBodyAndSetHeader writes to ctx.RespHeader which IS the http.ResponseWriter // header map — the Signature header will be flushed when WriteHeader is called. diff --git a/core/module/handler/responsestep_test.go b/core/module/handler/responsestep_test.go index eca2b9a4..18b86f3d 100644 --- a/core/module/handler/responsestep_test.go +++ b/core/module/handler/responsestep_test.go @@ -487,6 +487,7 @@ type mockSigner struct { signAckErr error returnSig string // returned by SignAck returnSignSig string // returned by Sign (default "") + signedBody []byte // the body SignAck was last asked to cover } func (m *mockSigner) Sign(_ context.Context, _ []byte, _ string, _, _ int64) (string, error) { @@ -494,8 +495,9 @@ func (m *mockSigner) Sign(_ context.Context, _ []byte, _ string, _, _ int64) (st return m.returnSignSig, nil } -func (m *mockSigner) SignAck(_ context.Context, _ []byte, _ string, _ string, _, _ int64) (string, error) { +func (m *mockSigner) SignAck(_ context.Context, body []byte, _ string, _ string, _, _ int64) (string, error) { m.signAckCalled = true + m.signedBody = body if m.signAckErr != nil { return "", m.signAckErr } diff --git a/core/module/handler/stdHandler.go b/core/module/handler/stdHandler.go index cad19dc6..1995f7ff 100644 --- a/core/module/handler/stdHandler.go +++ b/core/module/handler/stdHandler.go @@ -55,6 +55,7 @@ type stdHandler struct { transportWrapper definition.TransportWrapper payloadTransformer definition.Step payloadStore definition.PayloadStore + mapper definition.Mapper // ackSigner is non-nil only when the "signAck" step is configured (Receiver // modules). It is also used to sign pipeline-NACK responses so that ALL // synchronous responses carry a Signature header per NFH-007 CON-004-02. @@ -227,7 +228,7 @@ func (h *stdHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } } - responseBody = sendAck(stepCtx, wrapped) + responseBody = sendResponse(stepCtx, wrapped) return } // Handle routing based on the defined route type. @@ -596,11 +597,39 @@ func (h *stdHandler) initPlugins(ctx context.Context, mgr PluginManager, cfg *Pl if h.payloadTransformer, err = loadPayloadTransformerStep(ctx, mgr, cfg.PayloadTransformer); err != nil { return err } + if h.mapper, err = LoadPlugin(ctx, "Mapper", cfg.Mapper, mgr.Mapper); err != nil { + return err + } log.Debugf(ctx, "All required plugins successfully loaded for stdHandler") return nil } +// loadProviderStep loads one provider step, checking up front for the +// dependencies it cannot be built without. Each produces a clear startup +// failure rather than a nil dereference on the first request to reach the step. +func (h *stdHandler) loadProviderStep(ctx context.Context, mgr PluginManager, cfg *plugin.Config) (definition.Step, error) { + if h.mapper == nil { + return nil, fmt.Errorf("failed to load ProviderStep plugin (%s): Mapper plugin not configured", cfg.ID) + } + if h.registry == nil { + return nil, fmt.Errorf("failed to load ProviderStep plugin (%s): Registry plugin not configured", cfg.ID) + } + // A registry serving signing keys need not also serve call plans -- they are + // separate interfaces for that reason -- so this narrowing is checked rather + // than assumed. + recordLookup, ok := h.registry.(definition.ProviderRecordLookup) + if !ok { + return nil, fmt.Errorf("failed to load ProviderStep plugin (%s): Registry plugin does not implement ProviderRecordLookup", cfg.ID) + } + step, err := mgr.ProviderStep(ctx, recordLookup, h.mapper, cfg) + if err != nil { + return nil, fmt.Errorf("failed to load ProviderStep plugin (%s): %w", cfg.ID, err) + } + log.Debugf(ctx, "Loaded ProviderStep plugin: %s", cfg.ID) + return step, nil +} + // initSteps initializes and validates processing steps for the processor. func (h *stdHandler) initSteps(ctx context.Context, mgr PluginManager, cfg *Config) error { steps := make(map[string]definition.Step) @@ -614,6 +643,17 @@ func (h *stdHandler) initSteps(ctx context.Context, mgr PluginManager, cfg *Conf steps[c.ID] = step } + // Load provider steps, which are handed the registry and mapper that plain + // plugin steps cannot receive. They land in the same id-keyed map, so a step + // list names them exactly like any other plugin step. + for _, c := range cfg.Plugins.ProviderSteps { + step, err := h.loadProviderStep(ctx, mgr, &c) + if err != nil { + return err + } + steps[c.ID] = step + } + // Register processing steps for _, step := range cfg.Steps { var s definition.Step diff --git a/core/module/handler/stdHandler_test.go b/core/module/handler/stdHandler_test.go index a50e0ec5..00214353 100644 --- a/core/module/handler/stdHandler_test.go +++ b/core/module/handler/stdHandler_test.go @@ -169,6 +169,14 @@ func (noopPluginManager) CatalogPublisher(_ context.Context, _ definition.KeyMan return nil, nil } +func (noopPluginManager) Mapper(_ context.Context, _ *plugin.Config) (definition.Mapper, error) { + return nil, nil +} + +func (noopPluginManager) ProviderStep(_ context.Context, _ definition.ProviderRecordLookup, _ definition.Mapper, _ *plugin.Config) (definition.Step, error) { + return nil, nil +} + func (noopPluginManager) CatalogBlobStore(_ context.Context, _ *plugin.Config) (definition.CatalogBlobStore, error) { return nil, nil } diff --git a/core/module/handler/step_instrumentor.go b/core/module/handler/step_instrumentor.go index dcfaee0b..152d9054 100644 --- a/core/module/handler/step_instrumentor.go +++ b/core/module/handler/step_instrumentor.go @@ -93,6 +93,7 @@ func (is *InstrumentedStep) Run(ctx *model.StepContext) error { } ctx.Body = stepCtx.Body + ctx.ResponseBody = stepCtx.ResponseBody ctx.Route = stepCtx.Route ctx.SubID = stepCtx.SubID ctx.Role = stepCtx.Role diff --git a/core/module/module_test.go b/core/module/module_test.go index acb5b432..b22d7bbf 100644 --- a/core/module/module_test.go +++ b/core/module/module_test.go @@ -93,6 +93,14 @@ func (m *mockPluginManager) CatalogPublisher(_ context.Context, _ definition.Key return nil, nil } +func (m *mockPluginManager) Mapper(_ context.Context, _ *plugin.Config) (definition.Mapper, error) { + return nil, nil +} + +func (m *mockPluginManager) ProviderStep(_ context.Context, _ definition.ProviderRecordLookup, _ definition.Mapper, _ *plugin.Config) (definition.Step, error) { + return nil, nil +} + func (m *mockPluginManager) CatalogBlobStore(_ context.Context, _ *plugin.Config) (definition.CatalogBlobStore, error) { return nil, nil } diff --git a/install/build-plugins.sh b/install/build-plugins.sh index 14820ef9..a185f29a 100755 --- a/install/build-plugins.sh +++ b/install/build-plugins.sh @@ -32,6 +32,7 @@ plugins=( "dediregistry" "oanregistry" "jsonmapper" + "mausamgram" "manifestloader" "reqpreprocessor" "otelsetup" diff --git a/pkg/model/model.go b/pkg/model/model.go index 7dd4825b..8e105c52 100644 --- a/pkg/model/model.go +++ b/pkg/model/model.go @@ -368,6 +368,15 @@ type StepContext struct { InboundAuthSignature string // Raw Base64 signature from the inbound Authorization header's signature="..." attribute IsCallerHandler bool // True when the handler is a Caller (outbound); false for Receiver (inbound) + // ResponseBody, when non-empty, is written as the synchronous response in + // place of the generated ACK envelope. It is how a step that has already + // obtained an answer -- a provider plugin that called upstream itself, rather + // than routing -- returns that answer to the caller. + // + // Empty means "generate the ACK", which is every module that does not set it. + // It is only consulted on the no-route path: once a Route is set the proxy + // owns the response. + ResponseBody []byte } // WithContext updates the existing StepContext with a new context. diff --git a/pkg/plugin/definition/step.go b/pkg/plugin/definition/step.go index 7f19115b..63f66b7f 100644 --- a/pkg/plugin/definition/step.go +++ b/pkg/plugin/definition/step.go @@ -25,3 +25,21 @@ type ResponseStep interface { type StepProvider interface { New(context.Context, map[string]string) (Step, func(), error) } + +// ProviderStep is a Step that serves one provider capability end to end: it +// resolves whatever the provider needs beyond the Beckn payload, calls it, and +// turns the answer back into Beckn. +// +// It is a plain Step at the pipeline's edge -- ProviderStepProvider exists only +// because it needs a registry and a mapper handed to it, which StepProvider +// cannot do. Everything provider-specific lives inside: the prerequisites the +// old per-provider services performed before a call (a station id resolved from +// coordinates, a token minted from credentials), and the call itself. +// +// A step that is handed a request for a capability it does not serve must do +// nothing and return nil. That is the whole dispatch mechanism: several provider +// steps sit in one pipeline, each recognises its own work, and adding a provider +// is one more entry rather than a change to a routing table. +type ProviderStepProvider interface { + New(ctx context.Context, registry ProviderRecordLookup, mapper Mapper, config map[string]string) (Step, func() error, error) +} diff --git a/pkg/plugin/implementation/catalogpublisher/handler_test.go b/pkg/plugin/implementation/catalogpublisher/handler_test.go index ee98707c..334a3485 100644 --- a/pkg/plugin/implementation/catalogpublisher/handler_test.go +++ b/pkg/plugin/implementation/catalogpublisher/handler_test.go @@ -145,6 +145,14 @@ func (m *catalogPublishTestManager) Registry(context.Context, definition.Cache, func (m *catalogPublishTestManager) KeyManager(context.Context, definition.RegistryLookup, *plugin.Config) (definition.KeyManager, error) { return fakeHandlerKeyManager{}, nil } +func (m *catalogPublishTestManager) Mapper(context.Context, *plugin.Config) (definition.Mapper, error) { + return nil, nil +} + +func (m *catalogPublishTestManager) ProviderStep(context.Context, definition.ProviderRecordLookup, definition.Mapper, *plugin.Config) (definition.Step, error) { + return nil, nil +} + func (m *catalogPublishTestManager) CatalogBlobStore(context.Context, *plugin.Config) (definition.CatalogBlobStore, error) { return fakeHandlerCatalogBlobStore{}, nil } diff --git a/pkg/plugin/implementation/internal/oanbinding/oanbinding.go b/pkg/plugin/implementation/internal/oanbinding/oanbinding.go new file mode 100644 index 00000000..cf8a43a7 --- /dev/null +++ b/pkg/plugin/implementation/internal/oanbinding/oanbinding.go @@ -0,0 +1,109 @@ +// Package oanbinding derives the capability binding a Beckn request is asking +// for, so a provider step can tell whether the request is its work and, if it +// is, which registry row describes the call. +// +// It is shared by every provider step rather than living in one, because the +// binding is a property of the OAN network's payloads and not of any provider. +package oanbinding + +import ( + "encoding/json" + "errors" + "fmt" + "strings" +) + +// separator joins a binding key's two halves. +const separator = "|" + +// ErrNoBinding reports a payload that names no capability binding. It is not a +// fault: a request for something else entirely reaches a provider step too, and +// the step's answer is to do nothing. +var ErrNoBinding = errors.New("oanbinding: payload names no capability binding") + +// Binding identifies one provider capability. +type Binding struct { + ParticipantID string + CapabilityCode string +} + +// Key renders the binding in the form the registry indexes on. +func (b Binding) Key() string { + return b.ParticipantID + separator + b.CapabilityCode +} + +// selectPayload is the part of a Beckn v2 payload a binding is derived from. +// +// Both commitments and resources are arrays, and both are read as such. The +// provider is named once per commitment and the type once per resource, so a +// single request can in principle carry several -- see From for what happens +// when it does. +type selectPayload struct { + Message struct { + Contract struct { + Commitments []struct { + Offer struct { + Provider struct { + ID string `json:"id"` + } `json:"provider"` + } `json:"offer"` + Resources []struct { + ResourceAttributes struct { + Type string `json:"@type"` + } `json:"resourceAttributes"` + } `json:"resources"` + } `json:"commitments"` + } `json:"contract"` + } `json:"message"` +} + +// From derives the capability binding a payload is asking for. +// +// Returns ErrNoBinding when the payload names no provider or no type, which is +// the ordinary case for a request a provider step is not meant to serve. +// +// A payload carrying more than one distinct provider or type is refused rather +// than resolved to its first: the two halves index one registry row describing +// one upstream call, so a request spanning several is asking for something this +// design cannot express. Guessing would silently serve part of it. +func From(body []byte) (Binding, error) { + var payload selectPayload + if err := json.Unmarshal(body, &payload); err != nil { + return Binding{}, fmt.Errorf("oanbinding: payload could not be read: %w", err) + } + + var providers, types []string + for _, commitment := range payload.Message.Contract.Commitments { + providers = appendDistinct(providers, commitment.Offer.Provider.ID) + for _, resource := range commitment.Resources { + types = appendDistinct(types, resource.ResourceAttributes.Type) + } + } + + if len(providers) == 0 || len(types) == 0 { + return Binding{}, ErrNoBinding + } + if len(providers) > 1 { + return Binding{}, fmt.Errorf("oanbinding: payload names %d providers (%s); one request maps to one call", + len(providers), strings.Join(providers, ", ")) + } + if len(types) > 1 { + return Binding{}, fmt.Errorf("oanbinding: payload names %d resource types (%s); one request maps to one call", + len(types), strings.Join(types, ", ")) + } + + return Binding{ParticipantID: providers[0], CapabilityCode: types[0]}, nil +} + +// appendDistinct adds value if it is neither empty nor already present. +func appendDistinct(values []string, value string) []string { + if value == "" { + return values + } + for _, existing := range values { + if existing == value { + return values + } + } + return append(values, value) +} diff --git a/pkg/plugin/implementation/internal/oanbinding/oanbinding_test.go b/pkg/plugin/implementation/internal/oanbinding/oanbinding_test.go new file mode 100644 index 00000000..d1f1ea41 --- /dev/null +++ b/pkg/plugin/implementation/internal/oanbinding/oanbinding_test.go @@ -0,0 +1,159 @@ +package oanbinding + +import ( + "errors" + "strings" + "testing" +) + +// realSelectPayload is a verbatim /select request captured from the OAN network +// on 29 Aug 2026. It is the reason this package reads through contract and +// commitments rather than off message directly: the design notes showed the +// shallower message.offer.provider.id, and the wire does not. +const realSelectPayload = `{ + "context": { "version": "2.0.0", "action": "select", + "networkId": "da.gov.in/vistaar", + "bapId": "seeker-network-vistaar.da.gov.in", + "bppId": "provider-network-vistaar.da.gov.in", + "transactionId": "9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44", + "messageId": "7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22", + "timestamp": "2026-08-26T06:12:01.330Z" }, + "message": { "contract": { "commitments": [{ + "status": { "descriptor": { "code": "DRAFT", "name": "Draft" } }, + "resources": [{ + "id": "res:mausamgram:point-forecast", + "resourceAttributes": { + "@context": "https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld", + "@type": "openagrinet:WeatherObservation", + "subjectCategories": ["Weather"], + "location": { "type": "Point", "coordinates": [73.7898, 19.9975] }, + "validity": { "startsAt": "2026-08-26", "endsAt": "2026-08-30" } + } + }], + "offer": { + "id": "offer:mausamgram:open-data", + "resourceIds": ["res:mausamgram:point-forecast"], + "provider": { "id": "mausamgram", + "descriptor": { "code": "IMD-NWP-01", "name": "IMD Mausamgram NWP" } } + } + }] } } +}` + +func TestFromReadsARealSelectPayload(t *testing.T) { + t.Parallel() + + got, err := From([]byte(realSelectPayload)) + if err != nil { + t.Fatalf("From() returned an unexpected error: %v", err) + } + if got.ParticipantID != "mausamgram" { + t.Errorf("participant = %q, want mausamgram", got.ParticipantID) + } + if got.CapabilityCode != "openagrinet:WeatherObservation" { + t.Errorf("capability = %q, want openagrinet:WeatherObservation", got.CapabilityCode) + } + if want := "mausamgram|openagrinet:WeatherObservation"; got.Key() != want { + t.Errorf("key = %q, want %q", got.Key(), want) + } +} + +// A payload that names no capability is the ordinary case for a request a +// provider step is not meant to serve, so it is reported as a sentinel a caller +// can recognise rather than as a fault. +func TestFromReportsAPayloadWithNoBinding(t *testing.T) { + t.Parallel() + + testCases := []struct{ name, body string }{ + {"an empty object", `{}`}, + {"no message", `{"context":{"action":"select"}}`}, + {"no contract", `{"message":{}}`}, + {"no commitments", `{"message":{"contract":{}}}`}, + {"an empty commitments array", `{"message":{"contract":{"commitments":[]}}}`}, + {"a commitment naming no provider", `{"message":{"contract":{"commitments":[{"resources":[{"resourceAttributes":{"@type":"t"}}]}]}}}`}, + {"a commitment with no resources", `{"message":{"contract":{"commitments":[{"offer":{"provider":{"id":"p"}}}]}}}`}, + {"a resource with no type", `{"message":{"contract":{"commitments":[{"offer":{"provider":{"id":"p"}},"resources":[{"resourceAttributes":{}}]}]}}}`}, + {"an empty provider id", `{"message":{"contract":{"commitments":[{"offer":{"provider":{"id":""}},"resources":[{"resourceAttributes":{"@type":"t"}}]}]}}}`}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + if _, err := From([]byte(tc.body)); !errors.Is(err, ErrNoBinding) { + t.Errorf("expected ErrNoBinding, got %v", err) + } + }) + } +} + +// Both levels are arrays, so a payload can carry several. One binding key +// describes one upstream call, so a request spanning more than one is refused +// rather than silently resolved to whichever came first. +func TestFromRefusesAnAmbiguousPayload(t *testing.T) { + t.Parallel() + + testCases := []struct{ name, body, wants string }{ + { + name: "two providers across commitments", + body: `{"message":{"contract":{"commitments":[ + {"offer":{"provider":{"id":"one"}},"resources":[{"resourceAttributes":{"@type":"t"}}]}, + {"offer":{"provider":{"id":"two"}},"resources":[{"resourceAttributes":{"@type":"t"}}]}]}}}`, + wants: "2 providers", + }, + { + name: "two types within one commitment", + body: `{"message":{"contract":{"commitments":[ + {"offer":{"provider":{"id":"one"}},"resources":[ + {"resourceAttributes":{"@type":"a"}}, + {"resourceAttributes":{"@type":"b"}}]}]}}}`, + wants: "2 resource types", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + _, err := From([]byte(tc.body)) + if err == nil { + t.Fatal("expected an ambiguous payload to be refused") + } + if errors.Is(err, ErrNoBinding) { + t.Error("ambiguity is not absence: it must not report ErrNoBinding") + } + if !strings.Contains(err.Error(), tc.wants) { + t.Errorf("error %q should say %q", err, tc.wants) + } + }) + } +} + +// Repetition is not ambiguity: several commitments naming the same provider and +// type describe one call, and must resolve rather than be refused. +func TestFromAcceptsRepetitionOfTheSameBinding(t *testing.T) { + t.Parallel() + + body := `{"message":{"contract":{"commitments":[ + {"offer":{"provider":{"id":"p"}},"resources":[{"resourceAttributes":{"@type":"t"}}]}, + {"offer":{"provider":{"id":"p"}},"resources":[{"resourceAttributes":{"@type":"t"}}]}]}}}` + + got, err := From([]byte(body)) + if err != nil { + t.Fatalf("From() returned an unexpected error: %v", err) + } + if got.Key() != "p|t" { + t.Errorf("key = %q, want p|t", got.Key()) + } +} + +func TestFromReportsAnUnreadablePayload(t *testing.T) { + t.Parallel() + + _, err := From([]byte(`{"message":`)) + if err == nil { + t.Fatal("expected unreadable JSON to be reported") + } + if errors.Is(err, ErrNoBinding) { + t.Error("a broken payload is not an absent binding") + } +} diff --git a/pkg/plugin/implementation/mausamgram/cmd/plugin.go b/pkg/plugin/implementation/mausamgram/cmd/plugin.go new file mode 100644 index 00000000..bfc2b8e5 --- /dev/null +++ b/pkg/plugin/implementation/mausamgram/cmd/plugin.go @@ -0,0 +1,74 @@ +package main + +import ( + "context" + "errors" + "fmt" + "strconv" + + "github.com/beckn-one/beckn-onix/pkg/log" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/mausamgram" +) + +// mausamgramProvider implements definition.ProviderStepProvider. +type mausamgramProvider struct{} + +// newStepFunc creates a new step. Indirected for tests. +var newStepFunc = mausamgram.New + +// parseConfig turns the plugin config map into a typed Config. Anything absent +// is left zero: mausamgram.New applies the defaults and validates the auth +// scheme, so those rules live in one place. +func (p mausamgramProvider) parseConfig(config map[string]string) (*mausamgram.Config, error) { + cfg := &mausamgram.Config{ + BindingKey: config["bindingKey"], + AuthScheme: config["authScheme"], + UsernameEnv: config["usernameEnv"], + PasswordEnv: config["passwordEnv"], + HeaderName: config["headerName"], + HeaderValueEnv: config["headerValueEnv"], + } + + if raw, exists := config["maxResponseBytes"]; exists && raw != "" { + value, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid maxResponseBytes value '%s': %w", raw, err) + } + if value <= 0 { + return nil, fmt.Errorf("maxResponseBytes must be positive, got %d", value) + } + cfg.MaxResponseBytes = value + } + + return cfg, nil +} + +// New creates a new mausamgram provider step instance. +func (p mausamgramProvider) New(ctx context.Context, registry definition.ProviderRecordLookup, mapper definition.Mapper, config map[string]string) (definition.Step, func() error, error) { + if ctx == nil { + return nil, nil, errors.New("context cannot be nil") + } + + cfg, err := p.parseConfig(config) + if err != nil { + log.Errorf(ctx, err, "Failed to parse mausamgram configuration") + return nil, nil, fmt.Errorf("failed to parse mausamgram configuration: %w", err) + } + + step, closer, err := newStepFunc(ctx, registry, mapper, cfg) + if err != nil { + log.Errorf(ctx, err, "Failed to create mausamgram step") + return nil, nil, err + } + + log.Infof(ctx, "Mausamgram step created successfully") + return step, closer, nil +} + +// Provider is the exported plugin instance. +var Provider = mausamgramProvider{} + +// Compile-time proof the provider satisfies the interface the manager asserts +// against. A mismatch is otherwise a runtime cast failure at startup. +var _ definition.ProviderStepProvider = Provider diff --git a/pkg/plugin/implementation/mausamgram/cmd/plugin_test.go b/pkg/plugin/implementation/mausamgram/cmd/plugin_test.go new file mode 100644 index 00000000..f3d3cc4b --- /dev/null +++ b/pkg/plugin/implementation/mausamgram/cmd/plugin_test.go @@ -0,0 +1,163 @@ +package main + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/mausamgram" +) + +type stubRegistry struct{} + +func (stubRegistry) ProviderRecord(context.Context, string) (*model.ProviderRecord, error) { + return nil, nil +} + +type stubMapper struct{} + +func (stubMapper) Transform(context.Context, string, string, any) ([]byte, error) { return nil, nil } + +func TestParseConfig(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + config map[string]string + expected *mausamgram.Config + expectedErr string + }{ + { + // Everything absent is left zero: mausamgram.New defaults it, so the + // rules are defined in exactly one place. + name: "leaves everything unset for New to default", + config: map[string]string{}, + expected: &mausamgram.Config{}, + }, + { + name: "reads every supported setting", + config: map[string]string{ + "bindingKey": "other|capability", + "authScheme": "basic", + "usernameEnv": "U", + "passwordEnv": "P", + "headerName": "X-Key", + "headerValueEnv": "V", + "maxResponseBytes": "2048", + }, + expected: &mausamgram.Config{ + BindingKey: "other|capability", + AuthScheme: "basic", + UsernameEnv: "U", + PasswordEnv: "P", + HeaderName: "X-Key", + HeaderValueEnv: "V", + MaxResponseBytes: 2048, + }, + }, + { + name: "rejects a malformed response cap", + config: map[string]string{"maxResponseBytes": "lots"}, + expectedErr: "invalid maxResponseBytes value 'lots'", + }, + { + name: "rejects a non-positive response cap", + config: map[string]string{"maxResponseBytes": "0"}, + expectedErr: "maxResponseBytes must be positive", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got, err := mausamgramProvider{}.parseConfig(tc.config) + + if tc.expectedErr != "" { + if err == nil { + t.Fatalf("expected error %q but got none", tc.expectedErr) + } + if !strings.Contains(err.Error(), tc.expectedErr) { + t.Errorf("expected error containing %q, got %q", tc.expectedErr, err.Error()) + } + return + } + + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !reflect.DeepEqual(got, tc.expected) { + t.Errorf("expected config %+v, got %+v", tc.expected, got) + } + }) + } +} + +func TestNew(t *testing.T) { + t.Parallel() + + t.Run("rejects a nil context", func(t *testing.T) { + t.Parallel() + + //nolint:staticcheck // deliberately passing a nil context to assert the guard. + _, _, err := mausamgramProvider{}.New(nil, stubRegistry{}, stubMapper{}, map[string]string{}) + if err == nil { + t.Fatal("expected an error for a nil context, got none") + } + }) + + t.Run("rejects an unparseable config", func(t *testing.T) { + t.Parallel() + + _, _, err := mausamgramProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, + map[string]string{"maxResponseBytes": "lots"}) + if err == nil { + t.Fatal("expected an error for an invalid cap, got none") + } + }) + + t.Run("propagates an invalid auth scheme from New", func(t *testing.T) { + t.Parallel() + + _, _, err := mausamgramProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, + map[string]string{"authScheme": "oauth"}) + if err == nil { + t.Fatal("expected an unknown auth scheme to be refused") + } + }) + + t.Run("builds a step from an empty config", func(t *testing.T) { + t.Parallel() + + step, closer, err := mausamgramProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, map[string]string{}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if step == nil { + t.Fatal("expected a step, got nil") + } + if err := closer(); err != nil { + t.Errorf("expected the closer to succeed, got: %v", err) + } + }) + + // Deliberately NOT parallel: this swaps the package-level newStepFunc. + t.Run("propagates a construction failure", func(t *testing.T) { + original := newStepFunc + t.Cleanup(func() { newStepFunc = original }) + + wantErr := errors.New("boom") + newStepFunc = func(context.Context, definition.ProviderRecordLookup, definition.Mapper, *mausamgram.Config) (*mausamgram.Step, func() error, error) { + return nil, nil, wantErr + } + + _, _, err := mausamgramProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, map[string]string{}) + if !errors.Is(err, wantErr) { + t.Errorf("expected the construction error to propagate, got %v", err) + } + }) +} diff --git a/pkg/plugin/implementation/mausamgram/dispatch_test.go b/pkg/plugin/implementation/mausamgram/dispatch_test.go new file mode 100644 index 00000000..c8b17e62 --- /dev/null +++ b/pkg/plugin/implementation/mausamgram/dispatch_test.go @@ -0,0 +1,87 @@ +package mausamgram_test + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/mausamgram" +) + +// fixedMapper returns canned results, so this test is about dispatch and +// nothing else. +type fixedMapper struct{ answer string } + +func (m fixedMapper) Transform(_ context.Context, mappingRef, _ string, _ any) ([]byte, error) { + if strings.Contains(mappingRef, "request") { + return []byte(`{}`), nil + } + return []byte(m.answer), nil +} + +// Two provider steps in one pipeline, as a second provider would be added. +// Each must serve its own capability and leave the other's alone -- that is the +// whole dispatch mechanism, so it is worth a test rather than an assumption. +func TestTwoProviderStepsDispatchByBindingKey(t *testing.T) { + var calledA, calledB bool + providerA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calledA = true + fmt.Fprint(w, `{"from":"A"}`) + })) + defer providerA.Close() + providerB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calledB = true + fmt.Fprint(w, `{"from":"B"}`) + })) + defer providerB.Close() + + newProviderStep := func(t *testing.T, bindingKey, upstreamURL, answer string) definition.Step { + t.Helper() + plan := &model.ProviderRecord{ + BindingKey: bindingKey, BaseURL: upstreamURL, + RequestMapping: "https://m.example.com/request.yaml", + ResponseMapping: "https://m.example.com/response.yaml", + Actions: map[string]model.ActionPlan{ + "select": {Method: http.MethodGet, Path: "/x", RetryMax: 1}, + }, + } + step, closer, err := mausamgram.New(context.Background(), + &stubRegistry{plan: plan}, + fixedMapper{answer: answer}, + &mausamgram.Config{BindingKey: bindingKey}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = closer() }) + return step + } + + steps := []definition.Step{ + newProviderStep(t, "mausamgram|openagrinet:WeatherObservation", providerA.URL, `{"served":"A"}`), + newProviderStep(t, "agmarknet|openagrinet:MarketPrice", providerB.URL, `{"served":"B"}`), + } + + // A request for the FIRST capability, run through both steps in order, as a + // pipeline would. + ctx := &model.StepContext{Context: t.Context(), Body: []byte(selectRequest)} + for i, step := range steps { + if err := step.Run(ctx); err != nil { + t.Fatalf("step %d: %v", i, err) + } + } + + if !calledA { + t.Error("provider A was not called for its own capability") + } + if calledB { + t.Error("provider B was called for a capability that is not its own") + } + if got := string(ctx.ResponseBody); !strings.Contains(got, `"A"`) { + t.Errorf("answer = %s, want A's -- a later step overwrote it", got) + } +} diff --git a/pkg/plugin/implementation/mausamgram/mappings_test.go b/pkg/plugin/implementation/mausamgram/mappings_test.go new file mode 100644 index 00000000..200dfed8 --- /dev/null +++ b/pkg/plugin/implementation/mausamgram/mappings_test.go @@ -0,0 +1,285 @@ +package mausamgram_test + +// mappings_test.go runs the shipped mapping files through the real mapper and +// the real provider step. It is the only test that proves the three pieces fit: +// a mapping is JSONata inside YAML fetched over HTTP, and nothing but running +// it establishes that what is published actually produces valid Beckn. +// +// An external test package on purpose -- it uses the plugins exactly as the +// adapter does, through their exported surface and nothing else. + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/mausamgram" +) + +// mappingsDir is where the shipped mappings live, relative to this package. +const mappingsDir = "../../../../config/mappings/mausamgram" + +// selectRequest is the verbatim /select captured from the OAN network. +const selectRequest = `{ + "context": { "version": "2.0.0", "action": "select", + "networkId": "da.gov.in/vistaar", + "bapId": "seeker-network-vistaar.da.gov.in", + "bapUri": "https://seeker-network-vistaar.da.gov.in/beckn", + "bppId": "provider-network-vistaar.da.gov.in", + "bppUri": "https://provider-network-vistaar.da.gov.in/beckn", + "transactionId": "9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44", + "messageId": "7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22", + "timestamp": "2026-08-26T06:12:01.330Z" }, + "message": { "contract": { "commitments": [{ + "status": { "descriptor": { "code": "DRAFT", "name": "Draft" } }, + "resources": [{ + "id": "res:mausamgram:point-forecast", + "resourceAttributes": { + "@context": "https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld", + "@type": "openagrinet:WeatherObservation", + "subjectCategories": ["Weather"], + "location": { "type": "Point", "coordinates": [73.7898, 19.9975] }, + "validity": { "startsAt": "2026-08-26", "endsAt": "2026-08-30" } + } + }], + "offer": { + "id": "offer:mausamgram:open-data", + "resourceIds": ["res:mausamgram:point-forecast"], + "provider": { "id": "mausamgram", + "descriptor": { "code": "IMD-NWP-01", "name": "IMD Mausamgram NWP" } } + } + }] } } +}` + +// providerResponse is Mausamgram's own shape, with the field names the old +// per-provider service read: fcstdayN carrying date, rain, tmin, tmax, rhmin, +// rhmax, wspd and a warning. Three days, not five, so the mapping is exercised +// against a provider that returned fewer than the maximum. +const providerResponse = `{ + "location": { "lat": 19.9975, "lon": 73.7898 }, + "fcstday1": { "date": "2026-08-26", "rain": 12.4, "tmin": 22.1, "tmax": 30.6, + "rhmin": 55, "rhmax": 92, "wspd": 4.2, + "weather_warning": "Heavy rainfall warning" }, + "fcstday2": { "date": "2026-08-27", "rain": 3.1, "tmin": 23.0, "tmax": 31.2, + "rhmin": 50, "rhmax": 88, "wspd": 3.4, + "cloud_message": "Partly cloudy" }, + "fcstday3": { "date": "2026-08-28", "tmin": 23.4, "tmax": 32.0 } +}` + +// serveMappings publishes the shipped mapping files over HTTP, the way the +// registry's references point at them. +func serveMappings(t *testing.T) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := os.ReadFile(filepath.Join(mappingsDir, filepath.Base(r.URL.Path))) + if err != nil { + t.Errorf("could not read the mapping %q: %v", r.URL.Path, err) + w.WriteHeader(http.StatusNotFound) + return + } + fmt.Fprint(w, string(body)) + })) +} + +// stubRegistry answers with the call plan the live registry holds for this +// capability. +type stubRegistry struct{ plan *model.ProviderRecord } + +func (s *stubRegistry) ProviderRecord(context.Context, string) (*model.ProviderRecord, error) { + return s.plan, nil +} + +// TestShippedMappingsServeARealSelect runs the published mappings end to end. +func TestShippedMappingsServeARealSelect(t *testing.T) { + mappings := serveMappings(t) + defer mappings.Close() + + var gotQuery string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotQuery = r.URL.RawQuery + fmt.Fprint(w, providerResponse) + })) + defer upstream.Close() + + mapper, closeMapper, err := jsonmapper.New(context.Background(), &jsonmapper.Config{}) + if err != nil { + t.Fatalf("failed to build the mapper: %v", err) + } + defer closeMapper() + + registry := &stubRegistry{plan: &model.ProviderRecord{ + BindingKey: mausamgram.DefaultBindingKey, + ParticipantID: "mausamgram", + CapabilityCode: "openagrinet:WeatherObservation", + BaseURL: upstream.URL, + RequestMapping: mappings.URL + "/request.yaml", + ResponseMapping: mappings.URL + "/response.yaml", + Actions: map[string]model.ActionPlan{ + "select": {Method: http.MethodGet, Path: "/get-daily", TimeoutMs: 30000, RetryMax: 3}, + }, + }} + + step, closeStep, err := mausamgram.New(context.Background(), registry, mapper, &mausamgram.Config{}) + if err != nil { + t.Fatalf("failed to build the step: %v", err) + } + defer closeStep() + + stepCtx := &model.StepContext{Context: t.Context(), Body: []byte(selectRequest)} + if err := step.Run(stepCtx); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + + // --- the request reached the provider correctly ------------------------- + // request.yaml declares select with no transform, so these parameters are + // the point the step resolved, not something a mapping produced. + for _, want := range []string{"lat=19.9975", "lon=73.7898"} { + if !strings.Contains(gotQuery, want) { + t.Errorf("upstream query %q is missing %q", gotQuery, want) + } + } + + // --- the response mapping produced Beckn -------------------------------- + if len(stepCtx.ResponseBody) == 0 { + t.Fatal("the step produced no answer") + } + var answer map[string]any + if err := json.Unmarshal(stepCtx.ResponseBody, &answer); err != nil { + t.Fatalf("the answer is not JSON: %v\n%s", err, stepCtx.ResponseBody) + } + + beckncontext, _ := answer["context"].(map[string]any) + if beckncontext["action"] != "on_select" { + t.Errorf("action = %v, want on_select", beckncontext["action"]) + } + // The transaction has to survive the round trip, or the caller cannot match + // the answer to what it asked. + if beckncontext["transactionId"] != "9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44" { + t.Errorf("transactionId = %v, want the one from the request", beckncontext["transactionId"]) + } + if beckncontext["bppId"] != "provider-network-vistaar.da.gov.in" { + t.Errorf("bppId = %v, want the one from the request", beckncontext["bppId"]) + } + + commitment := firstCommitment(t, answer) + status, _ := commitment["status"].(map[string]any) + descriptor, _ := status["descriptor"].(map[string]any) + if descriptor["code"] != "QUOTED" { + t.Errorf("status = %v, want QUOTED", descriptor["code"]) + } + if commitment["offer"] == nil { + t.Error("the quoted commitment carries no offer") + } + + resources, _ := commitment["resources"].([]any) + if len(resources) != 3 { + t.Fatalf("got %d resources, want 3 -- one per day the provider answered with", len(resources)) + } + + // --- the first day, in full --------------------------------------------- + first, _ := resources[0].(map[string]any) + if first["id"] != "res:mausamgram:forecast:2026-08-26" { + t.Errorf("resource id = %v, want it derived from the forecast date", first["id"]) + } + + attributes, _ := first["resourceAttributes"].(map[string]any) + if attributes["@type"] != "openagrinet:WeatherObservation" { + t.Errorf("@type = %v, want openagrinet:WeatherObservation", attributes["@type"]) + } + if attributes["observationType"] != "Forecast" { + t.Errorf("observationType = %v, want Forecast", attributes["observationType"]) + } + if attributes["advisory"] != "Heavy rainfall warning" { + t.Errorf("advisory = %v, want the provider's warning", attributes["advisory"]) + } + + // The point came from _local, not from the provider: it is the request's + // own coordinates, in GeoJSON order. + location, _ := attributes["location"].(map[string]any) + coordinates, _ := location["coordinates"].([]any) + if len(coordinates) != 2 || coordinates[0] != 73.7898 || coordinates[1] != 19.9975 { + t.Errorf("coordinates = %v, want [73.7898, 19.9975] in GeoJSON order", coordinates) + } + + parameters, _ := attributes["parameters"].([]any) + if len(parameters) != 6 { + t.Errorf("got %d parameters, want 6 for a fully-reported day", len(parameters)) + } + assertParameter(t, parameters, "Rainfall", "Total", "mm", 12.4) + assertParameter(t, parameters, "Temperature", "Minimum", "Cel", 22.1) + assertParameter(t, parameters, "WindSpeed", "Average", "m/s", 4.2) + + // --- a day the provider reported only partially -------------------------- + // Readings it did not take are absent, not present and empty: a consumer + // must be able to tell "no rainfall recorded" from "zero rainfall". + third, _ := resources[2].(map[string]any) + thirdAttributes, _ := third["resourceAttributes"].(map[string]any) + thirdParameters, _ := thirdAttributes["parameters"].([]any) + if len(thirdParameters) != 2 { + t.Errorf("got %d parameters for a partly-reported day, want only the 2 taken", len(thirdParameters)) + } + if thirdAttributes["advisory"] != nil { + t.Errorf("advisory = %v, want it absent when the provider gave none", thirdAttributes["advisory"]) + } +} + +// A file serves the actions it declares and no others. An action it does not +// carry is refused rather than served by whichever mapping happened to be there. +func TestShippedMappingsAreRefusedForAnotherAction(t *testing.T) { + mappings := serveMappings(t) + defer mappings.Close() + + mapper, closeMapper, err := jsonmapper.New(context.Background(), &jsonmapper.Config{}) + if err != nil { + t.Fatalf("failed to build the mapper: %v", err) + } + defer closeMapper() + + _, err = mapper.Transform(context.Background(), mappings.URL+"/request.yaml", "confirm", + map[string]any{"_local": map[string]any{"lat": 1.0, "lon": 2.0}}) + if err == nil { + t.Fatal("expected an unserved action to be refused") + } + // The refusal names what the file does serve, so a missing mapping is a + // one-line fix rather than a hunt. + if !strings.Contains(err.Error(), "select") { + t.Errorf("error %q should say which actions the file serves", err) + } +} + +func firstCommitment(t *testing.T, answer map[string]any) map[string]any { + t.Helper() + message, _ := answer["message"].(map[string]any) + contract, _ := message["contract"].(map[string]any) + commitments, _ := contract["commitments"].([]any) + if len(commitments) == 0 { + t.Fatalf("the answer carries no commitments: %v", answer) + } + commitment, _ := commitments[0].(map[string]any) + return commitment +} + +func assertParameter(t *testing.T, parameters []any, name, aggregation, unit string, value float64) { + t.Helper() + for _, raw := range parameters { + parameter, _ := raw.(map[string]any) + if parameter["parameter"] == name && parameter["aggregation"] == aggregation { + if parameter["unit"] != unit { + t.Errorf("%s/%s unit = %v, want %v", name, aggregation, parameter["unit"], unit) + } + if parameter["value"] != value { + t.Errorf("%s/%s value = %v, want %v", name, aggregation, parameter["value"], value) + } + return + } + } + t.Errorf("no %s/%s parameter in %v", name, aggregation, parameters) +} diff --git a/pkg/plugin/implementation/mausamgram/mausamgram.go b/pkg/plugin/implementation/mausamgram/mausamgram.go new file mode 100644 index 00000000..677c05dd --- /dev/null +++ b/pkg/plugin/implementation/mausamgram/mausamgram.go @@ -0,0 +1,527 @@ +// Package mausamgram serves the IMD Mausamgram point-forecast capability. +// +// It is the first provider step, and the shape every other one follows: it +// recognises its own capability, resolves what the provider needs beyond the +// Beckn payload, calls it, and lets the mapper translate in both directions. +// Nothing about weather forecasts appears outside this package, and nothing +// about mapping appears inside it. +package mausamgram + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "sort" + "strconv" + "strings" + "time" + + "github.com/beckn-one/beckn-onix/pkg/log" + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/oanbinding" +) + +// Defaults applied when the registry or the operator leaves a setting out. +const ( + // DefaultBindingKey is the capability this step serves. It is configurable + // so a deployment can rename the participant without a rebuild, but it has a + // default because a step that serves nothing is never what an operator meant. + DefaultBindingKey = "mausamgram|openagrinet:WeatherObservation" + DefaultTimeout = 30 * time.Second + DefaultRetryMax = 3 + // DefaultMaxResponseBytes caps what is read from the provider. The response + // is mapped in memory, so an unbounded one is an unbounded allocation. + DefaultMaxResponseBytes = 4 << 20 // 4 MiB +) + +// Auth schemes this step can present upstream. Credentials themselves are never +// configured here or held in the registry -- config names the environment +// variable to read, so a secret reaches the process through its environment and +// nothing else. +const ( + AuthSchemeNone = "none" + AuthSchemeBasic = "basic" + AuthSchemeHeader = "header" +) + +// codeUpstreamUnavailable reports a provider that could not be reached or +// answered with a failure. It is not this adapter's fault and not the caller's. +const codeUpstreamUnavailable = "NET_DOWNSTREAM_UNAVAILABLE" + +// Config holds configuration parameters for the step. +type Config struct { + // BindingKey is the capability this step answers to. A request for anything + // else passes through untouched. + BindingKey string `yaml:"bindingKey" json:"bindingKey"` + + // AuthScheme is how credentials are presented upstream: none, basic or + // header. Providers differ here -- basic auth, a raw token header, a field + // in the body -- which is why it is configuration and not an assumption. + AuthScheme string `yaml:"authScheme" json:"authScheme"` + + // UsernameEnv and PasswordEnv name the environment variables holding basic + // credentials. They are variable NAMES, never the values. + UsernameEnv string `yaml:"usernameEnv" json:"usernameEnv"` + PasswordEnv string `yaml:"passwordEnv" json:"passwordEnv"` + + // HeaderName and HeaderValueEnv configure the header scheme: which header to + // set, and which environment variable holds its value. + HeaderName string `yaml:"headerName" json:"headerName"` + HeaderValueEnv string `yaml:"headerValueEnv" json:"headerValueEnv"` + + // MaxResponseBytes caps what is read from the provider. + MaxResponseBytes int64 `yaml:"maxResponseBytes" json:"maxResponseBytes"` +} + +// Step serves the Mausamgram capability. It is safe for concurrent use. +type Step struct { + config *Config + registry definition.ProviderRecordLookup + mapper definition.Mapper + httpClient *http.Client +} + +// New creates the step. +func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper definition.Mapper, cfg *Config) (*Step, func() error, error) { + if registry == nil { + return nil, nil, errors.New("mausamgram: a provider record lookup is required") + } + if mapper == nil { + return nil, nil, errors.New("mausamgram: a mapper is required") + } + if cfg == nil { + cfg = &Config{} + } + if err := applyDefaults(cfg); err != nil { + return nil, nil, err + } + + step := &Step{ + config: cfg, + registry: registry, + mapper: mapper, + // Timeout is set per request from the registry's own budget, so the + // client carries none of its own. + httpClient: &http.Client{}, + } + + closer := func() error { + log.Debugf(ctx, "Cleaning up mausamgram step resources") + step.httpClient.CloseIdleConnections() + return nil + } + + log.Infof(ctx, "Mausamgram step created for binding %s", cfg.BindingKey) + return step, closer, nil +} + +// applyDefaults fills in what was left out and rejects what cannot be defaulted. +func applyDefaults(cfg *Config) error { + if cfg.BindingKey == "" { + cfg.BindingKey = DefaultBindingKey + } + if cfg.AuthScheme == "" { + cfg.AuthScheme = AuthSchemeNone + } + if cfg.MaxResponseBytes <= 0 { + cfg.MaxResponseBytes = DefaultMaxResponseBytes + } + + switch cfg.AuthScheme { + case AuthSchemeNone: + case AuthSchemeBasic: + if cfg.UsernameEnv == "" || cfg.PasswordEnv == "" { + return errors.New("mausamgram: authScheme basic requires usernameEnv and passwordEnv") + } + case AuthSchemeHeader: + if cfg.HeaderName == "" || cfg.HeaderValueEnv == "" { + return errors.New("mausamgram: authScheme header requires headerName and headerValueEnv") + } + default: + return fmt.Errorf("mausamgram: unknown authScheme %q: must be none, basic or header", cfg.AuthScheme) + } + return nil +} + +// Run serves the request when it is for this step's capability, and does +// nothing when it is not. +// +// Doing nothing is the dispatch mechanism: several provider steps sit in one +// pipeline and each recognises its own work, so adding a provider is one more +// entry rather than a change to a routing table. +func (s *Step) Run(ctx *model.StepContext) error { + binding, err := oanbinding.From(ctx.Body) + if errors.Is(err, oanbinding.ErrNoBinding) { + return nil + } + if err != nil { + return err + } + if binding.Key() != s.config.BindingKey { + log.Debugf(ctx, "mausamgram: %s is not this step's capability, passing through", binding.Key()) + return nil + } + + plan, err := s.registry.ProviderRecord(ctx, binding.Key()) + if err != nil { + return fmt.Errorf("mausamgram: no call plan for %s: %w", binding.Key(), err) + } + + return s.serve(ctx, plan) +} + +// serve runs the exchange this step exists for: resolve, map out, call, map back. +func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { + action := extractAction(ctx.Body) + call, served := plan.Actions[action] + if !served { + // The capability publishes no endpoint for this action, so it does not + // serve it. Refused here rather than after a call to whichever endpoint + // happened to be on the record -- naming what it does serve turns a + // registry mistake into a one-line fix. + return model.NewBadReqErr("", fmt.Errorf( + "mausamgram: %s does not serve action %q; it serves %s", + plan.BindingKey, action, strings.Join(servedActions(plan), ", "))) + } + + local, err := resolvePoint(ctx.Body) + if err != nil { + return err + } + + beckn, err := decodeBody(ctx.Body) + if err != nil { + return err + } + + upstreamRequest, err := s.buildRequest(ctx, plan.RequestMapping, action, beckn, local) + if err != nil { + return err + } + + upstreamResponse, err := s.call(ctx, plan.BaseURL, call, upstreamRequest) + if err != nil { + return err + } + + answer, err := decodeBody(upstreamResponse) + if err != nil { + return fmt.Errorf("mausamgram: provider answered with something that is not JSON: %w", err) + } + + // _local stays in scope: the provider does not repeat the point it was asked + // about, so the output's own coordinates have no other source. + // + // The response mapping is asked for by the action it PRODUCES, not the one + // that arrived: a select is answered by an on_select. Each mapping file is + // therefore keyed by the Beckn actions it actually deals in. + becknResponse, err := s.mapper.Transform(ctx, plan.ResponseMapping, callbackAction(action), map[string]any{ + "beckn": beckn, + "_local": local, + "response": answer, + }) + if err != nil { + return err + } + + ctx.ResponseBody = becknResponse + log.Infof(ctx, "mausamgram: served %s in %d bytes", plan.BindingKey, len(becknResponse)) + return nil +} + +// servedActions lists the actions a capability covers, sorted so the same +// record reads the same way twice. +func servedActions(plan *model.ProviderRecord) []string { + names := make([]string, 0, len(plan.Actions)) + for action := range plan.Actions { + names = append(names, action) + } + sort.Strings(names) + return names +} + +// buildRequest produces what the provider is sent. +// +// A mapping that declares the action but supplies no transform is saying the +// request needs no document built for it: this provider takes its parameters in +// the query, they are already resolved, and putting them through a fetch and a +// compile to arrive at the same two fields buys nothing. In that case the +// resolved values ARE the parameters. +// +// Anything else -- a provider wanting a body in its own shape -- goes through +// the mapping, which is what the mapping is for. +func (s *Step) buildRequest(ctx context.Context, mappingRef, action string, beckn any, local point) ([]byte, error) { + mapped, err := s.mapper.Transform(ctx, mappingRef, action, map[string]any{ + "beckn": beckn, + "_local": local, + }) + if err == nil { + return mapped, nil + } + if !errors.Is(err, definition.ErrNoTransform) { + return nil, err + } + + log.Debugf(ctx, "mausamgram: %s declares %s with no transform; sending the resolved point", mappingRef, action) + parameters, err := json.Marshal(local) + if err != nil { + return nil, fmt.Errorf("mausamgram: could not encode the resolved point: %w", err) + } + return parameters, nil +} + +// point is what this provider needs beyond the Beckn payload: a latitude and a +// longitude, as separate numbers. +type point struct { + Lat float64 `json:"lat"` + Lon float64 `json:"lon"` +} + +// resolvePoint reads the coordinates the request is asking about. +// +// This is the prerequisite step: the work a provider needs done before it can be +// called, which in the old per-provider services was tangled together with +// building the response. Here it produces values and nothing else, and the +// mapping decides what they are called upstream. +func resolvePoint(body []byte) (point, error) { + var payload struct { + Message struct { + Contract struct { + Commitments []struct { + Resources []struct { + ResourceAttributes struct { + Location struct { + Coordinates []float64 `json:"coordinates"` + } `json:"location"` + } `json:"resourceAttributes"` + } `json:"resources"` + } `json:"commitments"` + } `json:"contract"` + } `json:"message"` + } + if err := json.Unmarshal(body, &payload); err != nil { + return point{}, fmt.Errorf("mausamgram: request could not be read: %w", err) + } + + for _, commitment := range payload.Message.Contract.Commitments { + for _, resource := range commitment.Resources { + coordinates := resource.ResourceAttributes.Location.Coordinates + if len(coordinates) < 2 { + continue + } + // GeoJSON order: longitude first, then latitude. Reading these the + // other way round yields a point in the wrong hemisphere that is + // still a valid request, so it fails as wrong data rather than as an + // error. + return point{Lon: coordinates[0], Lat: coordinates[1]}, nil + } + } + return point{}, model.NewBadReqErr("", + errors.New("mausamgram: request carries no location coordinates")) +} + +// callbackAction is the action that answers the given one. Beckn pairs every +// request with an on_-prefixed callback -- select with on_select, confirm with +// on_confirm -- and that pairing is the protocol's, not this provider's. +func callbackAction(action string) string { + if action == "" { + return "" + } + return "on_" + action +} + +// extractAction reads the Beckn action a request is for. +func extractAction(body []byte) string { + var payload struct { + Context struct { + Action string `json:"action"` + } `json:"context"` + } + if err := json.Unmarshal(body, &payload); err != nil { + return "" + } + return payload.Context.Action +} + +// decodeBody turns raw JSON into the generic value a mapping reads. +func decodeBody(body []byte) (any, error) { + var decoded any + if err := json.Unmarshal(body, &decoded); err != nil { + return nil, fmt.Errorf("mausamgram: could not read JSON: %w", err) + } + return decoded, nil +} + +// call makes the upstream request described by the plan, retrying within its +// budget. +func (s *Step) call(ctx context.Context, baseURL string, call model.ActionPlan, mapped []byte) ([]byte, error) { + endpoint, err := buildEndpoint(baseURL, call, mapped) + if err != nil { + return nil, err + } + + timeout := DefaultTimeout + if call.TimeoutMs > 0 { + timeout = time.Duration(call.TimeoutMs) * time.Millisecond + } + attempts := DefaultRetryMax + if call.RetryMax > 0 { + attempts = call.RetryMax + } + + var lastErr error + for attempt := 1; attempt <= attempts; attempt++ { + body, err := s.attempt(ctx, call, endpoint, mapped, timeout) + if err == nil { + return body, nil + } + lastErr = err + log.Warnf(ctx, "mausamgram: attempt %d/%d failed: %v", attempt, attempts, err) + } + return nil, model.NewCodedErr(http.StatusBadGateway, codeUpstreamUnavailable, + fmt.Errorf("mausamgram: provider did not answer after %d attempts: %w", attempts, lastErr)) +} + +// attempt makes one upstream request. +func (s *Step) attempt(ctx context.Context, call model.ActionPlan, endpoint string, mapped []byte, timeout time.Duration) ([]byte, error) { + attemptCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + req, err := http.NewRequestWithContext(attemptCtx, call.Method, endpoint, requestBody(call.Method, mapped)) + if err != nil { + return nil, fmt.Errorf("could not build the request: %w", err) + } + if hasBody(call.Method) { + req.Header.Set("Content-Type", "application/json") + } + if err := s.authenticate(req); err != nil { + return nil, err + } + + resp, err := s.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(io.LimitReader(resp.Body, s.config.MaxResponseBytes+1)) + if err != nil { + return nil, fmt.Errorf("could not read the response: %w", err) + } + if int64(len(body)) > s.config.MaxResponseBytes { + return nil, fmt.Errorf("response exceeds the %d byte limit", s.config.MaxResponseBytes) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("provider returned %s", resp.Status) + } + return body, nil +} + +// authenticate presents this provider's credentials, read from the environment +// at call time so a rotated secret takes effect without a restart. +func (s *Step) authenticate(req *http.Request) error { + switch s.config.AuthScheme { + case AuthSchemeBasic: + username, password := os.Getenv(s.config.UsernameEnv), os.Getenv(s.config.PasswordEnv) + if username == "" || password == "" { + return fmt.Errorf("mausamgram: %s and %s must both be set for basic auth", + s.config.UsernameEnv, s.config.PasswordEnv) + } + req.SetBasicAuth(username, password) + case AuthSchemeHeader: + value := os.Getenv(s.config.HeaderValueEnv) + if value == "" { + return fmt.Errorf("mausamgram: %s must be set for header auth", s.config.HeaderValueEnv) + } + req.Header.Set(s.config.HeaderName, value) + } + return nil +} + +// buildEndpoint joins the plan's base URL and path, carrying the mapped request +// as query parameters when the method takes no body. +func buildEndpoint(baseURL string, call model.ActionPlan, mapped []byte) (string, error) { + endpoint := strings.TrimSuffix(baseURL, "/") + call.Path + if hasBody(call.Method) { + return endpoint, nil + } + + query, err := asQuery(mapped) + if err != nil { + return "", err + } + if query == "" { + return endpoint, nil + } + if strings.Contains(endpoint, "?") { + return endpoint + "&" + query, nil + } + return endpoint + "?" + query, nil +} + +// asQuery renders a mapped request as query parameters. +// +// A method with no body still needs the mapping's output somewhere, and the +// query string is the only place it can go. Only scalars are carried: a nested +// value has no single obvious encoding, and inventing one here would put a +// convention in Go that belongs in the mapping. +func asQuery(mapped []byte) (string, error) { + if len(bytes.TrimSpace(mapped)) == 0 { + return "", nil + } + var fields map[string]any + if err := json.Unmarshal(mapped, &fields); err != nil { + return "", fmt.Errorf("mausamgram: mapped request is not an object, so it cannot become a query: %w", err) + } + + values := url.Values{} + for name, value := range fields { + rendered, ok := renderScalar(value) + if !ok { + return "", fmt.Errorf("mausamgram: mapped field %q is not a scalar and cannot become a query parameter", name) + } + values.Set(name, rendered) + } + return values.Encode(), nil +} + +// renderScalar renders a JSON scalar as a query parameter value. +func renderScalar(value any) (string, bool) { + switch typed := value.(type) { + case string: + return typed, true + case bool: + return strconv.FormatBool(typed), true + case float64: + // 'g' with -1 precision round-trips without inventing trailing zeros, so + // 19.9975 stays 19.9975 rather than becoming 19.997500. + return strconv.FormatFloat(typed, 'g', -1, 64), true + default: + return "", false + } +} + +// requestBody returns the body to send, which is none for methods that take none. +func requestBody(method string, mapped []byte) io.Reader { + if !hasBody(method) { + return nil + } + return bytes.NewReader(mapped) +} + +// hasBody reports whether a method carries a request body. +func hasBody(method string) bool { + switch strings.ToUpper(method) { + case http.MethodGet, http.MethodHead, http.MethodDelete, "": + return false + default: + return true + } +} diff --git a/pkg/plugin/implementation/mausamgram/mausamgram_test.go b/pkg/plugin/implementation/mausamgram/mausamgram_test.go new file mode 100644 index 00000000..e6e28c3f --- /dev/null +++ b/pkg/plugin/implementation/mausamgram/mausamgram_test.go @@ -0,0 +1,583 @@ +package mausamgram + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync/atomic" + "testing" + + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" +) + +const selectBody = `{ + "context": { "version": "2.0.0", "action": "select", "transactionId": "txn-1" }, + "message": { "contract": { "commitments": [{ + "resources": [{ "resourceAttributes": { + "@type": "openagrinet:WeatherObservation", + "location": { "type": "Point", "coordinates": [73.7898, 19.9975] } } }], + "offer": { "provider": { "id": "mausamgram" } } + }] } } +}` + +// --- test doubles ----------------------------------------------------------- + +type stubRegistry struct { + plan *model.ProviderRecord + err error +} + +func (s *stubRegistry) ProviderRecord(context.Context, string) (*model.ProviderRecord, error) { + return s.plan, s.err +} + +// stubMapper records what it was asked and returns canned results, so a test can +// assert what reached the mapping without writing one. +type stubMapper struct { + requestResult []byte + responseResult []byte + err error + requestErr error + + requestInput any + responseInput any + actions []string + refs []string +} + +func (s *stubMapper) Transform(_ context.Context, mappingRef, action string, input any) ([]byte, error) { + s.actions = append(s.actions, action) + s.refs = append(s.refs, mappingRef) + if s.err != nil { + return nil, s.err + } + if strings.Contains(mappingRef, "request") { + s.requestInput = input + if s.requestErr != nil { + return nil, s.requestErr + } + return s.requestResult, nil + } + s.responseInput = input + return s.responseResult, nil +} + +func testPlan(baseURL, method string) *model.ProviderRecord { + return &model.ProviderRecord{ + BindingKey: DefaultBindingKey, + ParticipantID: "mausamgram", + CapabilityCode: "openagrinet:WeatherObservation", + BaseURL: baseURL, + RequestMapping: "https://mappings.example.com/request.yaml", + ResponseMapping: "https://mappings.example.com/response.yaml", + Actions: map[string]model.ActionPlan{ + "select": {Method: method, Path: "/get-daily", TimeoutMs: 2000, RetryMax: 1}, + }, + } +} + +func newStep(t *testing.T, registry definition.ProviderRecordLookup, mapper definition.Mapper, tweak ...func(*Config)) *Step { + t.Helper() + + cfg := &Config{} + for _, apply := range tweak { + apply(cfg) + } + step, closer, err := New(context.Background(), registry, mapper, cfg) + if err != nil { + t.Fatalf("New() returned an unexpected error: %v", err) + } + t.Cleanup(func() { _ = closer() }) + return step +} + +func runStep(t *testing.T, step *Step, body string) (*model.StepContext, error) { + t.Helper() + ctx := &model.StepContext{Context: t.Context(), Body: []byte(body)} + return ctx, step.Run(ctx) +} + +// --- construction ----------------------------------------------------------- + +func TestNewRequiresItsDependencies(t *testing.T) { + t.Parallel() + + if _, _, err := New(context.Background(), nil, &stubMapper{}, &Config{}); err == nil { + t.Error("expected a missing registry to be refused") + } + if _, _, err := New(context.Background(), &stubRegistry{}, nil, &Config{}); err == nil { + t.Error("expected a missing mapper to be refused") + } +} + +func TestNewValidatesTheAuthScheme(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + config *Config + valid bool + }{ + {"none by default", &Config{}, true}, + {"basic with both variables", &Config{AuthScheme: AuthSchemeBasic, UsernameEnv: "U", PasswordEnv: "P"}, true}, + {"basic missing the password variable", &Config{AuthScheme: AuthSchemeBasic, UsernameEnv: "U"}, false}, + {"basic missing the username variable", &Config{AuthScheme: AuthSchemeBasic, PasswordEnv: "P"}, false}, + {"header with both settings", &Config{AuthScheme: AuthSchemeHeader, HeaderName: "X-Key", HeaderValueEnv: "V"}, true}, + {"header missing the value variable", &Config{AuthScheme: AuthSchemeHeader, HeaderName: "X-Key"}, false}, + {"an unknown scheme", &Config{AuthScheme: "oauth"}, false}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + _, _, err := New(context.Background(), &stubRegistry{}, &stubMapper{}, tc.config) + if tc.valid && err != nil { + t.Errorf("expected the config to be accepted, got %v", err) + } + if !tc.valid && err == nil { + t.Error("expected the config to be refused") + } + }) + } +} + +// --- dispatch --------------------------------------------------------------- + +// Passing through is how dispatch works: several provider steps share a +// pipeline, and each must leave alone what is not its own. +func TestRunPassesThroughWhatIsNotItsCapability(t *testing.T) { + t.Parallel() + + testCases := []struct{ name, body string }{ + {"another provider", strings.Replace(selectBody, `"id": "mausamgram"`, `"id": "agmarknet"`, 1)}, + {"another capability", strings.Replace(selectBody, "openagrinet:WeatherObservation", "openagrinet:MarketPrice", 1)}, + {"a payload with no binding at all", `{"context":{"action":"select"},"message":{}}`}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + registry := &stubRegistry{err: errors.New("the registry must not be consulted")} + ctx, err := runStep(t, newStep(t, registry, &stubMapper{}), tc.body) + if err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + if ctx.ResponseBody != nil { + t.Error("a passed-through request must not be answered") + } + }) + } +} + +func TestRunReportsAnUnreadablePayload(t *testing.T) { + t.Parallel() + + if _, err := runStep(t, newStep(t, &stubRegistry{}, &stubMapper{}), `{"message":`); err == nil { + t.Error("expected unreadable JSON to be reported") + } +} + +// --- the exchange ----------------------------------------------------------- + +func TestRunServesItsCapabilityEndToEnd(t *testing.T) { + t.Parallel() + + var gotQuery string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotQuery = r.URL.RawQuery + fmt.Fprint(w, `{"fcstday1":{"rain":12.4}}`) + })) + defer upstream.Close() + + mapper := &stubMapper{ + requestResult: []byte(`{"lat":19.9975,"lon":73.7898}`), + responseResult: []byte(`{"context":{"action":"on_select"}}`), + } + registry := &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)} + + ctx, err := runStep(t, newStep(t, registry, mapper), selectBody) + if err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + + if string(ctx.ResponseBody) != `{"context":{"action":"on_select"}}` { + t.Errorf("response body = %q, want the mapped answer", ctx.ResponseBody) + } + if !strings.Contains(gotQuery, "lat=19.9975") || !strings.Contains(gotQuery, "lon=73.7898") { + t.Errorf("upstream query = %q, want the mapped fields", gotQuery) + } + // Each leg asks for the action it deals in: the request translates a select, + // the response produces an on_select. Asking for the same name on both would + // make one file unable to hold both directions. + if want := []string{"select", "on_select"}; !slices.Equal(mapper.actions, want) { + t.Errorf("mapper was asked for %v, want %v", mapper.actions, want) + } +} + +// The response mapping sees the point resolved before the call. The provider +// does not echo it back, so nothing else can supply it. +func TestRunKeepsResolvedValuesInScopeForTheResponse(t *testing.T) { + t.Parallel() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"fcstday1":{"rain":12.4}}`) + })) + defer upstream.Close() + + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), selectBody) + if err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + + input, ok := mapper.responseInput.(map[string]any) + if !ok { + t.Fatalf("response input = %T, want a map", mapper.responseInput) + } + for _, key := range []string{"beckn", "_local", "response"} { + if _, present := input[key]; !present { + t.Errorf("response mapping cannot see %q", key) + } + } + + local, ok := input["_local"].(point) + if !ok { + t.Fatalf("_local = %T, want a point", input["_local"]) + } + // GeoJSON order: the payload carries [lon, lat], so reading it positionally + // the other way round would put this point in the wrong hemisphere. + if local.Lat != 19.9975 || local.Lon != 73.7898 { + t.Errorf("_local = %+v, want lat 19.9975 lon 73.7898", local) + } +} + +func TestRunSendsTheMappedBodyForAMethodThatTakesOne(t *testing.T) { + t.Parallel() + + var gotBody, gotType string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body := make([]byte, r.ContentLength) + _, _ = r.Body.Read(body) + gotBody = string(body) + gotType = r.Header.Get("Content-Type") + fmt.Fprint(w, `{}`) + })) + defer upstream.Close() + + mapper := &stubMapper{requestResult: []byte(`{"lat":19.9975}`), responseResult: []byte(`{}`)} + if _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodPost)}, mapper), selectBody); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + + if gotBody != `{"lat":19.9975}` { + t.Errorf("upstream body = %q, want the mapped request", gotBody) + } + if gotType != "application/json" { + t.Errorf("content type = %q, want application/json", gotType) + } +} + +// A provider taking query parameters needs no request document built: the +// mapping declares the action and leaves it empty, and the values the step +// already resolved become the parameters. +func TestRunSendsResolvedValuesWhenTheMappingDeclaresNoTransform(t *testing.T) { + t.Parallel() + + var gotQuery string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotQuery = r.URL.RawQuery + fmt.Fprint(w, `{}`) + })) + defer upstream.Close() + + mapper := &stubMapper{requestErr: definition.ErrNoTransform, responseResult: []byte(`{}`)} + if _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), selectBody); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + + for _, want := range []string{"lat=19.9975", "lon=73.7898"} { + if !strings.Contains(gotQuery, want) { + t.Errorf("query %q is missing %q -- the resolved point was not sent", gotQuery, want) + } + } +} + +// Only ErrNoTransform means "build it yourself". Any other mapping failure is a +// real failure and must not be papered over by sending the resolved values. +func TestRunDoesNotSubstituteForARealMappingFailure(t *testing.T) { + t.Parallel() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Error("the provider must not be called when the mapping failed") + })) + defer upstream.Close() + + wantErr := errors.New("mapping is broken") + mapper := &stubMapper{requestErr: wantErr, responseResult: []byte(`{}`)} + + _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), selectBody) + if !errors.Is(err, wantErr) { + t.Errorf("expected the mapping failure to propagate, got %v", err) + } +} + +// --- authentication --------------------------------------------------------- + +func TestRunPresentsBasicCredentialsFromTheEnvironment(t *testing.T) { + t.Setenv("TEST_MAUSAMGRAM_USER", "user-1") + t.Setenv("TEST_MAUSAMGRAM_KEY", "key-1") + + var gotUser, gotPass string + var hadAuth bool + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotUser, gotPass, hadAuth = r.BasicAuth() + fmt.Fprint(w, `{}`) + })) + defer upstream.Close() + + step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, + &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, + func(c *Config) { + c.AuthScheme = AuthSchemeBasic + c.UsernameEnv = "TEST_MAUSAMGRAM_USER" + c.PasswordEnv = "TEST_MAUSAMGRAM_KEY" + }) + + if _, err := runStep(t, step, selectBody); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + if !hadAuth || gotUser != "user-1" || gotPass != "key-1" { + t.Errorf("basic auth = (%q, %q, present=%v), want the configured credentials", gotUser, gotPass, hadAuth) + } +} + +func TestRunPresentsAHeaderCredentialFromTheEnvironment(t *testing.T) { + t.Setenv("TEST_MAUSAMGRAM_TOKEN", "token-1") + + var gotHeader string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotHeader = r.Header.Get("X-Api-Key") + fmt.Fprint(w, `{}`) + })) + defer upstream.Close() + + step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, + &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, + func(c *Config) { + c.AuthScheme = AuthSchemeHeader + c.HeaderName = "X-Api-Key" + c.HeaderValueEnv = "TEST_MAUSAMGRAM_TOKEN" + }) + + if _, err := runStep(t, step, selectBody); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + if gotHeader != "token-1" { + t.Errorf("X-Api-Key = %q, want token-1", gotHeader) + } +} + +// A configured credential that is not in the environment is a deployment fault, +// and must fail rather than call the provider unauthenticated. +func TestRunFailsWhenAConfiguredCredentialIsAbsent(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Error("the provider must not be called without its credentials") + })) + defer upstream.Close() + + step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, + &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, + func(c *Config) { + c.AuthScheme = AuthSchemeBasic + c.UsernameEnv = "TEST_MAUSAMGRAM_ABSENT_USER" + c.PasswordEnv = "TEST_MAUSAMGRAM_ABSENT_KEY" + }) + + if _, err := runStep(t, step, selectBody); err == nil { + t.Error("expected a missing credential to fail the request") + } +} + +// --- failures --------------------------------------------------------------- + +func TestRunPropagatesAFailedLookup(t *testing.T) { + t.Parallel() + + registry := &stubRegistry{err: definition.ErrProviderRecordNotFound} + _, err := runStep(t, newStep(t, registry, &stubMapper{}), selectBody) + if !errors.Is(err, definition.ErrProviderRecordNotFound) { + t.Errorf("expected the lookup failure to propagate, got %v", err) + } +} + +func TestRunPropagatesAFailedMapping(t *testing.T) { + t.Parallel() + + wantErr := errors.New("mapping is broken") + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + defer upstream.Close() + + registry := &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)} + _, err := runStep(t, newStep(t, registry, &stubMapper{err: wantErr}), selectBody) + if !errors.Is(err, wantErr) { + t.Errorf("expected the mapping failure to propagate, got %v", err) + } +} + +func TestRunReportsAProviderThatWillNotAnswer(t *testing.T) { + t.Parallel() + + var attempts atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts.Add(1) + w.WriteHeader(http.StatusInternalServerError) + })) + defer upstream.Close() + + plan := testPlan(upstream.URL, http.MethodGet) + plan.Actions["select"] = model.ActionPlan{Method: http.MethodGet, Path: "/get-daily", RetryMax: 3} + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + + _, err := runStep(t, newStep(t, &stubRegistry{plan: plan}, mapper), selectBody) + if err == nil { + t.Fatal("expected a failing provider to be reported") + } + if got := attempts.Load(); got != 3 { + t.Errorf("made %d attempts, want the plan's 3", got) + } + + var coded *model.CodedErr + if !errors.As(err, &coded) || coded.HTTPStatus() != http.StatusBadGateway { + t.Errorf("expected a 502 coded error, got %v", err) + } +} + +// A capability that publishes no endpoint for an action does not serve it. The +// refusal has to come before the call, not after one to the wrong place. +func TestRunRefusesAnActionWithNoEndpoint(t *testing.T) { + t.Parallel() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Error("the provider must not be called for an action it does not serve") + })) + defer upstream.Close() + + // The plan serves select; the request is a confirm for the same capability. + confirmBody := strings.Replace(selectBody, `"action": "select"`, `"action": "confirm"`, 1) + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + + _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), confirmBody) + if err == nil { + t.Fatal("expected an action with no endpoint to be refused") + } + if !strings.Contains(err.Error(), "confirm") { + t.Errorf("error %q should name the action that was asked for", err) + } + // Naming what the capability does serve turns a registry mistake into a + // one-line fix. + if !strings.Contains(err.Error(), "select") { + t.Errorf("error %q should say which actions the capability does serve", err) + } +} + +// Each action carries its own endpoint and budget: a confirm that commits +// rarely posts where a select that reads gets. +func TestRunUsesTheEndpointForTheRequestedAction(t *testing.T) { + t.Parallel() + + var gotPath, gotMethod string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath, gotMethod = r.URL.Path, r.Method + fmt.Fprint(w, `{}`) + })) + defer upstream.Close() + + plan := testPlan(upstream.URL, http.MethodGet) + plan.Actions["confirm"] = model.ActionPlan{Method: http.MethodPost, Path: "/book", TimeoutMs: 2000, RetryMax: 1} + + confirmBody := strings.Replace(selectBody, `"action": "select"`, `"action": "confirm"`, 1) + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + + if _, err := runStep(t, newStep(t, &stubRegistry{plan: plan}, mapper), confirmBody); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + if gotMethod != http.MethodPost || gotPath != "/book" { + t.Errorf("called %s %s, want POST /book -- the select endpoint was used", gotMethod, gotPath) + } +} + +func TestRunReportsAProviderAnsweringWithNonJSON(t *testing.T) { + t.Parallel() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "down for maintenance") + })) + defer upstream.Close() + + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), selectBody) + if err == nil { + t.Error("expected a non-JSON answer to be reported") + } +} + +func TestRunReportsARequestWithNoCoordinates(t *testing.T) { + t.Parallel() + + body := strings.Replace(selectBody, `"location": { "type": "Point", "coordinates": [73.7898, 19.9975] }`, `"location": {}`, 1) + registry := &stubRegistry{plan: testPlan("http://upstream.invalid", http.MethodGet)} + + _, err := runStep(t, newStep(t, registry, &stubMapper{}), body) + if err == nil { + t.Error("expected a request with no coordinates to be refused") + } +} + +// A mapping producing something that cannot become a query has to fail loudly: +// dropping the field would call the provider with the wrong question. +func TestRunRefusesAMappedQueryItCannotRender(t *testing.T) { + t.Parallel() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Error("the provider must not be called with an incomplete query") + })) + defer upstream.Close() + + mapper := &stubMapper{requestResult: []byte(`{"box":{"nested":true}}`), responseResult: []byte(`{}`)} + _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), selectBody) + if err == nil { + t.Error("expected a non-scalar mapped field to be refused") + } +} + +// --- query rendering -------------------------------------------------------- + +func TestAsQueryRendersScalarsWithoutInventingPrecision(t *testing.T) { + t.Parallel() + + got, err := asQuery([]byte(`{"lat":19.9975,"count":3,"name":"imd","live":true}`)) + if err != nil { + t.Fatalf("asQuery() returned an unexpected error: %v", err) + } + for _, want := range []string{"lat=19.9975", "count=3", "name=imd", "live=true"} { + if !strings.Contains(got, want) { + t.Errorf("query %q is missing %q", got, want) + } + } +} + +func TestAsQueryHandlesAnEmptyMapping(t *testing.T) { + t.Parallel() + + got, err := asQuery([]byte(`{}`)) + if err != nil || got != "" { + t.Errorf("asQuery({}) = (%q, %v), want an empty query and no error", got, err) + } +} diff --git a/pkg/plugin/manager.go b/pkg/plugin/manager.go index 9623ae2e..38df7a97 100644 --- a/pkg/plugin/manager.go +++ b/pkg/plugin/manager.go @@ -700,6 +700,47 @@ func (m *Manager) Crawler(ctx context.Context, registry definition.RegistryLooku return crawler, nil } +// Mapper returns a Mapper instance based on the provided configuration. +func (m *Manager) Mapper(ctx context.Context, cfg *Config) (definition.Mapper, error) { + mp, err := provider[definition.MapperProvider](m.plugins, cfg.ID) + if err != nil { + return nil, fmt.Errorf("failed to load provider for %s: %w", cfg.ID, err) + } + mapper, closer, err := mp.New(ctx, cfg.Config) + if err != nil { + return nil, err + } + if closer != nil { + m.closers = append(m.closers, func() { + if err := closer(); err != nil { + log.Errorf(context.Background(), err, "Failed to close mapper plugin") + } + }) + } + return mapper, nil +} + +// ProviderStep returns a ProviderStep instance based on the provided +// configuration, handing it the registry and mapper it needs. +func (m *Manager) ProviderStep(ctx context.Context, registry definition.ProviderRecordLookup, mapper definition.Mapper, cfg *Config) (definition.Step, error) { + pp, err := provider[definition.ProviderStepProvider](m.plugins, cfg.ID) + if err != nil { + return nil, fmt.Errorf("failed to load provider for %s: %w", cfg.ID, err) + } + step, closer, err := pp.New(ctx, registry, mapper, cfg.Config) + if err != nil { + return nil, err + } + if closer != nil { + m.closers = append(m.closers, func() { + if err := closer(); err != nil { + log.Errorf(context.Background(), err, "Failed to close provider step plugin %s", cfg.ID) + } + }) + } + return step, nil +} + // Validator implements handler.PluginManager. func (m *Manager) Validator(ctx context.Context, cfg *Config) (definition.SchemaValidator, error) { panic("unimplemented") From 497c908813a0d039d1f34396c1b1c0b9585fc24d Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 31 Aug 2026 13:26:30 +0530 Subject: [PATCH 029/130] fix: read the updated registry schema contract [OpenAgriNet/engineering-tracker#41] The registry schemas at OpenAgriNet/discovery-service docs/registry/schemas.md changed shape. This reads the new one. Participant is flat. type -- node or upstream -- decides which fields apply, where the old shape wrapped them in a "node" or "upstream" object. baseUrl is one field for both, and role (BAP/BPP/NETWORK) is separate from type. A binding's actions carry their own mappings and their own status. Retiring one action is now one field on one entry, leaving the capability and every other action live; an inactive entry is skipped exactly as an absent one is. One mapping file per binding-action, holding both directions, replacing the per-direction pair. The halves are not independent -- the response mapping reads what the request mapping resolved into _local -- and two references hid that. A half that is absent or empty reports ErrNoTransform, which is a statement rather than an omission; a half that will not compile is an error, and the two must not collapse or an unmapped upstream answer would go out as a Beckn response. mappings stays a fully-qualified URL, carried verbatim. The documented contract specifies a repo-relative path resolved against an operator-configured root, and that is the safer shape -- it stops a registry row choosing which host this adapter fetches, compiles and runs a mapping from. It is deliberately not adopted yet: the network has not settled on a fixed location for published mappings, so the URL stays in the record and the local ProviderSchema pattern is relaxed to match. Who may write a registry row is therefore part of the mapper's threat model, and that is written down where the check lives. Also the contract's action budget defaults: timeoutMs 15000 and retryMax 0, and retryMax now counts retries rather than total attempts -- so an action that does not ask for retries is called exactly once. A retry on a non-idempotent action is a second booking. The registry's auth block is still not read, deliberately: an upstream's credential is the provider plugin's own configuration, and reading both would create two places that can disagree about how to authenticate a call. The captured-registry test is re-captured from the live registry in the new shape, and the end-to-end select through the local stack returns three mapped forecast resources. --- cmd/adapter/main_test.go | 10 + config/mappings/mausamgram/request.yaml | 19 - config/mappings/mausamgram/response.yaml | 92 ----- .../weather-observation.select.yaml | 106 ++++++ pkg/model/model.go | 27 +- pkg/plugin/definition/mapper.go | 22 +- .../implementation/jsonmapper/README.md | 90 +++-- .../implementation/jsonmapper/cmd/plugin.go | 1 + .../implementation/jsonmapper/jsonmapper.go | 158 ++++----- .../jsonmapper/jsonmapper_test.go | 331 +++++++++--------- .../mausamgram/cmd/plugin_test.go | 4 +- .../mausamgram/dispatch_test.go | 11 +- .../mausamgram/mappings_test.go | 48 ++- .../implementation/mausamgram/mausamgram.go | 51 ++- .../mausamgram/mausamgram_test.go | 68 +++- .../implementation/oanregistry/README.md | 39 ++- .../implementation/oanregistry/oanregistry.go | 73 ++-- .../oanregistry/oanregistry_test.go | 123 ++++--- .../oanregistry/providerrecord.go | 55 +-- .../oanregistry/providerrecord_test.go | 41 ++- 20 files changed, 734 insertions(+), 635 deletions(-) delete mode 100644 config/mappings/mausamgram/request.yaml delete mode 100644 config/mappings/mausamgram/response.yaml create mode 100644 config/mappings/mausamgram/weather-observation.select.yaml diff --git a/cmd/adapter/main_test.go b/cmd/adapter/main_test.go index ee0a57b8..f868f8b3 100644 --- a/cmd/adapter/main_test.go +++ b/cmd/adapter/main_test.go @@ -74,6 +74,16 @@ func (m *MockPluginManager) Cache(ctx context.Context, cfg *plugin.Config) (defi return nil, nil } +// Mapper returns a mock implementation of the Mapper interface. +func (m *MockPluginManager) Mapper(ctx context.Context, cfg *plugin.Config) (definition.Mapper, error) { + return nil, nil +} + +// ProviderStep returns a mock implementation of the provider Step interface. +func (m *MockPluginManager) ProviderStep(ctx context.Context, registry definition.ProviderRecordLookup, mapper definition.Mapper, cfg *plugin.Config) (definition.Step, error) { + return nil, nil +} + // Registry returns a mock implementation of the RegistryLookup interface. func (m *MockPluginManager) Registry(ctx context.Context, cache definition.Cache, cfg *plugin.Config) (definition.RegistryLookup, error) { return nil, nil diff --git a/config/mappings/mausamgram/request.yaml b/config/mappings/mausamgram/request.yaml deleted file mode 100644 index 4086949b..00000000 --- a/config/mappings/mausamgram/request.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Mausamgram: OAN requests -> the provider's own. -# -# Keyed by the Beckn action each mapping translates. -# -# An action declared with no mapping needs no request document built: this -# provider takes its parameters in the query string, the step has already -# resolved them, and putting them through a transform to arrive at the same two -# fields would buy nothing. The values the step resolved become the parameters. -# -# It is still declared, so this file remains the statement of what the capability -# serves. An action absent from here is one it does not serve at all -- a -# different thing, and refused. -# -# An action that DOES need a document -- a provider wanting a body in its own -# shape -- carries its transform here, reading: -# beckn the inbound Beckn payload -# _local whatever the step resolved before the call -actions: - select: "" diff --git a/config/mappings/mausamgram/response.yaml b/config/mappings/mausamgram/response.yaml deleted file mode 100644 index ef903204..00000000 --- a/config/mappings/mausamgram/response.yaml +++ /dev/null @@ -1,92 +0,0 @@ -# Mausamgram: the provider's own responses -> OAN. -# -# Keyed by the Beckn action each mapping PRODUCES, not the one that arrived: a -# select is answered by an on_select. Each file therefore names the actions it -# actually deals in. -# -# Every mapping reads: -# beckn the original request, for the context to echo and the offer to -# quote against -# _local what the provider step resolved before the call. The provider does -# not repeat it back, so the output's own coordinates have no other -# source -# response the provider's answer, in its own shape -actions: - on_select: | - ( - $lat := _local.lat; - $lon := _local.lon; - $days := [ - response.fcstday1, response.fcstday2, response.fcstday3, - response.fcstday4, response.fcstday5 - ]; - - $reading := function($name, $aggregation, $unit, $value) { - $exists($value) ? { - "parameter": $name, - "aggregation": $aggregation, - "unit": $unit, - "value": $value - } - }; - - { - "context": { - "version": beckn.context.version, - "action": "on_select", - "networkId": beckn.context.networkId, - "bapId": beckn.context.bapId, - "bapUri": beckn.context.bapUri, - "bppId": beckn.context.bppId, - "bppUri": beckn.context.bppUri, - "transactionId": beckn.context.transactionId, - "messageId": beckn.context.messageId, - "timestamp": $now() - }, - "message": { - "contract": { - "commitments": [ - { - "status": { - "descriptor": { "code": "QUOTED", "name": "Quoted" } - }, - "offer": beckn.message.contract.commitments[0].offer, - "resources": $map($days, function($day) { - { - "id": "res:mausamgram:forecast:" & $day.date, - "resourceAttributes": { - "@context": "https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld", - "@type": "openagrinet:WeatherObservation", - "informationMode": "Direct", - "observationType": "Forecast", - "subjectCategories": ["Weather"], - "source": { - "sourceId": "mausamgram", - "sourceName": "IMD Mausamgram NWP" - }, - "location": { - "type": "Point", - "coordinates": [$lon, $lat] - }, - "validity": { - "startsAt": $day.date, - "endsAt": $day.date - }, - "parameters": [ - $reading("Rainfall", "Total", "mm", $day.rain), - $reading("Temperature", "Minimum", "Cel", $day.tmin), - $reading("Temperature", "Maximum", "Cel", $day.tmax), - $reading("Humidity", "Minimum", "%", $day.rhmin), - $reading("Humidity", "Maximum", "%", $day.rhmax), - $reading("WindSpeed", "Average", "m/s", $day.wspd) - ], - "advisory": $day.weather_warning ? $day.weather_warning : $day.cloud_message - } - } - }) - } - ] - } - } - } - ) diff --git a/config/mappings/mausamgram/weather-observation.select.yaml b/config/mappings/mausamgram/weather-observation.select.yaml new file mode 100644 index 00000000..5d2659fe --- /dev/null +++ b/config/mappings/mausamgram/weather-observation.select.yaml @@ -0,0 +1,106 @@ +# Mausamgram, openagrinet:WeatherObservation, select. Both directions, one file. +# +# One file per binding-action rather than one per direction, because the two +# halves are not independent: the response half below reads _local, and the only +# reason _local holds a lat and a lon is that the request half put them there. +# Splitting them across two registry fields hid that dependency; this does not. +# +# The registry entry pointing here decides which action this serves, so nothing +# in the file names it. The filename's action segment must match that entry -- +# a mismatch would apply a correct mapping to the wrong call, silently. +# +# Both halves read: +# beckn the inbound Beckn payload -- the context to echo, the offer to +# quote against +# _local whatever the provider step resolved before the call +# and the response half additionally reads: +# response the provider's answer, in its own shape + +# An empty request half is a statement, not an omission: this provider takes its +# parameters in the query string, the step has already resolved them, and putting +# them through a transform to arrive at the same two fields would buy nothing. +# The step is told so explicitly (ErrNoTransform) rather than being handed an +# empty document. +request: "" + +# Keyed by direction, not by the action it produces: a select is answered by an +# on_select over the same HTTP round trip, so the callback is this half rather +# than an action of its own. +response: | + ( + $lat := _local.lat; + $lon := _local.lon; + $days := [ + response.fcstday1, response.fcstday2, response.fcstday3, + response.fcstday4, response.fcstday5 + ]; + + $reading := function($name, $aggregation, $unit, $value) { + $exists($value) ? { + "parameter": $name, + "aggregation": $aggregation, + "unit": $unit, + "value": $value + } + }; + + { + "context": { + "version": beckn.context.version, + "action": "on_select", + "networkId": beckn.context.networkId, + "bapId": beckn.context.bapId, + "bapUri": beckn.context.bapUri, + "bppId": beckn.context.bppId, + "bppUri": beckn.context.bppUri, + "transactionId": beckn.context.transactionId, + "messageId": beckn.context.messageId, + "timestamp": $now() + }, + "message": { + "contract": { + "commitments": [ + { + "status": { + "descriptor": { "code": "QUOTED", "name": "Quoted" } + }, + "offer": beckn.message.contract.commitments[0].offer, + "resources": $map($days, function($day) { + { + "id": "res:mausamgram:forecast:" & $day.date, + "resourceAttributes": { + "@context": "https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld", + "@type": "openagrinet:WeatherObservation", + "informationMode": "Direct", + "observationType": "Forecast", + "subjectCategories": ["Weather"], + "source": { + "sourceId": "mausamgram", + "sourceName": "IMD Mausamgram NWP" + }, + "location": { + "type": "Point", + "coordinates": [$lon, $lat] + }, + "validity": { + "startsAt": $day.date, + "endsAt": $day.date + }, + "parameters": [ + $reading("Rainfall", "Total", "mm", $day.rain), + $reading("Temperature", "Minimum", "Cel", $day.tmin), + $reading("Temperature", "Maximum", "Cel", $day.tmax), + $reading("Humidity", "Minimum", "%", $day.rhmin), + $reading("Humidity", "Maximum", "%", $day.rhmax), + $reading("WindSpeed", "Average", "m/s", $day.wspd) + ], + "advisory": $day.weather_warning ? $day.weather_warning : $day.cloud_message + } + } + }) + } + ] + } + } + } + ) diff --git a/pkg/model/model.go b/pkg/model/model.go index 8e105c52..5efb9ff5 100644 --- a/pkg/model/model.go +++ b/pkg/model/model.go @@ -71,14 +71,11 @@ type SubscriberRecord struct { MetaArrays map[string][]string // array-shaped meta values (e.g. NFH-014's meta.catalog_index_urls: [{url}, ...]) — kept separate from Meta rather than widening it to map[string]any, so every existing caller of Meta[key] keeps working unchanged } -// ProviderRecord is the resolved call plan for one provider capability: what to -// call, how to call it, and which mappings translate in and out. It is assembled -// from two registry records -- the capability binding and the participant that -// owns it -- so a caller resolves a whole plan in one lookup rather than knowing -// how the registry splits them. -// -// Mapping references are carried verbatim. They are fully-qualified URLs the -// mapper fetches; this type does not interpret them. +// ProviderRecord is the resolved call plan for one provider capability: where +// the provider is, and per Beckn action, how to reach it. It is assembled from +// two registry records -- the capability binding and the participant that owns +// it -- so a caller resolves a whole plan in one lookup rather than knowing how +// the registry splits them. type ProviderRecord struct { BindingKey string // "|" ParticipantID string @@ -90,13 +87,10 @@ type ProviderRecord struct { // Actions is the call plan per Beckn action. A capability serves several -- // a select that reads and a confirm that commits -- and they rarely share an - // endpoint or a method, so each carries its own. + // endpoint, a method or a mapping, so each carries its own. // // An action absent here is one this capability does not serve. Actions map[string]ActionPlan - - RequestMapping string - ResponseMapping string } // ActionPlan is how to make one action's upstream call. @@ -104,6 +98,15 @@ type ActionPlan struct { Method string Path string + // Mappings references one file carrying BOTH directions for this action. + // One file rather than two because the response mapping usually depends on + // what the request mapping did -- swapping GeoJSON coordinates into named + // lat/lon, say -- and splitting them across two references hides that. + // + // Carried verbatim: it is a URL the mapper fetches, and this type does not + // interpret it. + Mappings string + // TimeoutMs and RetryMax are this action's own budget, and are zero when the // registry does not set them -- the caller applies its defaults. They are // per action because a confirm that commits deserves a different budget from diff --git a/pkg/plugin/definition/mapper.go b/pkg/plugin/definition/mapper.go index e2850e96..458a2c8b 100644 --- a/pkg/plugin/definition/mapper.go +++ b/pkg/plugin/definition/mapper.go @@ -5,6 +5,17 @@ import ( "errors" ) +// Direction names which half of a mapping to run. A mapping file carries both, +// because the response half usually depends on what the request half did. +type Direction string + +const ( + // DirectionRequest translates an inbound payload into what the upstream wants. + DirectionRequest Direction = "request" + // DirectionResponse translates the upstream's answer back. + DirectionResponse Direction = "response" +) + // Mapper transforms a document with a mapping fetched from a reference. // // It exists so that translating between OAN's Beckn payloads and a provider's @@ -16,11 +27,12 @@ type Mapper interface { // Transform runs the mapping at mappingRef over input and returns the // result. // - // action is the Beckn action of the request being served. The mapping - // reference must identify itself as being for that action, and Transform - // refuses if it does not: running a select mapping over a confirm payload - // would otherwise succeed quietly and produce nonsense. - Transform(ctx context.Context, mappingRef, action string, input any) ([]byte, error) + // mappingRef is what the registry carries verbatim: the URL of one published + // file holding both directions. + // + // Which action the mapping serves is settled by the registry entry that + // named it, so only the direction is passed here. + Transform(ctx context.Context, mappingRef string, direction Direction, input any) ([]byte, error) } // MapperProvider initializes a new Mapper. diff --git a/pkg/plugin/implementation/jsonmapper/README.md b/pkg/plugin/implementation/jsonmapper/README.md index ccbdc1d2..5fbbf68e 100644 --- a/pkg/plugin/implementation/jsonmapper/README.md +++ b/pkg/plugin/implementation/jsonmapper/README.md @@ -15,7 +15,7 @@ mapper serve all of them. Its first caller is the OAN provider flow, where it translates between Beckn payloads and each provider's own request and response shapes -- so adding a -provider is two mapping files and a registry row rather than another +provider is one mapping file and a registry row rather than another transformation routine. It is **not** a pipeline step. A provider plugin holds it and calls it twice -- @@ -25,39 +25,58 @@ generic. ## Mapping files -A mapping file carries every action one capability serves, keyed by action name: +One file per binding-action, carrying **both directions**: ```yaml -actions: - select: | - { - "lat": _local.lat, - "lon": _local.lon - } - confirm: | - { - "booking_id": beckn.message.contract.commitments[0].id - } +# mappings/mausamgram/weather-observation.select.yaml +request: | + { + "lat": _local.lat, + "lon": _local.lon + } + +response: | + { + "rainfall": response.fcstday1.rain, + "at": { "type": "Point", "coordinates": [_local.lon, _local.lat] } + } ``` -**Request files are keyed by the action they translate** (`select`); **response -files by the action they produce** (`on_select`). Each file therefore names the -Beckn actions it actually deals in, and the filename carries no meaning — naming -a file after one action while it serves several would be worse than not naming -it. - -One file per direction rather than per action means a transaction walking -`select` then `confirm` pays one fetch, not one per step. An action the file does -not declare is refused, and the error names the ones it does serve. - -A mapping that fails to compile takes down only its own action: a typo in -`confirm` is no reason for `select` to stop being served. - -References come from the registry (`requestMapping` / `responseMapping` on a -capability binding) and are fully-qualified `http`/`https` URLs. Anything else -- -a bare path, a `file://`, a URL with no host -- is refused: references are -external input, and an unchecked one would let a registry record name a local -file and have the adapter read it. +**One file rather than two because the halves are not independent.** The response +above reads `_local`, and the only reason `_local` holds a lat and a lon is that +the request put them there. Splitting them across two registry fields hid that; +this does not. It also means the response leg of a round trip is already fetched +and compiled by the time it is needed. + +A half that is absent or empty is a *statement*, not an omission: it reports +`definition.ErrNoTransform`, so a caller that has nothing to build knows to build +its own request rather than sending an empty document. A half that will not +compile is a different thing and reported as an error — the two must not collapse, +or an unmapped upstream answer would go out as a Beckn response. + +A broken half takes down only itself: a typo in the response mapping is no reason +to stop making the call, and finding out on the way back beats finding out before +the call was made. + +Which action a file serves is settled by the registry entry pointing at it, so +nothing inside names it and the filename carries no meaning to this plugin. (The +registry contract does require the filename's action segment to match the action +it sits under — that is checked where the records are written.) + +### References + +The registry carries the full URL of one published file, and this plugin fetches +it verbatim. Anything that is not a fetchable `http`/`https` URL — a bare path, a +`file://`, a URL with no host — is refused: a reference is external input, and an +unchecked one would let a registry record name a local file and have the adapter +read it. + +**What that check cannot do is constrain which host.** A registry record chooses +that, and this plugin fetches, *compiles* and runs what comes back. So who may +write a registry record is part of this plugin's threat model. (A reference +carried as a path under an operator-configured root would close that off; the +network has not settled on a fixed location for published mappings, so the URL +stays in the record for now.) ## What a mapping can read @@ -71,13 +90,12 @@ file and have the adapter read it. rarely repeats what it was asked, so values resolved before the call are often the only source for them in the output — the coordinates of a forecast, say. -## Why the action is a key, not a convention +## Why the direction is a parameter, not a convention -Nothing else in the pipeline knows the action. A binding key is -`participantId|capabilityCode` and carries none, so without this a capability -publishing one mapping would run it for every action that reached it — a -`confirm` served by a `select` mapping, succeeding quietly and producing -nonsense. +The caller makes one round trip and needs both halves of it, and nothing in the +file distinguishes them by position. Passing the direction explicitly is what +keeps a response mapping from ever being applied to an outbound request — +which would succeed quietly and produce nonsense. Making the action a key in the file rather than a part of its name means the file states which actions it serves, instead of a convention someone has to remember. diff --git a/pkg/plugin/implementation/jsonmapper/cmd/plugin.go b/pkg/plugin/implementation/jsonmapper/cmd/plugin.go index dda9de19..a58ea44e 100644 --- a/pkg/plugin/implementation/jsonmapper/cmd/plugin.go +++ b/pkg/plugin/implementation/jsonmapper/cmd/plugin.go @@ -20,6 +20,7 @@ var newMapperFunc = jsonmapper.New // parseConfig turns the plugin config map into a typed Config. Anything absent // is left zero: jsonmapper.New applies the defaults, so they live in one place. + func (o jsonMapperProvider) parseConfig(config map[string]string) (*jsonmapper.Config, error) { cfg := &jsonmapper.Config{} diff --git a/pkg/plugin/implementation/jsonmapper/jsonmapper.go b/pkg/plugin/implementation/jsonmapper/jsonmapper.go index dfe159ac..afbcfb80 100644 --- a/pkg/plugin/implementation/jsonmapper/jsonmapper.go +++ b/pkg/plugin/implementation/jsonmapper/jsonmapper.go @@ -17,7 +17,6 @@ import ( "io" "net/http" "net/url" - "sort" "strings" "sync" "time" @@ -43,18 +42,19 @@ const ( // rather than a fault of this adapter. const codeAdaptationFailed = "SCH_SCHEMA_ADAPTATION_FAILED" -// mappingFile is the published form of a mapping: every action one capability -// serves, in one file, keyed by action name. +// mappingFile is the published form of a mapping: one binding-action, both +// directions. // -// Request files are keyed by the action they translate ("select"); response -// files by the action they produce ("on_select"). Each file therefore names the -// Beckn actions it deals in, and the filename says nothing -- naming a file -// after one action while it serves several would be worse than not naming it. +// One file rather than two because the response mapping usually depends on what +// the request mapping did -- swapping GeoJSON coordinates into named lat and lon, +// say -- and splitting them across two registry fields hides that. Which action +// the file serves is decided by the registry entry that points at it, so nothing +// inside needs to name it. // -// One file per direction rather than per action means a transaction walking -// select then confirm pays one fetch, not one per step. +// An empty request is a statement rather than an omission; see ErrNoTransform. type mappingFile struct { - Actions map[string]string `yaml:"actions"` + Request string `yaml:"request"` + Response string `yaml:"response"` } // Config holds configuration parameters for the mapper. @@ -96,20 +96,21 @@ type Config struct { // compiled expressions would remove even that, and is the upgrade if one // mapping ever becomes hot enough to matter. type cacheEntry struct { - // actions holds one compiled mapping per action the file serves. A file is - // fetched and compiled as a whole, so every action it declares is ready - // after the first request for any of them. - actions map[string]*compiledAction + // directions holds the compiled halves the file carries. A file is fetched + // and compiled as a whole, so both are ready after the first request for + // either. + directions map[definition.Direction]*compiledMapping // err is a failure that applies to the whole file -- it could not be // fetched, or not parsed -- as opposed to one action failing to compile. err error expiresAt time.Time } -// compiledAction is one action's mapping, or the failure that stopped it -// compiling. Failures are held per action deliberately: a typo in confirm is no -// reason for select to stop being served. -type compiledAction struct { +// compiledMapping is one half of a mapping, or the failure that stopped it +// compiling. Failures are held per half deliberately: a typo in the response +// mapping is no reason for the request half to stop working, and finding out on +// the way out beats finding out before the call was even made. +type compiledMapping struct { expression jsonata.Expression evaluating *sync.Mutex err error @@ -174,10 +175,11 @@ func applyDefaults(cfg *Config) { } } -// Transform runs the mapping at mappingRef over input. -func (m *Mapper) Transform(ctx context.Context, mappingRef, action string, input any) ([]byte, error) { - if action == "" { - return nil, fmt.Errorf("jsonmapper: cannot resolve a mapping in %q without an action", mappingRef) +// Transform runs one direction of the mapping at mappingRef over input. +func (m *Mapper) Transform(ctx context.Context, mappingRef string, direction definition.Direction, input any) ([]byte, error) { + if direction != definition.DirectionRequest && direction != definition.DirectionResponse { + return nil, fmt.Errorf("jsonmapper: mapping %q: %q is not a direction; want %q or %q", + mappingRef, direction, definition.DirectionRequest, definition.DirectionResponse) } entry, err := m.compiled(ctx, mappingRef) @@ -185,28 +187,14 @@ func (m *Mapper) Transform(ctx context.Context, mappingRef, action string, input return nil, err } - mapping, served := entry.actions[action] - if !served { - // Naming what the file does serve turns a deploy mistake into a one-line - // fix, rather than a hunt through registry rows. - return nil, fmt.Errorf("jsonmapper: mapping %q does not serve action %q; it serves %s", - mappingRef, action, strings.Join(servedActions(entry), ", ")) + mapping, present := entry.directions[direction] + if !present { + return nil, fmt.Errorf("jsonmapper: mapping %q carries no %s half", mappingRef, direction) } if mapping.err != nil { return nil, mapping.err } - return m.evaluate(ctx, mapping, mappingRef, action, input) -} - -// servedActions lists the actions a file serves, in a stable order so the same -// mistake reads the same way twice. -func servedActions(entry cacheEntry) []string { - names := make([]string, 0, len(entry.actions)) - for name := range entry.actions { - names = append(names, name) - } - sort.Strings(names) - return names + return m.evaluate(ctx, mapping, mappingRef, direction, input) } // compiled returns the compiled mapping for a reference, fetching and compiling @@ -216,8 +204,8 @@ func (m *Mapper) compiled(ctx context.Context, mappingRef string) (cacheEntry, e return entry, entry.err } - actions, err := m.fetchAndCompile(ctx, mappingRef) - return m.remember(mappingRef, actions, err), err + directions, err := m.fetchAndCompile(ctx, mappingRef) + return m.remember(mappingRef, directions, err), err } // cached returns a live cache entry, if there is one. @@ -235,15 +223,15 @@ func (m *Mapper) cached(mappingRef string) (cacheEntry, bool) { // remember caches a compiled mapping, or the failure that stopped it compiling. // A failure gets the shorter TTL: it should stop hammering a broken reference // without outlasting the fix. -func (m *Mapper) remember(mappingRef string, actions map[string]*compiledAction, err error) cacheEntry { +func (m *Mapper) remember(mappingRef string, directions map[definition.Direction]*compiledMapping, err error) cacheEntry { ttl := m.config.CacheTTL if err != nil { ttl = m.config.NegativeTTL } entry := cacheEntry{ - actions: actions, - err: err, - expiresAt: time.Now().Add(ttl), + directions: directions, + err: err, + expiresAt: time.Now().Add(ttl), } m.mu.Lock() @@ -272,43 +260,41 @@ func (m *Mapper) cachedCount() int { } // fetchAndCompile retrieves a mapping and turns it into a runnable expression. -func (m *Mapper) fetchAndCompile(ctx context.Context, mappingRef string) (map[string]*compiledAction, error) { +func (m *Mapper) fetchAndCompile(ctx context.Context, mappingRef string) (map[definition.Direction]*compiledMapping, error) { body, err := m.fetch(ctx, mappingRef) if err != nil { return nil, err } - sources, err := parseActions(body) + file, err := parseMapping(body) if err != nil { return nil, fmt.Errorf("jsonmapper: mapping %q: %w", mappingRef, err) } - // Every action is compiled now rather than on first use, so one fetch - // leaves the whole file ready. A compile failure is recorded against its own - // action and goes no further than that action. - actions := make(map[string]*compiledAction, len(sources)) - for action, source := range sources { - actions[action] = m.compileAction(ctx, mappingRef, action, source) - } - log.Debugf(ctx, "JSON mapper compiled %d action(s) from mapping: %s", len(actions), mappingRef) - return actions, nil + // Both halves are compiled now rather than on first use, so one fetch leaves + // the file ready in both directions. A compile failure is recorded against + // its own half and goes no further. + directions := make(map[definition.Direction]*compiledMapping, 2) + directions[definition.DirectionRequest] = m.compileMapping(ctx, mappingRef, definition.DirectionRequest, file.Request) + directions[definition.DirectionResponse] = m.compileMapping(ctx, mappingRef, definition.DirectionResponse, file.Response) + log.Debugf(ctx, "JSON mapper compiled mapping: %s", mappingRef) + return directions, nil } -// compileAction compiles one action's mapping, keeping any failure local to it. -func (m *Mapper) compileAction(ctx context.Context, mappingRef, action, source string) *compiledAction { +// compileMapping compiles one half, keeping any failure local to it. +func (m *Mapper) compileMapping(ctx context.Context, mappingRef string, direction definition.Direction, source string) *compiledMapping { if strings.TrimSpace(source) == "" { - // Declared, but with nothing to build. That is a statement rather than an - // omission -- see definition.ErrNoTransform -- so it is held as this - // action's outcome and reported to whoever asks for it, while every other - // action in the file is unaffected. - return &compiledAction{err: fmt.Errorf("jsonmapper: mapping %q action %q: %w", - mappingRef, action, definition.ErrNoTransform)} + // Present but empty. That is a statement rather than an omission -- see + // definition.ErrNoTransform -- so it is held as this half's outcome and + // reported to whoever asks for it, leaving the other half unaffected. + return &compiledMapping{err: fmt.Errorf("jsonmapper: mapping %q %s half: %w", + mappingRef, direction, definition.ErrNoTransform)} } expression, err := m.instance.Compile(source, false) if err != nil { - log.Errorf(ctx, err, "JSON mapper could not compile action %s of %s: %v", action, mappingRef, err) - return &compiledAction{err: fmt.Errorf("jsonmapper: mapping %q action %q failed to compile: %w", mappingRef, action, err)} + log.Errorf(ctx, err, "JSON mapper could not compile the %s half of %s: %v", direction, mappingRef, err) + return &compiledMapping{err: fmt.Errorf("jsonmapper: mapping %q %s half failed to compile: %w", mappingRef, direction, err)} } - return &compiledAction{expression: expression, evaluating: &sync.Mutex{}} + return &compiledMapping{expression: expression, evaluating: &sync.Mutex{}} } // fetch retrieves a mapping's bytes, bounded in both time and size. @@ -348,9 +334,13 @@ func (m *Mapper) fetch(ctx context.Context, mappingRef string) ([]byte, error) { // verifyFetchable rejects a reference this mapper will not retrieve. // -// References come from the registry, which makes them external input: an -// unchecked one would let a registry record name a file path or an internal -// scheme and have the adapter read it. +// A reference is a fully-qualified http or https URL, carried verbatim from the +// registry. That makes it external input, so it is checked rather than trusted: +// without this a registry record could name a file path or an internal scheme +// and have the adapter read it. What the check cannot constrain is WHICH host -- +// a registry record chooses that, and this mapper compiles and runs what comes +// back from it. Who may write a registry record is therefore part of this +// plugin's threat model, not an unrelated concern. func verifyFetchable(mappingRef string) error { if mappingRef == "" { return errors.New("jsonmapper: mapping reference is empty") @@ -368,16 +358,18 @@ func verifyFetchable(mappingRef string) error { return nil } -// parseActions reads the actions a published mapping serves. -func parseActions(body []byte) (map[string]string, error) { +// parseMapping reads the two halves a published mapping carries. +func parseMapping(body []byte) (mappingFile, error) { var file mappingFile if err := yaml.Unmarshal(body, &file); err != nil { - return nil, fmt.Errorf("could not be parsed: %w", err) + return mappingFile{}, fmt.Errorf("could not be parsed: %w", err) } - if len(file.Actions) == 0 { - return nil, errors.New("serves no actions") + if strings.TrimSpace(file.Request) == "" && strings.TrimSpace(file.Response) == "" { + // Neither half present at all -- not an empty request, which is + // meaningful, but a file that says nothing. + return mappingFile{}, errors.New("carries neither a request nor a response half") } - return file.Actions, nil + return file, nil } // marshalInput renders the named inputs a mapping reads -- beckn, _local and, @@ -392,24 +384,24 @@ func marshalInput(input any) ([]byte, error) { } // evaluate runs a compiled mapping over the input document. -func (m *Mapper) evaluate(ctx context.Context, mapping *compiledAction, mappingRef, action string, input any) ([]byte, error) { +func (m *Mapper) evaluate(ctx context.Context, mapping *compiledMapping, mappingRef string, direction definition.Direction, input any) ([]byte, error) { document, err := marshalInput(input) if err != nil { return nil, fmt.Errorf("jsonmapper: mapping %q: %w", mappingRef, err) } - // See compiledAction: Evaluate mutates the expression, so one action's - // mapping serves one request at a time. Other actions in the same file are - // unaffected, and marshalling above is deliberately outside the lock. + // See compiledMapping: Evaluate mutates the expression, so one half serves + // one request at a time. The other half is unaffected, and marshalling above + // is deliberately outside the lock. mapping.evaluating.Lock() result, err := mapping.expression.Evaluate(document, nil) mapping.evaluating.Unlock() if err != nil { // The mapping is valid and the payload is not what it expected, so this // is the caller's request being wrong rather than this adapter failing. - log.Errorf(ctx, err, "JSON mapping %s action %s failed to evaluate: %v", mappingRef, action, err) + log.Errorf(ctx, err, "JSON mapping %s %s half failed to evaluate: %v", mappingRef, direction, err) return nil, model.NewBadReqErr(codeAdaptationFailed, - fmt.Errorf("mapping %q action %q could not be applied: %w", mappingRef, action, err)) + fmt.Errorf("mapping %q %s half could not be applied: %w", mappingRef, direction, err)) } return result, nil } diff --git a/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go b/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go index 6aaac5e6..0d410382 100644 --- a/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go +++ b/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go @@ -15,19 +15,20 @@ import ( "github.com/beckn-one/beckn-onix/pkg/plugin/definition" ) -// twoActionMapping is the published form: one file, every action it serves. -// Request files are keyed by the action they translate; response files by the -// action they produce, which is why on_select rather than select appears there. -const twoActionMapping = `actions: - select: | - { "lat": _local.lat, "txn": beckn.context.transactionId } - confirm: | - { "booking": beckn.context.messageId } +// bothDirections is the published form: one binding-action, both halves. The +// response half reads _local, which is what makes one file the right unit -- +// it only works because the request half put lat and lon there. +const bothDirections = `request: | + { "lat": _local.lat, "txn": beckn.context.transactionId } + +response: | + { "lat": _local.lat, "txn": beckn.context.transactionId, "rain": response.fcstday1.rain } ` -const responseMapping = `actions: - on_select: | - { "lat": _local.lat, "txn": beckn.context.transactionId, "rain": response.fcstday1.rain } +// requestOnly is the shape of a mapping for an action whose answer needs no +// translation, or whose response half has not been written yet. +const requestOnly = `request: | + { "lat": _local.lat } ` func requestInput() map[string]any { @@ -37,6 +38,12 @@ func requestInput() map[string]any { } } +func responseInput() map[string]any { + input := requestInput() + input["response"] = map[string]any{"fcstday1": map[string]any{"rain": 12.4}} + return input +} + // newMappingServer serves body at every path and counts what was asked for. func newMappingServer(t *testing.T, body string, fetches *atomic.Int32) *httptest.Server { t.Helper() @@ -70,19 +77,22 @@ func newTestMapper(t *testing.T, tweak ...func(*Config)) *Mapper { return mapper } -// ref builds a mapping reference. The filename carries no meaning -- the file's -// own keys say which actions it serves. -func ref(base string) string { return base + "/mappings/anything.yaml" } +// ref is what the registry carries: the fully-qualified URL of one published +// file. Which action it serves is decided by the registry entry pointing at it, +// so the name carries no meaning here. +func ref(base string) string { + return base + "/mappings/mausamgram/weather-observation.select.yaml" +} // --- transformation -------------------------------------------------------- -func TestTransformRunsTheMappingForTheRequestedAction(t *testing.T) { +func TestTransformRunsTheRequestHalf(t *testing.T) { t.Parallel() - srv := newMappingServer(t, twoActionMapping, nil) + srv := newMappingServer(t, bothDirections, nil) defer srv.Close() - got, err := newTestMapper(t).Transform(context.Background(), ref(srv.URL), "select", requestInput()) + got, err := newTestMapper(t).Transform(context.Background(), ref(srv.URL), definition.DirectionRequest, requestInput()) if err != nil { t.Fatalf("Transform() returned an unexpected error: %v", err) } @@ -99,44 +109,17 @@ func TestTransformRunsTheMappingForTheRequestedAction(t *testing.T) { } } -// One file, several actions, each reached by name. This is what the format -// exists for: adding an action is a new key, not a new file. -func TestTransformPicksTheRightActionFromOneFile(t *testing.T) { +// The response half reads the upstream answer under response, and still reads +// _local -- which is the argument for one file: the provider does not repeat the +// point it was asked about, so the answer is only mappable next to the request +// that produced it. +func TestTransformRunsTheResponseHalfAlongsideTheRequest(t *testing.T) { t.Parallel() - srv := newMappingServer(t, twoActionMapping, nil) + srv := newMappingServer(t, bothDirections, nil) defer srv.Close() - mapper := newTestMapper(t) - selected, err := mapper.Transform(context.Background(), ref(srv.URL), "select", requestInput()) - if err != nil { - t.Fatalf("select: %v", err) - } - confirmed, err := mapper.Transform(context.Background(), ref(srv.URL), "confirm", requestInput()) - if err != nil { - t.Fatalf("confirm: %v", err) - } - - if !strings.Contains(string(selected), `"lat"`) { - t.Errorf("select produced %s, want the select mapping's output", selected) - } - if !strings.Contains(string(confirmed), `"booking"`) { - t.Errorf("confirm produced %s, want the confirm mapping's output", confirmed) - } -} - -// The response leg is keyed by the action it produces, so a caller asks for -// on_select rather than select. -func TestTransformExposesTheResponseAlongsideTheRequest(t *testing.T) { - t.Parallel() - - srv := newMappingServer(t, responseMapping, nil) - defer srv.Close() - - input := requestInput() - input["response"] = map[string]any{"fcstday1": map[string]any{"rain": 12.4}} - - got, err := newTestMapper(t).Transform(context.Background(), ref(srv.URL), "on_select", input) + got, err := newTestMapper(t).Transform(context.Background(), ref(srv.URL), definition.DirectionResponse, responseInput()) if err != nil { t.Fatalf("Transform() returned an unexpected error: %v", err) } @@ -155,107 +138,106 @@ func TestTransformExposesTheResponseAlongsideTheRequest(t *testing.T) { } } -// --- declared but empty ----------------------------------------------------- - -// The ordinary case for a provider taking query parameters: the action is -// declared so the file still says what the capability serves, but there is no -// document to build. -func TestTransformReportsADeclaredButEmptyAction(t *testing.T) { +// The two halves are separate expressions, not one applied twice. +func TestTransformKeepsTheHalvesApart(t *testing.T) { t.Parallel() - mapping := `actions: - select: "" - confirm: | - { "booking": beckn.context.messageId } + mapping := `request: | + { "leg": "out" } + +response: | + { "leg": "back" } ` srv := newMappingServer(t, mapping, nil) defer srv.Close() mapper := newTestMapper(t) - _, err := mapper.Transform(context.Background(), ref(srv.URL), "select", requestInput()) - if !errors.Is(err, definition.ErrNoTransform) { - t.Errorf("expected ErrNoTransform, got %v", err) + out, err := mapper.Transform(context.Background(), ref(srv.URL), definition.DirectionRequest, requestInput()) + if err != nil { + t.Fatalf("request: %v", err) + } + back, err := mapper.Transform(context.Background(), ref(srv.URL), definition.DirectionResponse, responseInput()) + if err != nil { + t.Fatalf("response: %v", err) } - // Its neighbours are unaffected: one action needing no transform says - // nothing about the rest of the file. - if _, err := mapper.Transform(context.Background(), ref(srv.URL), "confirm", requestInput()); err != nil { - t.Errorf("a sibling action must still be served: %v", err) + if !strings.Contains(string(out), `"out"`) { + t.Errorf("request produced %s, want the request half's output", out) + } + if !strings.Contains(string(back), `"back"`) { + t.Errorf("response produced %s, want the response half's output", back) } } -// Declared-but-empty and absent are different facts and must not collapse: the -// first says "I serve this, build it yourself", the second says "I do not serve -// this at all". Confusing them would send an empty request where the answer -// should have been a refusal. -func TestTransformSeparatesAnEmptyActionFromAnAbsentOne(t *testing.T) { - t.Parallel() +// --- a half that supplies no transform -------------------------------------- - srv := newMappingServer(t, "actions:\n select: \"\"\n", nil) - defer srv.Close() - mapper := newTestMapper(t) +// The ordinary case for a provider taking query parameters, or one whose answer +// is already in shape: the file carries the other half, and this one is reported +// as its own sentinel so a caller that does not handle it fails loudly rather +// than sending an empty document. +func TestTransformReportsAHalfWithNoTransform(t *testing.T) { + t.Parallel() - if _, err := mapper.Transform(context.Background(), ref(srv.URL), "select", requestInput()); !errors.Is(err, definition.ErrNoTransform) { - t.Errorf("declared-but-empty should report ErrNoTransform, got %v", err) - } - err := func() error { - _, err := mapper.Transform(context.Background(), ref(srv.URL), "confirm", requestInput()) - return err - }() - if errors.Is(err, definition.ErrNoTransform) { - t.Error("an absent action must not report ErrNoTransform -- it is not served at all") - } - if err == nil { - t.Error("expected an absent action to be refused") + testCases := []struct { + name string + mapping string + }{ + {"the half is absent", requestOnly}, + {"the half is present and empty", "request: |\n { \"lat\": _local.lat }\nresponse: \"\"\n"}, } -} -// --- an action the file does not serve -------------------------------------- + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() -// A capability that publishes no mapping for an action does not serve it. The -// refusal has to be clear, because the alternative -- running whichever mapping -// happened to be there -- succeeds quietly and produces nonsense. -func TestTransformRefusesAnActionTheFileDoesNotServe(t *testing.T) { - t.Parallel() + srv := newMappingServer(t, tc.mapping, nil) + defer srv.Close() + mapper := newTestMapper(t) - srv := newMappingServer(t, twoActionMapping, nil) - defer srv.Close() + _, err := mapper.Transform(context.Background(), ref(srv.URL), definition.DirectionResponse, responseInput()) + if !errors.Is(err, definition.ErrNoTransform) { + t.Errorf("expected ErrNoTransform, got %v", err) + } - _, err := newTestMapper(t).Transform(context.Background(), ref(srv.URL), "init", requestInput()) - if err == nil { - t.Fatal("expected an unserved action to be refused") - } - if !strings.Contains(err.Error(), "init") { - t.Errorf("error %q should name the action that was asked for", err) - } - // Naming what it does serve turns a deploy mistake into a one-line fix. - if !strings.Contains(err.Error(), "select") { - t.Errorf("error %q should say which actions the mapping does serve", err) + // The other half is unaffected: one direction needing no transform + // says nothing about the other. + if _, err := mapper.Transform(context.Background(), ref(srv.URL), definition.DirectionRequest, requestInput()); err != nil { + t.Errorf("the other half must still be served: %v", err) + } + }) } } -func TestTransformRefusesAnEmptyAction(t *testing.T) { +// --- direction validation --------------------------------------------------- + +// A direction outside the two is a caller bug, not a mapping problem, and must +// not be read as either half. +func TestTransformRefusesAnUnknownDirection(t *testing.T) { t.Parallel() - srv := newMappingServer(t, twoActionMapping, nil) + srv := newMappingServer(t, bothDirections, nil) defer srv.Close() + mapper := newTestMapper(t) - if _, err := newTestMapper(t).Transform(context.Background(), ref(srv.URL), "", requestInput()); err == nil { - t.Fatal("expected an empty action to be refused") + for _, direction := range []definition.Direction{"", "on_select", "REQUEST"} { + if _, err := mapper.Transform(context.Background(), ref(srv.URL), direction, requestInput()); err == nil { + t.Errorf("expected direction %q to be refused", direction) + } } } -// The filename says nothing. Naming a file after one action while it serves -// several would be worse than naming it nothing at all. +// The filename says nothing: the registry entry that points at a file decides +// which action it serves, so a mapper reading meaning into the path would give +// the same file two answers. func TestTransformIgnoresTheFilename(t *testing.T) { t.Parallel() - srv := newMappingServer(t, twoActionMapping, nil) + srv := newMappingServer(t, bothDirections, nil) defer srv.Close() mapper := newTestMapper(t) for _, name := range []string{"/anything.yaml", "/confirm.yaml", "/x/y/z"} { - if _, err := mapper.Transform(context.Background(), srv.URL+name, "select", requestInput()); err != nil { + if _, err := mapper.Transform(context.Background(), srv.URL+name, definition.DirectionRequest, requestInput()); err != nil { t.Errorf("Transform(%q) returned an unexpected error: %v", name, err) } } @@ -263,12 +245,18 @@ func TestTransformIgnoresTheFilename(t *testing.T) { // --- reference validation --------------------------------------------------- +// A reference comes from the registry, so it is external input rather than +// something to trust. This cannot constrain WHICH host -- a registry record +// chooses that -- but it can refuse a reference that is not a fetchable http +// URL at all, which is what stops a record naming a local file and having the +// adapter read it. func TestTransformRefusesAnUnusableReference(t *testing.T) { t.Parallel() testCases := []struct{ name, ref string }{ {"empty", ""}, {"a bare path with no scheme", "/mappings/select.yaml"}, + {"a relative path", "mappings/select.yaml"}, {"a file url", "file:///etc/passwd"}, {"a scheme that is not http", "ftp://example.com/select.yaml"}, {"no host", "http:///select.yaml"}, @@ -278,7 +266,7 @@ func TestTransformRefusesAnUnusableReference(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - if _, err := newTestMapper(t).Transform(context.Background(), tc.ref, "select", requestInput()); err == nil { + if _, err := newTestMapper(t).Transform(context.Background(), tc.ref, definition.DirectionRequest, requestInput()); err == nil { t.Errorf("expected reference %q to be refused", tc.ref) } }) @@ -297,9 +285,12 @@ func TestTransformReportsAFailedFetch(t *testing.T) { }{ {name: "a not-found status", status: http.StatusNotFound}, {name: "a server error status", status: http.StatusInternalServerError}, - {name: "malformed yaml", status: http.StatusOK, body: "actions: [unclosed"}, - {name: "no actions key", status: http.StatusOK, body: "other: value\n"}, - {name: "an empty actions map", status: http.StatusOK, body: "actions: {}\n"}, + {name: "malformed yaml", status: http.StatusOK, body: "request: [unclosed"}, + {name: "neither half", status: http.StatusOK, body: "other: value\n"}, + {name: "an empty document", status: http.StatusOK, body: "\n"}, + // Both halves present and empty is a file that says nothing, and is + // refused whole rather than per half -- there is no half left to serve. + {name: "both halves empty", status: http.StatusOK, body: "request: \"\"\nresponse: \"\"\n"}, } for _, tc := range testCases { @@ -315,40 +306,44 @@ func TestTransformReportsAFailedFetch(t *testing.T) { })) defer srv.Close() - if _, err := newTestMapper(t).Transform(context.Background(), ref(srv.URL), "select", requestInput()); err == nil { + if _, err := newTestMapper(t).Transform(context.Background(), ref(srv.URL), definition.DirectionRequest, requestInput()); err == nil { t.Error("expected an error") } }) } } -// One unusable action must not take the rest of the file down with it: a typo -// in confirm is no reason for select to stop being served. -func TestTransformIsolatesABrokenAction(t *testing.T) { +// One unusable half must not take the other down with it: a typo in the response +// mapping is no reason to stop making the call, and finding out on the way back +// beats finding out before the call was made. +func TestTransformIsolatesABrokenHalf(t *testing.T) { t.Parallel() - mapping := `actions: - select: | - { "lat": _local.lat } - confirm: | - {{{ - init: "" + mapping := `request: | + { "lat": _local.lat } + +response: | + {{{ ` srv := newMappingServer(t, mapping, nil) defer srv.Close() mapper := newTestMapper(t) - if _, err := mapper.Transform(context.Background(), ref(srv.URL), "select", requestInput()); err != nil { - t.Errorf("a healthy action must still be served: %v", err) + if _, err := mapper.Transform(context.Background(), ref(srv.URL), definition.DirectionRequest, requestInput()); err != nil { + t.Errorf("the healthy half must still be served: %v", err) } - if _, err := mapper.Transform(context.Background(), ref(srv.URL), "confirm", requestInput()); err == nil { - t.Error("expected an uncompilable action to be refused") + err := func() error { + _, err := mapper.Transform(context.Background(), ref(srv.URL), definition.DirectionResponse, responseInput()) + return err + }() + if err == nil { + t.Fatal("expected an uncompilable half to be refused") } - // An action declared with no mapping is a statement, not a fault: the caller - // builds that request itself. It is reported as its own sentinel so a caller - // that does not handle it fails loudly rather than sending nothing. - if _, err := mapper.Transform(context.Background(), ref(srv.URL), "init", requestInput()); !errors.Is(err, definition.ErrNoTransform) { - t.Errorf("expected ErrNoTransform for a declared-but-empty action, got %v", err) + // A half that will not compile is broken, not absent, and must not be read + // as "supplies no transform" -- that would send the upstream answer through + // unmapped. + if errors.Is(err, definition.ErrNoTransform) { + t.Error("an uncompilable half must not report ErrNoTransform") } } @@ -357,12 +352,12 @@ func TestTransformIsolatesABrokenAction(t *testing.T) { func TestTransformEnforcesASizeCap(t *testing.T) { t.Parallel() - oversized := "actions:\n select: |\n " + strings.Repeat("x", 2048) + "\n" + oversized := "request: |\n " + strings.Repeat("x", 2048) + "\n" srv := newMappingServer(t, oversized, nil) defer srv.Close() mapper := newTestMapper(t, func(c *Config) { c.MaxMappingBytes = 512 }) - if _, err := mapper.Transform(context.Background(), ref(srv.URL), "select", requestInput()); err == nil { + if _, err := mapper.Transform(context.Background(), ref(srv.URL), definition.DirectionRequest, requestInput()); err == nil { t.Fatal("expected an oversized mapping to be refused") } } @@ -383,7 +378,7 @@ func TestTransformBoundsTheFetch(t *testing.T) { done := make(chan error, 1) go func() { - _, err := mapper.Transform(context.Background(), ref(srv.URL), "select", requestInput()) + _, err := mapper.Transform(context.Background(), ref(srv.URL), definition.DirectionRequest, requestInput()) done <- err }() @@ -405,12 +400,12 @@ func TestTransformCompilesEachMappingOnce(t *testing.T) { t.Parallel() var fetches atomic.Int32 - srv := newMappingServer(t, twoActionMapping, &fetches) + srv := newMappingServer(t, bothDirections, &fetches) defer srv.Close() mapper := newTestMapper(t) for i := 0; i < 3; i++ { - if _, err := mapper.Transform(context.Background(), ref(srv.URL), "select", requestInput()); err != nil { + if _, err := mapper.Transform(context.Background(), ref(srv.URL), definition.DirectionRequest, requestInput()); err != nil { t.Fatalf("Transform() returned an unexpected error: %v", err) } } @@ -419,24 +414,24 @@ func TestTransformCompilesEachMappingOnce(t *testing.T) { } } -// One fetch serves every action in the file. This is the whole reason a file -// holds several: a transaction walking select then confirm pays one round trip, -// not one per action. -func TestTransformFetchesOnceForEveryActionInAFile(t *testing.T) { +// One fetch serves both halves. This is the practical gain of one file: the +// response leg of a round trip does not pay a second round trip to be mapped. +func TestTransformFetchesOnceForBothHalves(t *testing.T) { t.Parallel() var fetches atomic.Int32 - srv := newMappingServer(t, twoActionMapping, &fetches) + srv := newMappingServer(t, bothDirections, &fetches) defer srv.Close() mapper := newTestMapper(t) - for _, action := range []string{"select", "confirm", "select"} { - if _, err := mapper.Transform(context.Background(), ref(srv.URL), action, requestInput()); err != nil { - t.Fatalf("%s: %v", action, err) - } + if _, err := mapper.Transform(context.Background(), ref(srv.URL), definition.DirectionRequest, requestInput()); err != nil { + t.Fatalf("request: %v", err) + } + if _, err := mapper.Transform(context.Background(), ref(srv.URL), definition.DirectionResponse, responseInput()); err != nil { + t.Fatalf("response: %v", err) } if got := fetches.Load(); got != 1 { - t.Errorf("fetched %d times, want 1 -- a second action refetched the file", got) + t.Errorf("fetched %d times, want 1 -- the response half refetched the file", got) } } @@ -445,12 +440,12 @@ func TestTransformCachesPerReference(t *testing.T) { t.Parallel() var fetches atomic.Int32 - srv := newMappingServer(t, twoActionMapping, &fetches) + srv := newMappingServer(t, bothDirections, &fetches) defer srv.Close() mapper := newTestMapper(t) for _, r := range []string{srv.URL + "/a.yaml", srv.URL + "/b.yaml"} { - if _, err := mapper.Transform(context.Background(), r, "select", requestInput()); err != nil { + if _, err := mapper.Transform(context.Background(), r, definition.DirectionRequest, requestInput()); err != nil { t.Fatalf("Transform() returned an unexpected error: %v", err) } } @@ -473,7 +468,7 @@ func TestTransformNegativeCachesAFailure(t *testing.T) { mapper := newTestMapper(t) for i := 0; i < 3; i++ { - if _, err := mapper.Transform(context.Background(), ref(srv.URL), "select", requestInput()); err == nil { + if _, err := mapper.Transform(context.Background(), ref(srv.URL), definition.DirectionRequest, requestInput()); err == nil { t.Fatal("expected a missing mapping to fail") } } @@ -488,15 +483,15 @@ func TestTransformRefetchesAfterTheTTL(t *testing.T) { t.Parallel() var fetches atomic.Int32 - srv := newMappingServer(t, twoActionMapping, &fetches) + srv := newMappingServer(t, bothDirections, &fetches) defer srv.Close() mapper := newTestMapper(t, func(c *Config) { c.CacheTTL = 20 * time.Millisecond }) - if _, err := mapper.Transform(context.Background(), ref(srv.URL), "select", requestInput()); err != nil { + if _, err := mapper.Transform(context.Background(), ref(srv.URL), definition.DirectionRequest, requestInput()); err != nil { t.Fatalf("Transform() returned an unexpected error: %v", err) } time.Sleep(60 * time.Millisecond) - if _, err := mapper.Transform(context.Background(), ref(srv.URL), "select", requestInput()); err != nil { + if _, err := mapper.Transform(context.Background(), ref(srv.URL), definition.DirectionRequest, requestInput()); err != nil { t.Fatalf("Transform() returned an unexpected error: %v", err) } @@ -510,13 +505,13 @@ func TestTransformRefetchesAfterTheTTL(t *testing.T) { func TestTransformBoundsTheCache(t *testing.T) { t.Parallel() - srv := newMappingServer(t, twoActionMapping, nil) + srv := newMappingServer(t, bothDirections, nil) defer srv.Close() mapper := newTestMapper(t, func(c *Config) { c.MaxCacheEntries = 2 }) for i := 0; i < 5; i++ { if _, err := mapper.Transform(context.Background(), - fmt.Sprintf("%s/%d.yaml", srv.URL, i), "select", requestInput()); err != nil { + fmt.Sprintf("%s/%d.yaml", srv.URL, i), definition.DirectionRequest, requestInput()); err != nil { t.Fatalf("Transform() returned an unexpected error: %v", err) } } @@ -529,20 +524,24 @@ func TestTransformBoundsTheCache(t *testing.T) { // Every inbound request shares one mapper, so the cache is read and written // concurrently, and jsonata.Expression.Evaluate mutates what it is called on. -// Run with -race. +// Both halves are exercised: they hold separate locks, so a request and a +// response leg of the same mapping do run at the same time. Run with -race. func TestTransformIsSafeUnderConcurrentUse(t *testing.T) { t.Parallel() - srv := newMappingServer(t, twoActionMapping, nil) + srv := newMappingServer(t, bothDirections, nil) defer srv.Close() mapper := newTestMapper(t) - actions := []string{"select", "confirm"} errs := make(chan error, 20) for i := 0; i < 20; i++ { go func(i int) { + direction, input := definition.DirectionRequest, requestInput() + if i%2 == 1 { + direction, input = definition.DirectionResponse, responseInput() + } _, err := mapper.Transform(context.Background(), - fmt.Sprintf("%s/%d.yaml", srv.URL, i%3), actions[i%2], requestInput()) + fmt.Sprintf("%s/%d.yaml", srv.URL, i%3), direction, input) errs <- err }(i) } diff --git a/pkg/plugin/implementation/mausamgram/cmd/plugin_test.go b/pkg/plugin/implementation/mausamgram/cmd/plugin_test.go index f3d3cc4b..f932a559 100644 --- a/pkg/plugin/implementation/mausamgram/cmd/plugin_test.go +++ b/pkg/plugin/implementation/mausamgram/cmd/plugin_test.go @@ -20,7 +20,9 @@ func (stubRegistry) ProviderRecord(context.Context, string) (*model.ProviderReco type stubMapper struct{} -func (stubMapper) Transform(context.Context, string, string, any) ([]byte, error) { return nil, nil } +func (stubMapper) Transform(context.Context, string, definition.Direction, any) ([]byte, error) { + return nil, nil +} func TestParseConfig(t *testing.T) { t.Parallel() diff --git a/pkg/plugin/implementation/mausamgram/dispatch_test.go b/pkg/plugin/implementation/mausamgram/dispatch_test.go index c8b17e62..d13bc333 100644 --- a/pkg/plugin/implementation/mausamgram/dispatch_test.go +++ b/pkg/plugin/implementation/mausamgram/dispatch_test.go @@ -13,11 +13,15 @@ import ( "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/mausamgram" ) +// dispatchMappingRef stands in for the one reference an action carries. This +// test is about dispatch, so what is behind it never matters. +const dispatchMappingRef = "https://m.example.com/mausamgram/weather-observation.select.yaml" + // fixedMapper returns canned results, so this test is about dispatch and // nothing else. type fixedMapper struct{ answer string } -func (m fixedMapper) Transform(_ context.Context, mappingRef, _ string, _ any) ([]byte, error) { +func (m fixedMapper) Transform(_ context.Context, mappingRef string, _ definition.Direction, _ any) ([]byte, error) { if strings.Contains(mappingRef, "request") { return []byte(`{}`), nil } @@ -44,10 +48,9 @@ func TestTwoProviderStepsDispatchByBindingKey(t *testing.T) { t.Helper() plan := &model.ProviderRecord{ BindingKey: bindingKey, BaseURL: upstreamURL, - RequestMapping: "https://m.example.com/request.yaml", - ResponseMapping: "https://m.example.com/response.yaml", + Actions: map[string]model.ActionPlan{ - "select": {Method: http.MethodGet, Path: "/x", RetryMax: 1}, + "select": {Method: http.MethodGet, Path: "/x", Mappings: dispatchMappingRef, RetryMax: 1}, }, } step, closer, err := mausamgram.New(context.Background(), diff --git a/pkg/plugin/implementation/mausamgram/mappings_test.go b/pkg/plugin/implementation/mausamgram/mappings_test.go index 200dfed8..38d0bc6d 100644 --- a/pkg/plugin/implementation/mausamgram/mappings_test.go +++ b/pkg/plugin/implementation/mausamgram/mappings_test.go @@ -11,6 +11,7 @@ package mausamgram_test import ( "context" "encoding/json" + "errors" "fmt" "net/http" "net/http/httptest" @@ -20,6 +21,7 @@ import ( "testing" "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/mausamgram" ) @@ -27,6 +29,12 @@ import ( // mappingsDir is where the shipped mappings live, relative to this package. const mappingsDir = "../../../../config/mappings/mausamgram" +// shippedMapping is the file this binding-action publishes: one file, both +// directions. The registry carries its full URL; the action segment of the name +// must match the action that registry entry declares -- a mismatch would apply a +// correct mapping to the wrong call, silently. +const shippedMapping = "weather-observation.select.yaml" + // selectRequest is the verbatim /select captured from the OAN network. const selectRequest = `{ "context": { "version": "2.0.0", "action": "select", @@ -116,14 +124,13 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { defer closeMapper() registry := &stubRegistry{plan: &model.ProviderRecord{ - BindingKey: mausamgram.DefaultBindingKey, - ParticipantID: "mausamgram", - CapabilityCode: "openagrinet:WeatherObservation", - BaseURL: upstream.URL, - RequestMapping: mappings.URL + "/request.yaml", - ResponseMapping: mappings.URL + "/response.yaml", + BindingKey: mausamgram.DefaultBindingKey, + ParticipantID: "mausamgram", + CapabilityCode: "openagrinet:WeatherObservation", + BaseURL: upstream.URL, Actions: map[string]model.ActionPlan{ - "select": {Method: http.MethodGet, Path: "/get-daily", TimeoutMs: 30000, RetryMax: 3}, + "select": {Method: http.MethodGet, Path: "/get-daily", + Mappings: mappings.URL + "/" + shippedMapping, TimeoutMs: 30000, RetryMax: 3}, }, }} @@ -139,7 +146,7 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { } // --- the request reached the provider correctly ------------------------- - // request.yaml declares select with no transform, so these parameters are + // The request half is empty, so these parameters are // the point the step resolved, not something a mapping produced. for _, want := range []string{"lat=19.9975", "lon=73.7898"} { if !strings.Contains(gotQuery, want) { @@ -231,9 +238,11 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { } } -// A file serves the actions it declares and no others. An action it does not -// carry is refused rather than served by whichever mapping happened to be there. -func TestShippedMappingsAreRefusedForAnotherAction(t *testing.T) { +// The shipped file's request half is deliberately empty: this provider takes +// its parameters in the query string. That has to reach the step as +// ErrNoTransform rather than as an empty document, or the step would send a +// body-shaped nothing instead of building the query itself. +func TestShippedMappingsDeclareNoRequestTransform(t *testing.T) { mappings := serveMappings(t) defer mappings.Close() @@ -243,15 +252,16 @@ func TestShippedMappingsAreRefusedForAnotherAction(t *testing.T) { } defer closeMapper() - _, err = mapper.Transform(context.Background(), mappings.URL+"/request.yaml", "confirm", - map[string]any{"_local": map[string]any{"lat": 1.0, "lon": 2.0}}) - if err == nil { - t.Fatal("expected an unserved action to be refused") + ref := mappings.URL + "/" + shippedMapping + input := map[string]any{"_local": map[string]any{"lat": 1.0, "lon": 2.0}} + + if _, err := mapper.Transform(context.Background(), ref, definition.DirectionRequest, input); !errors.Is(err, definition.ErrNoTransform) { + t.Errorf("the request half should report ErrNoTransform, got %v", err) } - // The refusal names what the file does serve, so a missing mapping is a - // one-line fix rather than a hunt. - if !strings.Contains(err.Error(), "select") { - t.Errorf("error %q should say which actions the file serves", err) + // The response half of the same file is unaffected -- which is the point of + // holding both in one file rather than inferring one from the other. + if _, err := mapper.Transform(context.Background(), ref, definition.DirectionResponse, input); errors.Is(err, definition.ErrNoTransform) { + t.Error("the response half must carry a transform") } } diff --git a/pkg/plugin/implementation/mausamgram/mausamgram.go b/pkg/plugin/implementation/mausamgram/mausamgram.go index 677c05dd..40fe6d34 100644 --- a/pkg/plugin/implementation/mausamgram/mausamgram.go +++ b/pkg/plugin/implementation/mausamgram/mausamgram.go @@ -34,8 +34,12 @@ const ( // so a deployment can rename the participant without a rebuild, but it has a // default because a step that serves nothing is never what an operator meant. DefaultBindingKey = "mausamgram|openagrinet:WeatherObservation" - DefaultTimeout = 30 * time.Second - DefaultRetryMax = 3 + // DefaultTimeout and DefaultRetryMax are the registry contract's defaults + // for an action that leaves timeoutMs or retryMax out. Zero retries is + // deliberate: a provider that failed is retried only where the operator + // said so, because a retry on a non-idempotent action is a second booking. + DefaultTimeout = 15 * time.Second + DefaultRetryMax = 0 // DefaultMaxResponseBytes caps what is read from the provider. The response // is mapped in memory, so an unbounded one is an unbounded allocation. DefaultMaxResponseBytes = 4 << 20 // 4 MiB @@ -201,7 +205,7 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { return err } - upstreamRequest, err := s.buildRequest(ctx, plan.RequestMapping, action, beckn, local) + upstreamRequest, err := s.buildRequest(ctx, call.Mappings, beckn, local) if err != nil { return err } @@ -219,10 +223,10 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { // _local stays in scope: the provider does not repeat the point it was asked // about, so the output's own coordinates have no other source. // - // The response mapping is asked for by the action it PRODUCES, not the one - // that arrived: a select is answered by an on_select. Each mapping file is - // therefore keyed by the Beckn actions it actually deals in. - becknResponse, err := s.mapper.Transform(ctx, plan.ResponseMapping, callbackAction(action), map[string]any{ + // The same mapping reference as the request, other half: one file carries + // both directions for this action, because the response mapping usually + // depends on what the request mapping did. + becknResponse, err := s.mapper.Transform(ctx, call.Mappings, definition.DirectionResponse, map[string]any{ "beckn": beckn, "_local": local, "response": answer, @@ -249,16 +253,16 @@ func servedActions(plan *model.ProviderRecord) []string { // buildRequest produces what the provider is sent. // -// A mapping that declares the action but supplies no transform is saying the -// request needs no document built for it: this provider takes its parameters in -// the query, they are already resolved, and putting them through a fetch and a -// compile to arrive at the same two fields buys nothing. In that case the -// resolved values ARE the parameters. +// A mapping whose request half is empty is saying the request needs no document +// built for it: this provider takes its parameters in the query, they are +// already resolved, and putting them through a fetch and a compile to arrive at +// the same two fields buys nothing. In that case the resolved values ARE the +// parameters. // // Anything else -- a provider wanting a body in its own shape -- goes through // the mapping, which is what the mapping is for. -func (s *Step) buildRequest(ctx context.Context, mappingRef, action string, beckn any, local point) ([]byte, error) { - mapped, err := s.mapper.Transform(ctx, mappingRef, action, map[string]any{ +func (s *Step) buildRequest(ctx context.Context, mappingRef string, beckn any, local point) ([]byte, error) { + mapped, err := s.mapper.Transform(ctx, mappingRef, definition.DirectionRequest, map[string]any{ "beckn": beckn, "_local": local, }) @@ -269,7 +273,7 @@ func (s *Step) buildRequest(ctx context.Context, mappingRef, action string, beck return nil, err } - log.Debugf(ctx, "mausamgram: %s declares %s with no transform; sending the resolved point", mappingRef, action) + log.Debugf(ctx, "mausamgram: %s carries no request half; sending the resolved point", mappingRef) parameters, err := json.Marshal(local) if err != nil { return nil, fmt.Errorf("mausamgram: could not encode the resolved point: %w", err) @@ -327,16 +331,6 @@ func resolvePoint(body []byte) (point, error) { errors.New("mausamgram: request carries no location coordinates")) } -// callbackAction is the action that answers the given one. Beckn pairs every -// request with an on_-prefixed callback -- select with on_select, confirm with -// on_confirm -- and that pairing is the protocol's, not this provider's. -func callbackAction(action string) string { - if action == "" { - return "" - } - return "on_" + action -} - // extractAction reads the Beckn action a request is for. func extractAction(body []byte) string { var payload struct { @@ -371,10 +365,13 @@ func (s *Step) call(ctx context.Context, baseURL string, call model.ActionPlan, if call.TimeoutMs > 0 { timeout = time.Duration(call.TimeoutMs) * time.Millisecond } - attempts := DefaultRetryMax + // retryMax counts retries, not attempts, so the call itself is always made + // once. An absent retryMax and an explicit 0 are the same instruction. + retries := DefaultRetryMax if call.RetryMax > 0 { - attempts = call.RetryMax + retries = call.RetryMax } + attempts := retries + 1 var lastErr error for attempt := 1; attempt <= attempts; attempt++ { diff --git a/pkg/plugin/implementation/mausamgram/mausamgram_test.go b/pkg/plugin/implementation/mausamgram/mausamgram_test.go index e6e28c3f..fd78813d 100644 --- a/pkg/plugin/implementation/mausamgram/mausamgram_test.go +++ b/pkg/plugin/implementation/mausamgram/mausamgram_test.go @@ -37,6 +37,10 @@ func (s *stubRegistry) ProviderRecord(context.Context, string) (*model.ProviderR } // stubMapper records what it was asked and returns canned results, so a test can +// testMappingRef is the one reference an action carries: the URL of a single +// published file holding both halves. +const testMappingRef = "https://mappings.example.com/mausamgram/weather-observation.select.yaml" + // assert what reached the mapping without writing one. type stubMapper struct { requestResult []byte @@ -46,17 +50,17 @@ type stubMapper struct { requestInput any responseInput any - actions []string + directions []definition.Direction refs []string } -func (s *stubMapper) Transform(_ context.Context, mappingRef, action string, input any) ([]byte, error) { - s.actions = append(s.actions, action) +func (s *stubMapper) Transform(_ context.Context, mappingRef string, direction definition.Direction, input any) ([]byte, error) { + s.directions = append(s.directions, direction) s.refs = append(s.refs, mappingRef) if s.err != nil { return nil, s.err } - if strings.Contains(mappingRef, "request") { + if direction == definition.DirectionRequest { s.requestInput = input if s.requestErr != nil { return nil, s.requestErr @@ -69,14 +73,13 @@ func (s *stubMapper) Transform(_ context.Context, mappingRef, action string, inp func testPlan(baseURL, method string) *model.ProviderRecord { return &model.ProviderRecord{ - BindingKey: DefaultBindingKey, - ParticipantID: "mausamgram", - CapabilityCode: "openagrinet:WeatherObservation", - BaseURL: baseURL, - RequestMapping: "https://mappings.example.com/request.yaml", - ResponseMapping: "https://mappings.example.com/response.yaml", + BindingKey: DefaultBindingKey, + ParticipantID: "mausamgram", + CapabilityCode: "openagrinet:WeatherObservation", + BaseURL: baseURL, Actions: map[string]model.ActionPlan{ - "select": {Method: method, Path: "/get-daily", TimeoutMs: 2000, RetryMax: 1}, + "select": {Method: method, Path: "/get-daily", Mappings: testMappingRef, + TimeoutMs: 2000, RetryMax: 1}, }, } } @@ -216,8 +219,13 @@ func TestRunServesItsCapabilityEndToEnd(t *testing.T) { // Each leg asks for the action it deals in: the request translates a select, // the response produces an on_select. Asking for the same name on both would // make one file unable to hold both directions. - if want := []string{"select", "on_select"}; !slices.Equal(mapper.actions, want) { - t.Errorf("mapper was asked for %v, want %v", mapper.actions, want) + if want := []definition.Direction{definition.DirectionRequest, definition.DirectionResponse}; !slices.Equal(mapper.directions, want) { + t.Errorf("mapper was asked for %v, want %v", mapper.directions, want) + } + // Both halves come from the one file the action names. Two references here + // would mean the step had gone back to treating the legs as separate. + if want := []string{testMappingRef, testMappingRef}; !slices.Equal(mapper.refs, want) { + t.Errorf("mapper was handed %v, want both halves from %q", mapper.refs, testMappingRef) } } @@ -442,15 +450,17 @@ func TestRunReportsAProviderThatWillNotAnswer(t *testing.T) { defer upstream.Close() plan := testPlan(upstream.URL, http.MethodGet) - plan.Actions["select"] = model.ActionPlan{Method: http.MethodGet, Path: "/get-daily", RetryMax: 3} + plan.Actions["select"] = model.ActionPlan{Method: http.MethodGet, Path: "/get-daily", + Mappings: testMappingRef, RetryMax: 3} mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} _, err := runStep(t, newStep(t, &stubRegistry{plan: plan}, mapper), selectBody) if err == nil { t.Fatal("expected a failing provider to be reported") } - if got := attempts.Load(); got != 3 { - t.Errorf("made %d attempts, want the plan's 3", got) + // retryMax is retries, so the plan's 3 is the first call plus three more. + if got := attempts.Load(); got != 4 { + t.Errorf("made %d attempts, want 4 -- the call plus the plan's 3 retries", got) } var coded *model.CodedErr @@ -459,6 +469,32 @@ func TestRunReportsAProviderThatWillNotAnswer(t *testing.T) { } } +// An action that leaves retryMax out is called once. The contract's default is +// zero retries, and it has to stay zero: a retry on a non-idempotent action is +// a second booking, so retrying is only ever what the operator asked for. +func TestRunDoesNotRetryUnlessTheActionSaysSo(t *testing.T) { + t.Parallel() + + var attempts atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts.Add(1) + w.WriteHeader(http.StatusInternalServerError) + })) + defer upstream.Close() + + plan := testPlan(upstream.URL, http.MethodGet) + plan.Actions["select"] = model.ActionPlan{Method: http.MethodGet, Path: "/get-daily", + Mappings: testMappingRef} + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + + if _, err := runStep(t, newStep(t, &stubRegistry{plan: plan}, mapper), selectBody); err == nil { + t.Fatal("expected a failing provider to be reported") + } + if got := attempts.Load(); got != 1 { + t.Errorf("made %d attempts, want 1 -- an absent retryMax must mean no retries", got) + } +} + // A capability that publishes no endpoint for an action does not serve it. The // refusal has to come before the call, not after one to the wrong place. func TestRunRefusesAnActionWithNoEndpoint(t *testing.T) { diff --git a/pkg/plugin/implementation/oanregistry/README.md b/pkg/plugin/implementation/oanregistry/README.md index 099a7b84..a35bd96a 100644 --- a/pkg/plugin/implementation/oanregistry/README.md +++ b/pkg/plugin/implementation/oanregistry/README.md @@ -127,24 +127,41 @@ message.resourceAttributes["@type"] ─┴─▶ bindingKey "|< ``` Two reads, joined into one `model.ProviderRecord`: `baseUrl` from the -participant, the mappings and a **call plan per action** from the binding. +participant, a **call plan per action** from the binding. ```json { "bindingKey": "mausamgram|openagrinet:WeatherObservation", - "requestMapping": "...", "responseMapping": "...", - "actions": { - "select": { "method": "GET", "path": "/get-daily", "timeoutMs": 30000, "retryMax": 3 }, - "confirm": { "method": "POST", "path": "/book" } - } + "participantId": "mausamgram", + "capabilityCode": "openagrinet:WeatherObservation", + "status": "active", + "actions": [ + { "action": "select", "method": "GET", "path": "/get-daily", + "mappings": "https://.../mausamgram/weather-observation.select.yaml", + "timeoutMs": 30000, "retryMax": 3, "status": "active" }, + { "action": "confirm", "method": "POST", "path": "/book", + "mappings": "https://.../mausamgram/weather-observation.confirm.yaml", + "status": "inactive" } + ] } ``` -A capability serves several actions and they rarely share an endpoint — a -`confirm` that commits does not post where a `select` that reads gets — so the -endpoint, method and budget are per action. An action absent from `actions` is -one the capability does not serve, and a binding serving none at all is refused -outright rather than failing one action at a time. +A capability serves several actions and they rarely share an endpoint, a method +or a mapping — a `confirm` that commits does not post where a `select` that reads +gets — so all of it is per action. An action absent from `actions` is one the +capability does not serve, and a binding serving none at all is refused outright +rather than failing one action at a time. + +**An array rather than a keyed object, for two reasons.** A per-action `status` +is how one action is retired while the capability and every other action stay +live, and an entry that is not `active` is skipped exactly as if it were absent. +And the registry treats every nested object as an entity and injects an `osid` +into it, which a keyed map cannot carry. + +`mappings` is one reference per action carrying **both directions**, because the +response mapping usually depends on what the request mapping did. It is the +published file's URL, passed through verbatim — this plugin does not interpret +or resolve it. The owning participant is the one the **binding names**, not one parsed out of the binding key — the registry owns that relationship, not the key format. diff --git a/pkg/plugin/implementation/oanregistry/oanregistry.go b/pkg/plugin/implementation/oanregistry/oanregistry.go index cebf4219..53347209 100644 --- a/pkg/plugin/implementation/oanregistry/oanregistry.go +++ b/pkg/plugin/implementation/oanregistry/oanregistry.go @@ -155,39 +155,36 @@ type Client struct { cacheTTL time.Duration } -// participant is the subset of a registry record this plugin reads. The registry -// carries a good deal more -- Sunbird audit fields (osCreatedAt, osOwner, ...) and -// the participant and node osids among them -- and none of it is modelled here. -// encoding/json drops what it cannot place, so every field left out is one less -// thing to break when the registry schema moves. +// participant is the subset of a registry record this plugin reads. The +// registry carries more -- Sunbird audit fields, the display name, an auth +// block -- and none of it is modelled here. encoding/json drops what it cannot +// place, so every field left out is one less thing to break when the schema +// moves. // -// Only the participant's own status sits at this level. Keys hang off the node, -// each carrying its own identity and status, so resolving one is a walk rather -// than a field read. +// Flat, with no wrapper object, because type is the discriminator rather than +// the shape: a node speaks Beckn and publishes keys, an upstream is an ordinary +// API and publishes auth instead. baseUrl serves both -- for a node it is where +// Beckn messages go, for an upstream it is what a binding's path is appended to. +// +// The auth block is left out DELIBERATELY, not pending. An upstream's credential +// is the provider plugin's own configuration -- which scheme, and which +// environment variables hold the values -- so nothing is gained by also reading +// the registry's copy, and reading both would create two places that can +// disagree about how to authenticate a call. The registry publishes it as +// documentation of what a provider expects; the adapter presents what its +// operator configured. type participant struct { - ParticipantID string `json:"participantId"` - Status string `json:"status"` - Node node `json:"node"` - Upstream upstream `json:"upstream"` -} - -// upstream is the backend a provider participant fronts. It is present only on -// records that front one: a participant is either a network peer publishing keys -// under node, or a provider publishing an upstream, and the two do not overlap. -type upstream struct { - BaseURL string `json:"baseUrl"` -} - -// node is the network-facing half of a participant record. -type node struct { - SubscriberURL string `json:"subscriberUrl"` + ParticipantID string `json:"participantId"` Type string `json:"type"` + Role string `json:"role"` + Status string `json:"status"` + BaseURL string `json:"baseUrl"` Keys []key `json:"keys"` } -// key is one published key. A node publishes several -- separate signing and -// encryption keys, and more than one signing key while a rotation is in flight -- -// so a key is identified by its own OSID rather than by its position. +// key is one published key. A participant publishes several -- separate signing +// and encryption keys, and more than one signing key while a rotation is in +// flight -- so a key is identified by its own OSID rather than by its position. type key struct { OSID string `json:"osid"` KeyID string `json:"keyId"` @@ -441,7 +438,7 @@ func (c *Client) search(ctx context.Context, tracer trace.Tracer, participantID, // the case the second filter was originally meant to guard: a stale or // soft-deleted record sharing the participantId. for _, record := range records { - for _, k := range record.Node.Keys { + for _, k := range record.Keys { if k.OSID != keyID { continue } @@ -528,23 +525,27 @@ func toSubscription(p participant, k key, status string) model.Subscription { return model.Subscription{ Subscriber: model.Subscriber{ SubscriberID: p.ParticipantID, - URL: p.Node.SubscriberURL, - Type: p.Node.Type, + URL: p.BaseURL, + // role is the Beckn role -- BAP, BPP or NETWORK. type is the + // registry's own discriminator (node or upstream) and means + // something else entirely, so it is not what a subscriber's Type is. + Type: p.Role, }, KeyID: k.OSID, SigningPublicKey: k.publicKey(), - EncrPublicKey: encryptionKey(p.Node), + EncrPublicKey: encryptionKey(p), ValidFrom: validFrom, ValidUntil: validUntil, Status: status, } } -// encryptionKey returns the node's active encryption key, or "" when it publishes -// none. It is resolved by use rather than by id: the request header names the -// signing key only, so there is nothing to match an encryption key against. -func encryptionKey(n node) string { - for _, k := range n.Keys { +// encryptionKey returns the participant's active encryption key, or "" when it +// publishes none. It is resolved by use rather than by id: the request header +// names the signing key only, so there is nothing to match an encryption key +// against. +func encryptionKey(p participant) string { + for _, k := range p.Keys { if strings.EqualFold(k.Use, useEncr) && strings.EqualFold(k.Status, statusActive) { return k.publicKey() } diff --git a/pkg/plugin/implementation/oanregistry/oanregistry_test.go b/pkg/plugin/implementation/oanregistry/oanregistry_test.go index 684d5508..5acb47a6 100644 --- a/pkg/plugin/implementation/oanregistry/oanregistry_test.go +++ b/pkg/plugin/implementation/oanregistry/oanregistry_test.go @@ -91,12 +91,11 @@ func signingKey() key { func activeRecord() participant { return participant{ ParticipantID: testParticipantID, + Type: "node", + Role: "BPP", Status: "active", - Node: node{ - SubscriberURL: "https://providera.example.com/onix", - Type: "BPP", - Keys: []key{signingKey()}, - }, + BaseURL: "https://providera.example.com/onix", + Keys: []key{signingKey()}, } } @@ -445,14 +444,14 @@ func TestToSubscriptionMapsOptionalFields(t *testing.T) { t.Parallel() record := activeRecord() - record.Node.Keys = append(record.Node.Keys, key{ + record.Keys = append(record.Keys, key{ OSID: "1-abcdef00-0000-0000-0000-000000000000", Use: useEncr, Value: keyEncodingPrefix + "encryption-key", Status: "active", }) - got := toSubscription(record, record.Node.Keys[0], statusSubscribed) + got := toSubscription(record, record.Keys[0], statusSubscribed) if got.EncrPublicKey != "encryption-key" { t.Errorf("expected encryption key to be mapped, got %q", got.EncrPublicKey) @@ -474,7 +473,7 @@ func TestToSubscriptionMapsOptionalFields(t *testing.T) { k := key{OSID: testOSID, Use: useSign, Value: testPublicKey, Status: "active"} record := participant{ ParticipantID: testParticipantID, - Node: node{Keys: []key{k}}, + Keys: []key{k}, } got := toSubscription(record, k, statusSubscribed) @@ -495,14 +494,14 @@ func TestToSubscriptionMapsOptionalFields(t *testing.T) { t.Parallel() record := activeRecord() - record.Node.Keys = append(record.Node.Keys, key{ + record.Keys = append(record.Keys, key{ OSID: "1-abcdef00-0000-0000-0000-000000000000", Use: useEncr, Value: keyEncodingPrefix + "retired-encryption-key", Status: "inactive", }) - got := toSubscription(record, record.Node.Keys[0], statusSubscribed) + got := toSubscription(record, record.Keys[0], statusSubscribed) if got.EncrPublicKey != "" { t.Errorf("a retired encryption key must not be published, got %q", got.EncrPublicKey) @@ -659,7 +658,7 @@ func TestLookupWarnsOnAlgorithmMismatch(t *testing.T) { t.Parallel() record := activeRecord() - record.Node.Keys[0].Algorithm = "rsa-2048" + record.Keys[0].Algorithm = "rsa-2048" srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, recordJSON(t, record)) @@ -774,7 +773,7 @@ func TestLookupRejectsAKeyIdMismatch(t *testing.T) { t.Parallel() record := activeRecord() - record.Node.Keys[0].OSID = "1-99999999-0000-0000-0000-000000000000" + record.Keys[0].OSID = "1-99999999-0000-0000-0000-000000000000" srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, recordJSON(t, record)) @@ -798,7 +797,7 @@ func TestSearchDistinguishesMismatchFromNotFound(t *testing.T) { t.Parallel() otherKey := activeRecord() - otherKey.Node.Keys[0].OSID = "1-99999999-0000-0000-0000-000000000000" + otherKey.Keys[0].OSID = "1-99999999-0000-0000-0000-000000000000" testCases := []struct { name string @@ -840,8 +839,8 @@ func TestLookupSelectsTheRecordCarryingTheKey(t *testing.T) { t.Parallel() stale := activeRecord() - stale.Node.Keys[0].OSID = "1-00000000-0000-0000-0000-000000000000" - stale.Node.Keys[0].Value = "stale-key" + stale.Keys[0].OSID = "1-00000000-0000-0000-0000-000000000000" + stale.Keys[0].Value = "stale-key" srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, recordJSON(t, stale, activeRecord())) @@ -865,7 +864,7 @@ func TestLookupOnDuplicateRecords(t *testing.T) { t.Parallel() first, second := activeRecord(), activeRecord() - second.Node.Keys[0].Value = "a-different-key" + second.Keys[0].Value = "a-different-key" srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, recordJSON(t, first, second)) @@ -1359,62 +1358,59 @@ func assertOutcomeAttribute(t *testing.T, m metricdata.Metrics, outcome string) } // TestLookupAgainstCapturedRegistryResponse runs the plugin against a verbatim -// response captured from the real OAN registry on 29 Aug 2026, reformatted for +// response captured from the real OAN registry on 31 Aug 2026, reformatted for // readability with field order and values untouched. // -// It pins the deployed shape: the data envelope, keys nested under node as an -// array, the camelCase field names, the "base64:" encoding label, and the -// "active" status vocabulary at both levels. The capture it replaces described a -// flat record with snake_case fields, and this is the test that said so. +// It pins the deployed shape: the data envelope, one flat level with the keys +// array beside participantId rather than under a wrapper, the camelCase field +// names, the "base64:" encoding label, the osid the registry injects into every +// nested object, and the "active" status vocabulary at both levels. The capture +// it replaces described a record wrapped in a "node" object, and this is the +// test that said so. func TestLookupAgainstCapturedRegistryResponse(t *testing.T) { t.Parallel() const ( - capturedParticipantID = "oan-provider" - capturedKeyOSID = "1-e73cd04b-d992-4ecb-81e3-003f28ea36ea" - capturedNodeOSID = "1-bc829800-6acb-48ec-86cd-0f52de25abb9" - capturedParticipantOSID = "1-e1072144-938e-4ab1-87c0-efd5cc45f6e6" - capturedKey = "xq4+2oQ6MgSZdHHBMtNd1TmnPTmzY5UoZlqzf0yn6ZA=" - capturedURL = "https://provider-network-vistaar.da.gov.in/beckn" + capturedParticipantID = "provider.oan.local" + capturedKeyOSID = "1-d1a4a2b7-7bf5-42f5-bfc2-2c77119c4d64" + capturedParticipantOSID = "1-19087a97-f886-4fe4-bf14-3875437dc6f8" + capturedKey = "w1wDdr/xnO2yQYxdR/88enTkg0B//vVeIkXOfreClUQ=" + capturedURL = "https://provider.oan.local/beckn" ) const captured = `{ "totalCount": 1, "data": [ { - "participantId": "oan-provider", - "osUpdatedAt": "2026-08-29T06:47:56.019Z", - "osCreatedAt": "2026-08-29T06:47:56.019Z", + "osUpdatedAt": "2026-08-31T07:36:33.407Z", + "role": "BPP", "osUpdatedBy": "89bf9fcb-c6f7-4f08-80f9-18f47ce7667d", - "name": "OpenAgriNet provider adapter", + "osid": "1-19087a97-f886-4fe4-bf14-3875437dc6f8", + "type": "node", + "osOwner": [ + "89bf9fcb-c6f7-4f08-80f9-18f47ce7667d" + ], + "keys": [ + { + "osUpdatedAt": "2026-08-31T07:36:33.407Z", + "osUpdatedBy": "89bf9fcb-c6f7-4f08-80f9-18f47ce7667d", + "use": "sign", + "keyId": "k1", + "osid": "1-d1a4a2b7-7bf5-42f5-bfc2-2c77119c4d64", + "validFrom": "2026-01-01T00:00:00Z", + "osCreatedAt": "2026-08-31T07:36:33.407Z", + "osCreatedBy": "89bf9fcb-c6f7-4f08-80f9-18f47ce7667d", + "validUntil": "2030-01-01T00:00:00Z", + "alg": "ed25519", + "key": "base64:w1wDdr/xnO2yQYxdR/88enTkg0B//vVeIkXOfreClUQ=", + "status": "active" + } + ], + "participantId": "provider.oan.local", + "baseUrl": "https://provider.oan.local/beckn", + "osCreatedAt": "2026-08-31T07:36:33.407Z", + "name": "OAN provider layer adapter", "osCreatedBy": "89bf9fcb-c6f7-4f08-80f9-18f47ce7667d", - "osid": "1-e1072144-938e-4ab1-87c0-efd5cc45f6e6", - "osOwner": ["89bf9fcb-c6f7-4f08-80f9-18f47ce7667d"], - "node": { - "osid": "1-bc829800-6acb-48ec-86cd-0f52de25abb9", - "osUpdatedAt": "2026-08-29T06:47:56.019Z", - "osCreatedAt": "2026-08-29T06:47:56.019Z", - "osUpdatedBy": "89bf9fcb-c6f7-4f08-80f9-18f47ce7667d", - "subscriberUrl": "https://provider-network-vistaar.da.gov.in/beckn", - "osCreatedBy": "89bf9fcb-c6f7-4f08-80f9-18f47ce7667d", - "type": "BPP", - "keys": [ - { - "osUpdatedAt": "2026-08-29T06:47:56.019Z", - "osUpdatedBy": "89bf9fcb-c6f7-4f08-80f9-18f47ce7667d", - "use": "sign", - "keyId": "k1", - "osid": "1-e73cd04b-d992-4ecb-81e3-003f28ea36ea", - "validFrom": "2026-08-01T00:00:00Z", - "osCreatedAt": "2026-08-29T06:47:56.019Z", - "osCreatedBy": "89bf9fcb-c6f7-4f08-80f9-18f47ce7667d", - "validUntil": "2026-11-01T00:00:00Z", - "alg": "ed25519", - "key": "base64:xq4+2oQ6MgSZdHHBMtNd1TmnPTmzY5UoZlqzf0yn6ZA=", - "status": "active" - } - ] - }, "status": "active" } ] @@ -1455,7 +1451,7 @@ func TestLookupAgainstCapturedRegistryResponse(t *testing.T) { t.Errorf("key id = %q, want %q", got.KeyID, capturedKeyOSID) } if got.URL != capturedURL { - t.Errorf("endpoint url = %q, want the captured subscriberUrl %q", got.URL, capturedURL) + t.Errorf("endpoint url = %q, want the captured baseUrl %q", got.URL, capturedURL) } if got.Type != "BPP" { t.Errorf("type = %q, want %q", got.Type, "BPP") @@ -1467,13 +1463,12 @@ func TestLookupAgainstCapturedRegistryResponse(t *testing.T) { t.Error("expected the validity window to be parsed from the key's validFrom/validUntil") } - // The record carries three osids -- participant, node and key -- and only the - // key's identifies a signing key. Matching either of the other two would - // resolve the wrong thing, and would keep resolving it as soon as a second key - // were published. + // The record carries two osids -- the participant's and the key's -- and only + // the key's identifies a signing key. Matching the participant's would + // resolve the wrong thing, and would keep resolving it as soon as a second + // key were published. for _, tc := range []struct{ name, keyID string }{ {"participant osid", capturedParticipantOSID}, - {"node osid", capturedNodeOSID}, {"an unrelated osid", "1-00000000-0000-0000-0000-000000000000"}, } { mismatched, err := resolve(tc.keyID) diff --git a/pkg/plugin/implementation/oanregistry/providerrecord.go b/pkg/plugin/implementation/oanregistry/providerrecord.go index dd237f65..08895ff2 100644 --- a/pkg/plugin/implementation/oanregistry/providerrecord.go +++ b/pkg/plugin/implementation/oanregistry/providerrecord.go @@ -51,13 +51,11 @@ const ( // the enricher name -- and none of it is modelled: the enricher is resolved by // the provider plugin from its own code, not from the registry. type providerBinding struct { - BindingKey string `json:"bindingKey"` - ParticipantID string `json:"participantId"` - CapabilityCode string `json:"capabilityCode"` - Actions []actionPlan `json:"actions"` - RequestMapping string `json:"requestMapping"` - ResponseMapping string `json:"responseMapping"` - Status string `json:"status"` + BindingKey string `json:"bindingKey"` + ParticipantID string `json:"participantId"` + CapabilityCode string `json:"capabilityCode"` + Actions []actionPlan `json:"actions"` + Status string `json:"status"` } // actionPlan is one action's upstream call, as the registry publishes it. @@ -71,8 +69,10 @@ type actionPlan struct { Action string `json:"action"` Method string `json:"method"` Path string `json:"path"` + Mappings string `json:"mappings"` TimeoutMs int `json:"timeoutMs"` RetryMax int `json:"retryMax"` + Status string `json:"status"` } var ( @@ -201,24 +201,29 @@ func (c *Client) activeBinding(ctx context.Context, tracer trace.Tracer, binding log.Errorf(ctx, nil, "OAN registry capability binding bindingKey=%s names no participant", bindingKey) return providerBinding{}, outcomeBindingUnowned, nil } - if len(namedActions(binding)) == 0 { + if len(servableActions(binding)) == 0 { // Active, owned, and callable for nothing. Refusing here says so, rather // than letting every action fail one at a time further down. - log.Errorf(ctx, nil, "OAN registry capability binding bindingKey=%s serves no actions", bindingKey) + log.Errorf(ctx, nil, "OAN registry capability binding bindingKey=%s serves no active actions", bindingKey) return providerBinding{}, outcomeBindingNoActions, nil } return binding, outcomeFound, nil } -// namedActions returns the actions a binding can actually be reached by. -func namedActions(binding providerBinding) []actionPlan { - named := make([]actionPlan, 0, len(binding.Actions)) +// servableActions returns the actions a binding can actually serve. +// +// An entry has to be named to be reachable at all, and active to be served: a +// per-action status is how one action is retired while the capability and every +// other action stay live, so an inactive entry is skipped rather than failing +// the whole record. +func servableActions(binding providerBinding) []actionPlan { + servable := make([]actionPlan, 0, len(binding.Actions)) for _, plan := range binding.Actions { - if plan.Action != "" { - named = append(named, plan) + if plan.Action != "" && isActive(plan.Status) { + servable = append(servable, plan) } } - return named + return servable } // activeUpstream reads the participant that owns a binding and reports whether @@ -245,7 +250,7 @@ func (c *Client) activeUpstream(ctx context.Context, tracer trace.Tracer, partic log.Infof(ctx, "OAN registry participantId=%s is not usable: status=%q", participantID, owner.Status) return participant{}, outcomeParticipantInactive, nil } - if owner.Upstream.BaseURL == "" { + if owner.BaseURL == "" { // Active but unroutable. Denying here gives a clear reason rather than a // request sent to an empty host further down. log.Errorf(ctx, nil, "OAN registry participantId=%s publishes no upstream base url", participantID) @@ -272,26 +277,22 @@ func toProviderRecord(binding providerBinding, owner participant) *model.Provide // the whole record over one malformed row would take down the actions that // are fine. actions := make(map[string]model.ActionPlan, len(binding.Actions)) - for _, plan := range binding.Actions { - if plan.Action == "" { - continue - } + for _, plan := range servableActions(binding) { actions[plan.Action] = model.ActionPlan{ Method: plan.Method, Path: plan.Path, + Mappings: plan.Mappings, TimeoutMs: plan.TimeoutMs, RetryMax: plan.RetryMax, } } return &model.ProviderRecord{ - BindingKey: binding.BindingKey, - ParticipantID: binding.ParticipantID, - CapabilityCode: binding.CapabilityCode, - BaseURL: owner.Upstream.BaseURL, - Actions: actions, - RequestMapping: binding.RequestMapping, - ResponseMapping: binding.ResponseMapping, + BindingKey: binding.BindingKey, + ParticipantID: binding.ParticipantID, + CapabilityCode: binding.CapabilityCode, + BaseURL: owner.BaseURL, + Actions: actions, } } diff --git a/pkg/plugin/implementation/oanregistry/providerrecord_test.go b/pkg/plugin/implementation/oanregistry/providerrecord_test.go index 695513ae..97cec03b 100644 --- a/pkg/plugin/implementation/oanregistry/providerrecord_test.go +++ b/pkg/plugin/implementation/oanregistry/providerrecord_test.go @@ -18,12 +18,11 @@ import ( ) const ( - testBindingKey = "mausamgram|openagrinet:WeatherObservation" - testCapabilityCode = "openagrinet:WeatherObservation" - testProviderID = "mausamgram" - testBaseURL = "https://mausamgram.imd.gov.in/nwpapi" - testRequestMapping = "https://mappings.example.com/mausamgram/select.request.yaml" - testResponseMapping = "https://mappings.example.com/mausamgram/select.response.yaml" + testBindingKey = "mausamgram|openagrinet:WeatherObservation" + testCapabilityCode = "openagrinet:WeatherObservation" + testProviderID = "mausamgram" + testBaseURL = "https://mausamgram.imd.gov.in/nwpapi" + testMappings = "https://mappings.example.com/mausamgram/weather-observation.select.yaml" ) // envelopeJSON renders a registry search response in the data-envelope form. @@ -60,8 +59,9 @@ func arrayJSON[T any](t *testing.T, records ...T) string { func upstreamRecord() participant { return participant{ ParticipantID: testProviderID, + Type: "upstream", Status: "active", - Upstream: upstream{BaseURL: testBaseURL}, + BaseURL: testBaseURL, } } @@ -72,11 +72,10 @@ func bindingRecord() providerBinding { ParticipantID: testProviderID, CapabilityCode: testCapabilityCode, Actions: []actionPlan{ - {Action: "select", Method: "GET", Path: "/get-daily", TimeoutMs: 30000, RetryMax: 3}, + {Action: "select", Method: "GET", Path: "/get-daily", Mappings: testMappings, + TimeoutMs: 30000, RetryMax: 3, Status: "active"}, }, - RequestMapping: testRequestMapping, - ResponseMapping: testResponseMapping, - Status: "active", + Status: "active", } } @@ -129,8 +128,6 @@ func TestProviderRecordResolvesACallPlan(t *testing.T) { {"participant id", got.ParticipantID, testProviderID}, {"capability code", got.CapabilityCode, testCapabilityCode}, {"base url", got.BaseURL, testBaseURL}, - {"request mapping", got.RequestMapping, testRequestMapping}, - {"response mapping", got.ResponseMapping, testResponseMapping}, } { if field.got != field.want { t.Errorf("%s = %q, want %q", field.name, field.got, field.want) @@ -141,6 +138,9 @@ func TestProviderRecordResolvesACallPlan(t *testing.T) { if !served { t.Fatalf("no call plan for select, got actions %v", got.Actions) } + if call.Mappings != testMappings { + t.Errorf("mappings = %q, want %q", call.Mappings, testMappings) + } if call.Method != "GET" || call.Path != "/get-daily" { t.Errorf("select call = %s %s, want GET /get-daily", call.Method, call.Path) } @@ -156,7 +156,8 @@ func TestProviderRecordResolvesAnEndpointPerAction(t *testing.T) { binding := bindingRecord() binding.Actions = append(binding.Actions, - actionPlan{Action: "confirm", Method: "POST", Path: "/book", TimeoutMs: 60000, RetryMax: 1}) + actionPlan{Action: "confirm", Method: "POST", Path: "/book", Mappings: testMappings, + TimeoutMs: 60000, RetryMax: 1, Status: "active"}) srv := newRegistryServer(t, envelopeJSON(t, binding), envelopeJSON(t, upstreamRecord())) defer srv.Close() @@ -189,7 +190,8 @@ func TestProviderRecordLeavesAnAbsentBudgetAtZero(t *testing.T) { t.Parallel() binding := bindingRecord() - binding.Actions = []actionPlan{{Action: "select", Method: "GET", Path: "/get-daily"}} + binding.Actions = []actionPlan{{Action: "select", Method: "GET", Path: "/get-daily", + Mappings: testMappings, Status: "active"}} srv := newRegistryServer(t, envelopeJSON(t, binding), envelopeJSON(t, upstreamRecord())) defer srv.Close() @@ -255,8 +257,12 @@ func TestProviderRecordRefusals(t *testing.T) { noActionsBinding := bindingRecord() noActionsBinding.Actions = nil + inactiveActionBinding := bindingRecord() + inactiveActionBinding.Actions = []actionPlan{{Action: "select", Method: "GET", + Path: "/get-daily", Mappings: testMappings, Status: "inactive"}} + unnamedActionBinding := bindingRecord() - unnamedActionBinding.Actions = []actionPlan{{Method: "GET", Path: "/get-daily"}} + unnamedActionBinding.Actions = []actionPlan{{Method: "GET", Path: "/get-daily", Status: "active"}} inactiveUpstream := upstreamRecord() inactiveUpstream.Status = "inactive" @@ -265,7 +271,7 @@ func TestProviderRecordRefusals(t *testing.T) { emptyStatusUpstream.Status = "" noBaseURL := upstreamRecord() - noBaseURL.Upstream.BaseURL = "" + noBaseURL.BaseURL = "" testCases := []struct { name string @@ -279,6 +285,7 @@ func TestProviderRecordRefusals(t *testing.T) { {"a binding naming no participant", envelopeJSON(t, noParticipantBinding), envelopeJSON(t, upstreamRecord())}, {"a binding serving no action", envelopeJSON(t, noActionsBinding), envelopeJSON(t, upstreamRecord())}, {"a binding whose only action is unnamed", envelopeJSON(t, unnamedActionBinding), envelopeJSON(t, upstreamRecord())}, + {"a binding whose only action is retired", envelopeJSON(t, inactiveActionBinding), envelopeJSON(t, upstreamRecord())}, {"no participant owning the binding", envelopeJSON(t, bindingRecord()), envelopeJSON[participant](t)}, {"an inactive participant", envelopeJSON(t, bindingRecord()), envelopeJSON(t, inactiveUpstream)}, {"a participant with an empty status", envelopeJSON(t, bindingRecord()), envelopeJSON(t, emptyStatusUpstream)}, From 5e57bdfc8fb3b39b97e0d919a6f580e68747fe91 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 31 Aug 2026 23:57:33 +0530 Subject: [PATCH 030/130] fix: hand mappings only what a party sent [OpenAgriNet/engineering-tracker#66] Two things a mapping did not need. _local is gone. A mapping is handed the inbound payload, and on the way back the provider's answer, and nothing else. The values a provider plugin resolves before a call were also being passed in, which was a detour: the plugin holds them and used them to make the call, so a mapping reading them back was a second name for the same data. Where an answer needs one, it takes it from what the provider echoed -- the shipped mapping now reads response.location for the coordinates it was reading out of _local. ErrNoTransform is gone. A half that is absent or empty produces nothing, with no error, and what nothing means belongs to the caller rather than to a sentinel the mapper invents. On the request leg it means there is no document to send: for a method with no body the plugin sends the values it resolved, because it knows its provider and does not need the mapping's permission to call it; for a method with a body it sends no body, which is what an empty mapping says. A half that will not compile still reports an error, and that distinction is now carried by a test of its own: reading a broken half as "nothing" would send an unmapped upstream answer out as a Beckn response. One failure path is new. A response half that produces nothing leaves no Beckn answer, so the step fails rather than returning the provider's own shape under a valid signature. Its message says what was observed rather than guessing whether the transform was absent or simply matched nothing. Verified end to end: a signed /select returns three mapped forecast resources with the coordinates intact, which is what proves the mapping no longer needs _local to produce them. --- .../weather-observation.select.yaml | 26 ++-- pkg/plugin/definition/mapper.go | 28 ++--- .../implementation/jsonmapper/README.md | 33 +++-- .../implementation/jsonmapper/jsonmapper.go | 42 ++++--- .../jsonmapper/jsonmapper_test.go | 99 +++++++++------ .../mausamgram/mappings_test.go | 27 ++--- .../implementation/mausamgram/mausamgram.go | 65 ++++++---- .../mausamgram/mausamgram_test.go | 113 +++++++++++++++--- 8 files changed, 283 insertions(+), 150 deletions(-) diff --git a/config/mappings/mausamgram/weather-observation.select.yaml b/config/mappings/mausamgram/weather-observation.select.yaml index 5d2659fe..ba8234a0 100644 --- a/config/mappings/mausamgram/weather-observation.select.yaml +++ b/config/mappings/mausamgram/weather-observation.select.yaml @@ -1,9 +1,8 @@ # Mausamgram, openagrinet:WeatherObservation, select. Both directions, one file. # -# One file per binding-action rather than one per direction, because the two -# halves are not independent: the response half below reads _local, and the only -# reason _local holds a lat and a lon is that the request half put them there. -# Splitting them across two registry fields hid that dependency; this does not. +# One file per binding-action rather than one per direction, because both legs of +# one upstream call are one unit of configuration: they are published, reviewed +# and retired together, and a reference to one is a reference to the other. # # The registry entry pointing here decides which action this serves, so nothing # in the file names it. The filename's action segment must match that entry -- @@ -12,15 +11,18 @@ # Both halves read: # beckn the inbound Beckn payload -- the context to echo, the offer to # quote against -# _local whatever the provider step resolved before the call # and the response half additionally reads: # response the provider's answer, in its own shape +# +# Nothing else is in scope. Values the provider step resolved before the call are +# not passed in: the step holds them and used them to make the call, so a mapping +# reading them back would be a second name for the same data. Where the answer +# needs them, it takes them from what the provider echoed. -# An empty request half is a statement, not an omission: this provider takes its -# parameters in the query string, the step has already resolved them, and putting -# them through a transform to arrive at the same two fields would buy nothing. -# The step is told so explicitly (ErrNoTransform) rather than being handed an -# empty document. +# An empty request half means there is no request document to build. This +# provider takes its parameters in the query string and the step resolved them +# already, so it sends those directly. For a method that takes a body, an empty +# half would mean exactly what it says: no body. request: "" # Keyed by direction, not by the action it produces: a select is answered by an @@ -28,8 +30,8 @@ request: "" # than an action of its own. response: | ( - $lat := _local.lat; - $lon := _local.lon; + $lat := response.location.lat; + $lon := response.location.lon; $days := [ response.fcstday1, response.fcstday2, response.fcstday3, response.fcstday4, response.fcstday5 diff --git a/pkg/plugin/definition/mapper.go b/pkg/plugin/definition/mapper.go index 458a2c8b..369966fa 100644 --- a/pkg/plugin/definition/mapper.go +++ b/pkg/plugin/definition/mapper.go @@ -2,11 +2,10 @@ package definition import ( "context" - "errors" ) // Direction names which half of a mapping to run. A mapping file carries both, -// because the response half usually depends on what the request half did. +// because both legs of one upstream call belong together. type Direction string const ( @@ -32,6 +31,15 @@ type Mapper interface { // // Which action the mapping serves is settled by the registry entry that // named it, so only the direction is passed here. + // + // input carries what a party sent: the inbound payload, and on the way back + // the provider's answer. It deliberately does not carry values the caller + // resolved for itself -- the caller holds those already, so routing them + // through a mapping would be a detour and a second name for the same data. + // + // A direction the file has no transform for produces nothing, with no error. + // What nothing means belongs to the caller: on the request leg it means there + // is no document to send. Transform(ctx context.Context, mappingRef string, direction Direction, input any) ([]byte, error) } @@ -39,19 +47,3 @@ type Mapper interface { type MapperProvider interface { New(ctx context.Context, config map[string]string) (Mapper, func() error, error) } - -// ErrNoTransform reports an action a mapping file declares but leaves empty. -// -// An empty mapping is a statement, not an omission: this action needs no -// document built for it, because the caller supplies the request itself. A -// provider taking two query parameters is the ordinary case -- the values are -// already resolved, and passing them through a fetch and a compile to arrive at -// the same two fields buys nothing. -// -// It is a sentinel rather than an empty result so that a caller which does not -// handle it fails loudly. Returning (nil, nil) would let one send an empty -// request instead, which a provider answers with a 200 and the wrong data. -// -// An action absent from the file is a different thing entirely: that capability -// does not serve it, and Transform refuses. -var ErrNoTransform = errors.New("mapping declares this action but supplies no transform") diff --git a/pkg/plugin/implementation/jsonmapper/README.md b/pkg/plugin/implementation/jsonmapper/README.md index 5fbbf68e..c1fa59b4 100644 --- a/pkg/plugin/implementation/jsonmapper/README.md +++ b/pkg/plugin/implementation/jsonmapper/README.md @@ -31,28 +31,26 @@ One file per binding-action, carrying **both directions**: # mappings/mausamgram/weather-observation.select.yaml request: | { - "lat": _local.lat, - "lon": _local.lon + "lat": beckn.message.contract.commitments[0].resources[0].resourceAttributes.location.coordinates[1] } response: | { "rainfall": response.fcstday1.rain, - "at": { "type": "Point", "coordinates": [_local.lon, _local.lat] } + "at": response.location } ``` -**One file rather than two because the halves are not independent.** The response -above reads `_local`, and the only reason `_local` holds a lat and a lon is that -the request put them there. Splitting them across two registry fields hid that; -this does not. It also means the response leg of a round trip is already fetched -and compiled by the time it is needed. +**One file rather than two because both legs of one upstream call are one unit of +configuration.** They are published, reviewed and retired together, and a +reference to one is a reference to the other. It also means the response leg is +already fetched and compiled by the time it is needed — one round trip, one fetch. -A half that is absent or empty is a *statement*, not an omission: it reports -`definition.ErrNoTransform`, so a caller that has nothing to build knows to build -its own request rather than sending an empty document. A half that will not -compile is a different thing and reported as an error — the two must not collapse, -or an unmapped upstream answer would go out as a Beckn response. +**A half that is absent or empty produces nothing, with no error.** What nothing +means belongs to the caller: on the request leg it means there is no document to +send. A half that will not compile is a different thing and reported as an error — +the two must not collapse, or an unmapped upstream answer would go out as a Beckn +response. A broken half takes down only itself: a typo in the response mapping is no reason to stop making the call, and finding out on the way back beats finding out before @@ -83,12 +81,13 @@ stays in the record for now.) | key | request leg | response leg | |---|---|---| | `beckn` | the inbound Beckn payload | the inbound Beckn payload | -| `_local` | values the provider plugin resolved | the same values | | `response` | — | the provider's raw answer | -`_local` stays in scope on the response leg on purpose. A provider's answer -rarely repeats what it was asked, so values resolved before the call are often -the only source for them in the output — the coordinates of a forecast, say. +**What a party sent, and nothing else.** Values a provider plugin resolved before +the call are deliberately not passed in: the plugin holds them and used them to +make the call, so a mapping reading them back would be a detour and a second name +for the same data. Where the answer needs such a value, it takes it from what the +provider echoed. ## Why the direction is a parameter, not a convention diff --git a/pkg/plugin/implementation/jsonmapper/jsonmapper.go b/pkg/plugin/implementation/jsonmapper/jsonmapper.go index afbcfb80..a4bf6af8 100644 --- a/pkg/plugin/implementation/jsonmapper/jsonmapper.go +++ b/pkg/plugin/implementation/jsonmapper/jsonmapper.go @@ -45,13 +45,14 @@ const codeAdaptationFailed = "SCH_SCHEMA_ADAPTATION_FAILED" // mappingFile is the published form of a mapping: one binding-action, both // directions. // -// One file rather than two because the response mapping usually depends on what -// the request mapping did -- swapping GeoJSON coordinates into named lat and lon, -// say -- and splitting them across two registry fields hides that. Which action -// the file serves is decided by the registry entry that points at it, so nothing -// inside needs to name it. +// One file rather than two because both legs of one upstream call are one unit +// of configuration: they are published, reviewed and retired together, and a +// reference to one is a reference to the other. Which action the file serves is +// decided by the registry entry that points at it, so nothing inside names it. // -// An empty request is a statement rather than an omission; see ErrNoTransform. +// A half may be absent or empty. That is not an omission to report -- it means +// there is no transform for that direction, and what that means belongs to the +// caller. type mappingFile struct { Request string `yaml:"request"` Response string `yaml:"response"` @@ -110,12 +111,21 @@ type cacheEntry struct { // compiling. Failures are held per half deliberately: a typo in the response // mapping is no reason for the request half to stop working, and finding out on // the way out beats finding out before the call was even made. +// +// A nil expression with no error is a half the file carries no transform for -- +// held rather than treated as absent, so "there is no request half" and "the +// file could not be read" stay different answers. type compiledMapping struct { expression jsonata.Expression evaluating *sync.Mutex err error } +// hasTransform reports whether this half has something to run. +func (m *compiledMapping) hasTransform() bool { + return m != nil && m.expression != nil +} + // Mapper fetches, compiles and runs mappings. It is safe for concurrent use: // one mapper serves every inbound request. type Mapper struct { @@ -194,6 +204,12 @@ func (m *Mapper) Transform(ctx context.Context, mappingRef string, direction def if mapping.err != nil { return nil, mapping.err } + if !mapping.hasTransform() { + // Nothing to apply. That is an answer, not a failure: the caller decides + // what an absent transform means for the leg it is on. + log.Debugf(ctx, "JSON mapping %s carries no %s transform", mappingRef, direction) + return nil, nil + } return m.evaluate(ctx, mapping, mappingRef, direction, input) } @@ -283,11 +299,9 @@ func (m *Mapper) fetchAndCompile(ctx context.Context, mappingRef string) (map[de // compileMapping compiles one half, keeping any failure local to it. func (m *Mapper) compileMapping(ctx context.Context, mappingRef string, direction definition.Direction, source string) *compiledMapping { if strings.TrimSpace(source) == "" { - // Present but empty. That is a statement rather than an omission -- see - // definition.ErrNoTransform -- so it is held as this half's outcome and - // reported to whoever asks for it, leaving the other half unaffected. - return &compiledMapping{err: fmt.Errorf("jsonmapper: mapping %q %s half: %w", - mappingRef, direction, definition.ErrNoTransform)} + // No transform for this direction. Not an error: a request half is + // legitimately empty when the caller builds its own request. + return &compiledMapping{} } expression, err := m.instance.Compile(source, false) if err != nil { @@ -372,9 +386,9 @@ func parseMapping(body []byte) (mappingFile, error) { return file, nil } -// marshalInput renders the named inputs a mapping reads -- beckn, _local and, -// on the response leg, response -- as the single JSON document JSONata -// evaluates against. +// marshalInput renders the named inputs a mapping reads -- beckn and, on the +// response leg, response -- as the single JSON document JSONata evaluates +// against. func marshalInput(input any) ([]byte, error) { document, err := json.Marshal(input) if err != nil { diff --git a/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go b/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go index 0d410382..f9a9f1f3 100644 --- a/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go +++ b/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go @@ -3,7 +3,6 @@ package jsonmapper import ( "context" "encoding/json" - "errors" "fmt" "net/http" "net/http/httptest" @@ -15,26 +14,28 @@ import ( "github.com/beckn-one/beckn-onix/pkg/plugin/definition" ) -// bothDirections is the published form: one binding-action, both halves. The -// response half reads _local, which is what makes one file the right unit -- -// it only works because the request half put lat and lon there. +// bothDirections is the published form: one binding-action, both halves. +// +// A mapping reads only what a party sent -- the inbound payload, and on the way +// back the provider's answer. Values a provider plugin resolved before the call +// are not passed in: the plugin holds them already and uses them directly, so +// routing them through the mapping would be a detour. const bothDirections = `request: | - { "lat": _local.lat, "txn": beckn.context.transactionId } + { "txn": beckn.context.transactionId } response: | - { "lat": _local.lat, "txn": beckn.context.transactionId, "rain": response.fcstday1.rain } + { "txn": beckn.context.transactionId, "rain": response.fcstday1.rain } ` -// requestOnly is the shape of a mapping for an action whose answer needs no -// translation, or whose response half has not been written yet. +// requestOnly is the shape of a mapping whose answer needs no translation, or +// whose response half has not been written yet. const requestOnly = `request: | - { "lat": _local.lat } + { "txn": beckn.context.transactionId } ` func requestInput() map[string]any { return map[string]any{ - "beckn": map[string]any{"context": map[string]any{"transactionId": "txn-123", "messageId": "msg-1"}}, - "_local": map[string]any{"lat": 19.9975, "lon": 73.7898}, + "beckn": map[string]any{"context": map[string]any{"transactionId": "txn-123", "messageId": "msg-1"}}, } } @@ -101,18 +102,14 @@ func TestTransformRunsTheRequestHalf(t *testing.T) { if err := json.Unmarshal(got, &result); err != nil { t.Fatalf("failed to decode the result: %v", err) } - if result["lat"] != 19.9975 { - t.Errorf("lat = %v, want 19.9975 -- _local was not reachable", result["lat"]) - } if result["txn"] != "txn-123" { t.Errorf("txn = %v, want txn-123 -- beckn was not reachable", result["txn"]) } } -// The response half reads the upstream answer under response, and still reads -// _local -- which is the argument for one file: the provider does not repeat the -// point it was asked about, so the answer is only mappable next to the request -// that produced it. +// The response half reads the upstream answer under response, alongside the +// original payload under beckn -- the context to echo and the offer to quote +// against are only in the request that produced the answer. func TestTransformRunsTheResponseHalfAlongsideTheRequest(t *testing.T) { t.Parallel() @@ -131,7 +128,7 @@ func TestTransformRunsTheResponseHalfAlongsideTheRequest(t *testing.T) { for _, field := range []struct { key string want any - }{{"lat", 19.9975}, {"txn", "txn-123"}, {"rain", 12.4}} { + }{{"txn", "txn-123"}, {"rain", 12.4}} { if result[field.key] != field.want { t.Errorf("%s = %v, want %v", field.key, result[field.key], field.want) } @@ -169,13 +166,13 @@ response: | } } -// --- a half that supplies no transform -------------------------------------- +// --- a half with no transform ---------------------------------------------- -// The ordinary case for a provider taking query parameters, or one whose answer -// is already in shape: the file carries the other half, and this one is reported -// as its own sentinel so a caller that does not handle it fails loudly rather -// than sending an empty document. -func TestTransformReportsAHalfWithNoTransform(t *testing.T) { +// A half that is absent, or present and empty, has no transform to apply. That +// is not a failure and not a special case: it produces nothing, and the caller +// decides what nothing means for the leg it is on. A request half with no +// transform means no request document -- so no body. +func TestTransformProducesNothingForAHalfWithNoTransform(t *testing.T) { t.Parallel() testCases := []struct { @@ -183,7 +180,7 @@ func TestTransformReportsAHalfWithNoTransform(t *testing.T) { mapping string }{ {"the half is absent", requestOnly}, - {"the half is present and empty", "request: |\n { \"lat\": _local.lat }\nresponse: \"\"\n"}, + {"the half is present and empty", "request: |\n { \"txn\": beckn.context.transactionId }\nresponse: \"\"\n"}, } for _, tc := range testCases { @@ -194,20 +191,52 @@ func TestTransformReportsAHalfWithNoTransform(t *testing.T) { defer srv.Close() mapper := newTestMapper(t) - _, err := mapper.Transform(context.Background(), ref(srv.URL), definition.DirectionResponse, responseInput()) - if !errors.Is(err, definition.ErrNoTransform) { - t.Errorf("expected ErrNoTransform, got %v", err) + got, err := mapper.Transform(context.Background(), ref(srv.URL), definition.DirectionResponse, responseInput()) + if err != nil { + t.Errorf("a half with no transform is not an error, got %v", err) + } + if len(got) != 0 { + t.Errorf("produced %q, want nothing", got) } - // The other half is unaffected: one direction needing no transform + // The other half is unaffected: one direction having no transform // says nothing about the other. - if _, err := mapper.Transform(context.Background(), ref(srv.URL), definition.DirectionRequest, requestInput()); err != nil { + out, err := mapper.Transform(context.Background(), ref(srv.URL), definition.DirectionRequest, requestInput()) + if err != nil { t.Errorf("the other half must still be served: %v", err) } + if len(out) == 0 { + t.Error("the other half produced nothing, want its output") + } }) } } +// Nothing and a failure must stay distinguishable. A half that will not compile +// produces an error, not nothing -- reading it as nothing would send an unmapped +// upstream answer out as a Beckn response. +func TestTransformSeparatesNothingFromAFailure(t *testing.T) { + t.Parallel() + + mapping := `request: "" + +response: | + {{{ +` + srv := newMappingServer(t, mapping, nil) + defer srv.Close() + mapper := newTestMapper(t) + + got, err := mapper.Transform(context.Background(), ref(srv.URL), definition.DirectionRequest, requestInput()) + if err != nil || len(got) != 0 { + t.Errorf("the empty half: got %q, %v -- want nothing and no error", got, err) + } + + if _, err := mapper.Transform(context.Background(), ref(srv.URL), definition.DirectionResponse, responseInput()); err == nil { + t.Error("the uncompilable half must report an error, not nothing") + } +} + // --- direction validation --------------------------------------------------- // A direction outside the two is a caller bug, not a mapping problem, and must @@ -320,7 +349,7 @@ func TestTransformIsolatesABrokenHalf(t *testing.T) { t.Parallel() mapping := `request: | - { "lat": _local.lat } + { "txn": beckn.context.transactionId } response: | {{{ @@ -339,12 +368,6 @@ response: | if err == nil { t.Fatal("expected an uncompilable half to be refused") } - // A half that will not compile is broken, not absent, and must not be read - // as "supplies no transform" -- that would send the upstream answer through - // unmapped. - if errors.Is(err, definition.ErrNoTransform) { - t.Error("an uncompilable half must not report ErrNoTransform") - } } // A mapping is fetched into memory and compiled, so an unbounded one is an diff --git a/pkg/plugin/implementation/mausamgram/mappings_test.go b/pkg/plugin/implementation/mausamgram/mappings_test.go index 38d0bc6d..67ecf314 100644 --- a/pkg/plugin/implementation/mausamgram/mappings_test.go +++ b/pkg/plugin/implementation/mausamgram/mappings_test.go @@ -11,7 +11,6 @@ package mausamgram_test import ( "context" "encoding/json" - "errors" "fmt" "net/http" "net/http/httptest" @@ -208,8 +207,8 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { t.Errorf("advisory = %v, want the provider's warning", attributes["advisory"]) } - // The point came from _local, not from the provider: it is the request's - // own coordinates, in GeoJSON order. + // GeoJSON order, and the provider's own echo of the point: the mapping reads + // response.location rather than anything the step resolved. location, _ := attributes["location"].(map[string]any) coordinates, _ := location["coordinates"].([]any) if len(coordinates) != 2 || coordinates[0] != 73.7898 || coordinates[1] != 19.9975 { @@ -238,11 +237,10 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { } } -// The shipped file's request half is deliberately empty: this provider takes -// its parameters in the query string. That has to reach the step as -// ErrNoTransform rather than as an empty document, or the step would send a -// body-shaped nothing instead of building the query itself. -func TestShippedMappingsDeclareNoRequestTransform(t *testing.T) { +// The shipped file's request half is deliberately empty: this provider takes its +// parameters in the query string. That has to produce nothing rather than an +// empty document, so the step builds the query itself. +func TestShippedMappingsProduceNoRequestDocument(t *testing.T) { mappings := serveMappings(t) defer mappings.Close() @@ -253,15 +251,14 @@ func TestShippedMappingsDeclareNoRequestTransform(t *testing.T) { defer closeMapper() ref := mappings.URL + "/" + shippedMapping - input := map[string]any{"_local": map[string]any{"lat": 1.0, "lon": 2.0}} + input := map[string]any{"beckn": map[string]any{"context": map[string]any{"action": "select"}}} - if _, err := mapper.Transform(context.Background(), ref, definition.DirectionRequest, input); !errors.Is(err, definition.ErrNoTransform) { - t.Errorf("the request half should report ErrNoTransform, got %v", err) + got, err := mapper.Transform(context.Background(), ref, definition.DirectionRequest, input) + if err != nil { + t.Errorf("the empty request half must not be an error, got %v", err) } - // The response half of the same file is unaffected -- which is the point of - // holding both in one file rather than inferring one from the other. - if _, err := mapper.Transform(context.Background(), ref, definition.DirectionResponse, input); errors.Is(err, definition.ErrNoTransform) { - t.Error("the response half must carry a transform") + if len(got) != 0 { + t.Errorf("the request half produced %q, want nothing", got) } } diff --git a/pkg/plugin/implementation/mausamgram/mausamgram.go b/pkg/plugin/implementation/mausamgram/mausamgram.go index 40fe6d34..204839a1 100644 --- a/pkg/plugin/implementation/mausamgram/mausamgram.go +++ b/pkg/plugin/implementation/mausamgram/mausamgram.go @@ -205,7 +205,7 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { return err } - upstreamRequest, err := s.buildRequest(ctx, call.Mappings, beckn, local) + upstreamRequest, err := s.buildRequest(ctx, call, beckn, local) if err != nil { return err } @@ -220,20 +220,28 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { return fmt.Errorf("mausamgram: provider answered with something that is not JSON: %w", err) } - // _local stays in scope: the provider does not repeat the point it was asked - // about, so the output's own coordinates have no other source. - // // The same mapping reference as the request, other half: one file carries - // both directions for this action, because the response mapping usually - // depends on what the request mapping did. + // both directions for this action. + // + // The mapping is handed what each party sent and nothing else. The values + // resolved above are not passed in: this step holds them and used them to + // make the call, so handing them to the mapping would be a second name for + // the same data. becknResponse, err := s.mapper.Transform(ctx, call.Mappings, definition.DirectionResponse, map[string]any{ "beckn": beckn, - "_local": local, "response": answer, }) if err != nil { return err } + if len(becknResponse) == 0 { + // Either the file has no response half, or its transform matched nothing + // in this answer. Both leave no Beckn response to return, and returning + // the provider's own shape instead would be worse than failing. The + // message says what was observed rather than guessing which it was. + return fmt.Errorf("mausamgram: the response half of %s produced nothing, so %s cannot be answered", + call.Mappings, plan.BindingKey) + } ctx.ResponseBody = becknResponse log.Infof(ctx, "mausamgram: served %s in %d bytes", plan.BindingKey, len(becknResponse)) @@ -253,27 +261,40 @@ func servedActions(plan *model.ProviderRecord) []string { // buildRequest produces what the provider is sent. // -// A mapping whose request half is empty is saying the request needs no document -// built for it: this provider takes its parameters in the query, they are -// already resolved, and putting them through a fetch and a compile to arrive at -// the same two fields buys nothing. In that case the resolved values ARE the -// parameters. +// The mapping is handed the inbound payload and nothing else, and it produces a +// document or it produces nothing. Nothing is the ordinary case here: this +// provider takes its parameters in the query string, this step resolved them +// already, and putting them through a fetch and a compile to arrive at the same +// two fields would buy nothing. // -// Anything else -- a provider wanting a body in its own shape -- goes through -// the mapping, which is what the mapping is for. -func (s *Step) buildRequest(ctx context.Context, mappingRef string, beckn any, local point) ([]byte, error) { - mapped, err := s.mapper.Transform(ctx, mappingRef, definition.DirectionRequest, map[string]any{ - "beckn": beckn, - "_local": local, +// What nothing means depends on the method, and both readings are deliberate: +// +// - a method with no body -- the resolved values ARE the parameters. This step +// knows this provider, so it does not need the mapping's help to call it. +// - a method with a body -- there is no body. An empty mapping means an empty +// request, not the resolved values dressed up as one; a body is the +// mapping's business and it supplied none. +// +// A half with no transform and a transform that matched nothing are treated +// alike here, deliberately: for the request leg there is no document either way, +// and inventing one would send the provider something nobody asked for. +func (s *Step) buildRequest(ctx context.Context, call model.ActionPlan, beckn any, local point) ([]byte, error) { + mapped, err := s.mapper.Transform(ctx, call.Mappings, definition.DirectionRequest, map[string]any{ + "beckn": beckn, }) - if err == nil { + if err != nil { + return nil, err + } + if len(mapped) > 0 { return mapped, nil } - if !errors.Is(err, definition.ErrNoTransform) { - return nil, err + + if hasBody(call.Method) { + log.Debugf(ctx, "mausamgram: the request half of %s produced nothing; sending no body", call.Mappings) + return nil, nil } - log.Debugf(ctx, "mausamgram: %s carries no request half; sending the resolved point", mappingRef) + log.Debugf(ctx, "mausamgram: the request half of %s produced nothing; sending the resolved point", call.Mappings) parameters, err := json.Marshal(local) if err != nil { return nil, fmt.Errorf("mausamgram: could not encode the resolved point: %w", err) diff --git a/pkg/plugin/implementation/mausamgram/mausamgram_test.go b/pkg/plugin/implementation/mausamgram/mausamgram_test.go index fd78813d..06b699ad 100644 --- a/pkg/plugin/implementation/mausamgram/mausamgram_test.go +++ b/pkg/plugin/implementation/mausamgram/mausamgram_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "slices" + "sort" "strings" "sync/atomic" "testing" @@ -249,20 +250,55 @@ func TestRunKeepsResolvedValuesInScopeForTheResponse(t *testing.T) { if !ok { t.Fatalf("response input = %T, want a map", mapper.responseInput) } - for _, key := range []string{"beckn", "_local", "response"} { + for _, key := range []string{"beckn", "response"} { if _, present := input[key]; !present { t.Errorf("response mapping cannot see %q", key) } } - local, ok := input["_local"].(point) + // A mapping is handed only what a party sent. Values this step resolved + // before the call stay in the step: it holds them already and uses them + // directly, so passing them through the mapping would be a detour and a + // second name for the same data. + if len(input) != 2 { + t.Errorf("response input carries %d keys (%v), want exactly beckn and response", + len(input), keysOf(input)) + } +} + +// keysOf names what an input document carries, for a readable failure. +func keysOf(input map[string]any) []string { + names := make([]string, 0, len(input)) + for name := range input { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// The request leg is handed the payload and nothing else. +func TestRunGivesTheRequestMappingOnlyTheInboundPayload(t *testing.T) { + t.Parallel() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{}`) + })) + defer upstream.Close() + + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + if _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), selectBody); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + + input, ok := mapper.requestInput.(map[string]any) if !ok { - t.Fatalf("_local = %T, want a point", input["_local"]) + t.Fatalf("request input = %T, want a map", mapper.requestInput) } - // GeoJSON order: the payload carries [lon, lat], so reading it positionally - // the other way round would put this point in the wrong hemisphere. - if local.Lat != 19.9975 || local.Lon != 73.7898 { - t.Errorf("_local = %+v, want lat 19.9975 lon 73.7898", local) + if len(input) != 1 { + t.Errorf("request input carries %v, want only beckn", keysOf(input)) + } + if _, present := input["beckn"]; !present { + t.Error("request mapping cannot see beckn") } } @@ -292,10 +328,11 @@ func TestRunSendsTheMappedBodyForAMethodThatTakesOne(t *testing.T) { } } -// A provider taking query parameters needs no request document built: the -// mapping declares the action and leaves it empty, and the values the step -// already resolved become the parameters. -func TestRunSendsResolvedValuesWhenTheMappingDeclaresNoTransform(t *testing.T) { +// A provider taking query parameters needs no request document built. The +// mapping produces nothing, and the values this step resolved become the +// parameters -- the step knows this provider, so it does not need the mapping's +// help to call it. +func TestRunSendsResolvedValuesWhenTheMappingProducesNothing(t *testing.T) { t.Parallel() var gotQuery string @@ -305,7 +342,7 @@ func TestRunSendsResolvedValuesWhenTheMappingDeclaresNoTransform(t *testing.T) { })) defer upstream.Close() - mapper := &stubMapper{requestErr: definition.ErrNoTransform, responseResult: []byte(`{}`)} + mapper := &stubMapper{requestResult: nil, responseResult: []byte(`{}`)} if _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), selectBody); err != nil { t.Fatalf("Run() returned an unexpected error: %v", err) } @@ -317,8 +354,56 @@ func TestRunSendsResolvedValuesWhenTheMappingDeclaresNoTransform(t *testing.T) { } } -// Only ErrNoTransform means "build it yourself". Any other mapping failure is a -// real failure and must not be papered over by sending the resolved values. +// A method that takes a body, and a mapping that produces nothing, means no +// body -- not the resolved values dressed up as one. Query parameters are the +// step's own doing; a body is the mapping's, and there is nothing to send. +func TestRunSendsNoBodyWhenTheMappingProducesNothing(t *testing.T) { + t.Parallel() + + var gotBody string + var gotLength int64 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotLength = r.ContentLength + body := make([]byte, 64) + n, _ := r.Body.Read(body) + gotBody = string(body[:n]) + fmt.Fprint(w, `{}`) + })) + defer upstream.Close() + + mapper := &stubMapper{requestResult: nil, responseResult: []byte(`{}`)} + if _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodPost)}, mapper), selectBody); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + + if gotLength > 0 || gotBody != "" { + t.Errorf("upstream got a %d byte body (%q), want none", gotLength, gotBody) + } +} + +// A response half that produces nothing leaves no Beckn answer to return. +// Failing is the only honest outcome: answering with the provider's own shape +// would put a non-Beckn body on the wire under a valid signature. +func TestRunRefusesWhenTheResponseMappingProducesNothing(t *testing.T) { + t.Parallel() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"fcstday1":{"rain":12.4}}`) + })) + defer upstream.Close() + + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: nil} + ctx, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), selectBody) + if err == nil { + t.Fatal("expected an empty response mapping to be refused") + } + if len(ctx.ResponseBody) != 0 { + t.Errorf("ResponseBody = %q, want nothing written", ctx.ResponseBody) + } +} + +// A mapping that failed is a failure, and must not be papered over by sending +// the resolved values instead. func TestRunDoesNotSubstituteForARealMappingFailure(t *testing.T) { t.Parallel() From 8d05338af0e2ab75a1099288b30f60d283ce8c6b Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Tue, 1 Sep 2026 00:57:28 +0530 Subject: [PATCH 031/130] fix: refuse an unanswered request in a provider module [#1] A module that serves capabilities itself has no proxy behind it. When no step answered and no route was set, nothing ever will: there is no route to forward the request and nobody to send a callback. ONIX answered that with an ACK, which tells the caller "accepted, answer follows" and leaves it waiting for a message that is never coming. That is not a theoretical case. It is what a stale binding key looks like: the provider step reads a binding from the payload, does not recognise it, passes through -- which is the dispatch mechanism working correctly -- and the request falls out of the pipeline unanswered. The adapter then reports success. It cost two rounds of confusion during this work before the ACK was read as a symptom rather than as expected behaviour. So a module with provider steps now refuses an unanswered no-route request with 404 NET_ENTITY_NOT_FOUND. Modules without provider steps are untouched: an unanswered request there is the publisher or proxy path doing exactly what it should. 404 rather than AckNoCallbackErr, which exists for this shape and was the obvious pick. It maps to 202 Accepted, and a 2xx is what let this hide in the first place; it is also for a business outcome -- no inventory, provider closed -- where this is "nothing here serves that", which is what a 404 says. The check sits before the response steps, not after: ackSigner signs the body it expects to be written, so NACKing later would ship a signature over the ACK with a NACK body. No single provider step could make this decision. Several sit in one pipeline and each passes through what is not its own, so a step seeing a foreign binding cannot know whether a later step will serve it. Only the handler knows, once every step has run, that nobody did. Verified end to end: a select naming an unserved provider now returns 404 with NET_ENTITY_NOT_FOUND where it previously returned 200 ACK, and a select for the served capability still returns on_select with three mapped resources. --- core/module/handler/responsebody_test.go | 107 +++++++++++++++++++++++ core/module/handler/stdHandler.go | 36 +++++++- 2 files changed, 141 insertions(+), 2 deletions(-) diff --git a/core/module/handler/responsebody_test.go b/core/module/handler/responsebody_test.go index 7ccc87f9..59121226 100644 --- a/core/module/handler/responsebody_test.go +++ b/core/module/handler/responsebody_test.go @@ -166,6 +166,113 @@ func TestServeHTTPTreatsAnEmptyAnswerAsNoAnswer(t *testing.T) { } } +// --- an unanswered request in a provider module ------------------------------ + +// silentStep is the dispatch no-op: a provider step recognising the request as +// none of its business. Succeeding without answering is how several provider +// steps coexist in one pipeline. +type silentStep struct{} + +func (s *silentStep) Run(*model.StepContext) error { return nil } + +// A module that serves capabilities itself has no proxy behind it. When nothing +// answered and no route was set, nothing ever will: there is nobody to send a +// callback. An ACK there tells the caller "accepted, answer follows" and leaves +// it waiting forever, so this is a NACK. +func TestServeHTTPNacksAnUnansweredRequestInAProviderModule(t *testing.T) { + h := &stdHandler{ + SubscriberID: "test-sub", + role: model.RoleBPP, + moduleName: "test-module", + steps: []definition.Step{&silentStep{}}, + hasProviderSteps: true, + } + + recorder := serve(t, h, v2SelectBody) + + if recorder.Code == http.StatusOK { + t.Fatalf("status = 200 for a request nothing answered; that ACK promises a callback nobody will send") + } + var got model.Response + if err := json.Unmarshal(recorder.Body.Bytes(), &got); err != nil { + t.Fatalf("failed to decode the response: %v", err) + } + if got.Message.Status == model.StatusACK { + t.Errorf("status = %q, want a NACK", got.Message.Status) + } +} + +// The guard must not touch a request that was answered. +func TestServeHTTPStillWritesAnAnswerInAProviderModule(t *testing.T) { + answer := []byte(`{"context":{"action":"on_select"}}`) + h := &stdHandler{ + SubscriberID: "test-sub", + role: model.RoleBPP, + moduleName: "test-module", + steps: []definition.Step{&answeringStep{answer: answer}}, + hasProviderSteps: true, + } + + recorder := serve(t, h, v2SelectBody) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", recorder.Code) + } + if recorder.Body.String() != string(answer) { + t.Errorf("body = %s, want the step's own answer", recorder.Body.String()) + } +} + +// A routed request is untouched even in a provider module. This is the ordinary +// case for a module that serves one capability itself and proxies everything +// else: the provider step passes through, the router sets a route, and the proxy +// owns the response -- so the ACK it produces means what it says. +func TestServeHTTPLeavesARoutedRequestAloneInAProviderModule(t *testing.T) { + h := &stdHandler{ + SubscriberID: "test-sub", + role: model.RoleBPP, + moduleName: "test-module", + // proxy() reaches straight for httpClient.Transport, so a routed handler + // without one panics rather than failing. + httpClient: http.DefaultClient, + steps: []definition.Step{&silentStep{}, &routeSettingStep{}}, + hasProviderSteps: true, + } + + recorder := serve(t, h, v2SelectBody) + + // The route points at an unreachable host, so what comes back is the proxy's + // own failure. What matters is that it is the proxy answering at all: the + // guard belongs to the no-route branch and must not have fired. + if strings.Contains(recorder.Body.String(), "NET_ENTITY_NOT_FOUND") { + t.Errorf("body = %s -- the unanswered guard fired on a routed request", recorder.Body.String()) + } +} + +// A module with no provider steps is untouched. Its ACK still means what it has +// always meant: a proxy or a publisher carries the work on from here. +func TestServeHTTPStillAcksInAModuleWithoutProviderSteps(t *testing.T) { + h := &stdHandler{ + SubscriberID: "test-sub", + role: model.RoleBAP, + moduleName: "test-module", + steps: []definition.Step{&silentStep{}}, + } + + recorder := serve(t, h, v2SelectBody) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", recorder.Code) + } + var got model.Response + if err := json.Unmarshal(recorder.Body.Bytes(), &got); err != nil { + t.Fatalf("failed to decode the response: %v", err) + } + if got.Message.Status != model.StatusACK { + t.Errorf("status = %q, want the generated ACK", got.Message.Status) + } +} + // The instrumentor shallow-copies the context in but copies only named fields // back out. An answer written by an instrumented step has to survive that, or // it works unwrapped and vanishes wrapped -- and wrapped is the default. diff --git a/core/module/handler/stdHandler.go b/core/module/handler/stdHandler.go index 1995f7ff..8969ea6c 100644 --- a/core/module/handler/stdHandler.go +++ b/core/module/handler/stdHandler.go @@ -59,8 +59,12 @@ type stdHandler struct { // ackSigner is non-nil only when the "signAck" step is configured (Receiver // modules). It is also used to sign pipeline-NACK responses so that ALL // synchronous responses carry a Signature header per NFH-007 CON-004-02. - ackSigner *ackSignerStep - SubscriberID string + ackSigner *ackSignerStep + // hasProviderSteps records whether this module serves capabilities itself. + // Such a module has no proxy behind it, which is what makes an unanswered + // request a dead end rather than work in flight -- see ServeHTTP. + hasProviderSteps bool + SubscriberID string role model.Role basePath string httpClient *http.Client @@ -216,6 +220,33 @@ func (h *stdHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Restore request body and metadata before forwarding or publishing. syncRequestBody(r, stepCtx.Body) if stepCtx.Route == nil { + // A module that serves capabilities itself has no proxy behind it, so + // an unanswered request here is a dead end: no route to forward it, and + // nobody to send a callback. An ACK would tell the caller "accepted, + // answer follows" and leave it waiting for a message nobody will send, + // which is how a stale binding key hides as a healthy response. + // + // 404 rather than AckNoCallbackErr, which exists for this shape and + // would be the obvious pick: it maps to 202 Accepted, and a 2xx is what + // let this hide in the first place. It is also for a business outcome + // -- no inventory, provider closed -- where this is "nothing here + // serves that", which is what a 404 says. + // + // Checked before the response steps rather than after, because + // ackSigner signs the body it expects to be written; NACKing later + // would ship a signature over the ACK with a NACK body. + // + // Only for modules with provider steps. Elsewhere an unanswered + // request is the publisher path doing exactly what it should. + if h.hasProviderSteps && len(stepCtx.ResponseBody) == 0 { + err = model.NewNotFoundErr("", fmt.Errorf( + "this module serves no capability matching the request")) + log.Errorf(stepCtx, err, "No step answered and no route was set: %v", err) + h.signNackResponse(stepCtx, err) + responseBody = sendNack(stepCtx, wrapped, err) + return + } + // No routing — ONIX writes the ACK directly. Run response steps here // with resp=nil (publisher path semantics). for _, step := range h.responseSteps { @@ -653,6 +684,7 @@ func (h *stdHandler) initSteps(ctx context.Context, mgr PluginManager, cfg *Conf } steps[c.ID] = step } + h.hasProviderSteps = len(cfg.Plugins.ProviderSteps) > 0 // Register processing steps for _, step := range cfg.Steps { From 5ccf772e86514aa8288ab4695eb9c5836f7d9696 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Tue, 1 Sep 2026 01:43:35 +0530 Subject: [PATCH 032/130] fix: quote the resource the request selected [#1] on_select minted a resource per forecast day, with ids derived from the date. The offer, echoed from the request, still referenced the id the consumer selected -- so offer.resourceIds pointed at something that appeared nowhere in the answer. The spec says resourceIds are "references to resources covered by this offer", and ours resolved against nothing. The answer now quotes ONE resource, carrying the id the request selected, with the forecast days under resourceAttributes.observations. That is the better model independently of the bug: the consumer asked for a quote on one resource, so that is what is quoted, and the days are content of it rather than resources of their own. It also fixes the reference by construction rather than by patching resourceIds to match invented ids. Fields that are the same for every day -- the point, the source, the observation type -- now appear once at the top instead of being repeated per resource. Only what varies per day repeats. Mapping file and its test only; no code. Both ends of the translation are data, which is what makes a response-shape change configuration. Left alone deliberately: Commitment.resources requires "quantity", but the spec defines no quantity property on Resource and no Quantity schema, so any value would satisfy it. Inventing one would commit us to a shape the spec has not chosen. Verified end to end against the published mapping: one resource carrying the requested id, three observations inside it, offer.resourceIds resolving, no errors. --- .../weather-observation.select.yaml | 53 ++++++++++----- .../mausamgram/mappings_test.go | 64 ++++++++++++++----- 2 files changed, 84 insertions(+), 33 deletions(-) diff --git a/config/mappings/mausamgram/weather-observation.select.yaml b/config/mappings/mausamgram/weather-observation.select.yaml index ba8234a0..2f72054c 100644 --- a/config/mappings/mausamgram/weather-observation.select.yaml +++ b/config/mappings/mausamgram/weather-observation.select.yaml @@ -19,6 +19,19 @@ # reading them back would be a second name for the same data. Where the answer # needs them, it takes them from what the provider echoed. +# The answer quotes ONE resource, carrying the id the request selected. The +# forecast days are content of that resource, under resourceAttributes. +# observations -- not resources of their own. +# +# Minting a resource per day was the earlier shape, and it broke the offer: the +# offer is echoed from the request, so its resourceIds still pointed at the id +# the consumer selected while the resources carried freshly invented ones. +# Nothing in the answer resolved that reference. Keeping the selected id is what +# makes it stay true, rather than patching resourceIds to match. +# +# Fields that are the same for every day -- the point, the source, the +# observation type -- sit once at the top. Only what varies per day repeats. + # An empty request half means there is no request document to build. This # provider takes its parameters in the query string and the step resolved them # already, so it sends those directly. For a method that takes a body, an empty @@ -46,6 +59,8 @@ response: | } }; + $selected := beckn.message.contract.commitments[0]; + { "context": { "version": beckn.context.version, @@ -66,10 +81,10 @@ response: | "status": { "descriptor": { "code": "QUOTED", "name": "Quoted" } }, - "offer": beckn.message.contract.commitments[0].offer, - "resources": $map($days, function($day) { + "offer": $selected.offer, + "resources": [ { - "id": "res:mausamgram:forecast:" & $day.date, + "id": $selected.resources[0].id, "resourceAttributes": { "@context": "https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld", "@type": "openagrinet:WeatherObservation", @@ -84,22 +99,26 @@ response: | "type": "Point", "coordinates": [$lon, $lat] }, - "validity": { - "startsAt": $day.date, - "endsAt": $day.date - }, - "parameters": [ - $reading("Rainfall", "Total", "mm", $day.rain), - $reading("Temperature", "Minimum", "Cel", $day.tmin), - $reading("Temperature", "Maximum", "Cel", $day.tmax), - $reading("Humidity", "Minimum", "%", $day.rhmin), - $reading("Humidity", "Maximum", "%", $day.rhmax), - $reading("WindSpeed", "Average", "m/s", $day.wspd) - ], - "advisory": $day.weather_warning ? $day.weather_warning : $day.cloud_message + "observations": $map($days, function($day) { + { + "validity": { + "startsAt": $day.date, + "endsAt": $day.date + }, + "parameters": [ + $reading("Rainfall", "Total", "mm", $day.rain), + $reading("Temperature", "Minimum", "Cel", $day.tmin), + $reading("Temperature", "Maximum", "Cel", $day.tmax), + $reading("Humidity", "Minimum", "%", $day.rhmin), + $reading("Humidity", "Maximum", "%", $day.rhmax), + $reading("WindSpeed", "Average", "m/s", $day.wspd) + ], + "advisory": $day.weather_warning ? $day.weather_warning : $day.cloud_message + } + }) } } - }) + ] } ] } diff --git a/pkg/plugin/implementation/mausamgram/mappings_test.go b/pkg/plugin/implementation/mausamgram/mappings_test.go index 67ecf314..1934e59c 100644 --- a/pkg/plugin/implementation/mausamgram/mappings_test.go +++ b/pkg/plugin/implementation/mausamgram/mappings_test.go @@ -28,6 +28,10 @@ import ( // mappingsDir is where the shipped mappings live, relative to this package. const mappingsDir = "../../../../config/mappings/mausamgram" +// selectedResourceID is the resource the request selects, and therefore the one +// the answer quotes. It is the same string in both directions on purpose. +const selectedResourceID = "res:mausamgram:point-forecast" + // shippedMapping is the file this binding-action publishes: one file, both // directions. The registry carries its full URL; the action segment of the name // must match the action that registry entry declares -- a mismatch would apply a @@ -185,28 +189,42 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { t.Error("the quoted commitment carries no offer") } + // One resource, carrying the id the request selected. The consumer asked for + // a quote on one resource, so that is what is quoted -- the forecast days are + // content of it, not resources of their own. + // + // Minting a resource per day would leave offer.resourceIds pointing at an id + // that appears nowhere in the answer, because the offer is echoed from the + // request. Keeping the id is what makes that reference stay true. resources, _ := commitment["resources"].([]any) - if len(resources) != 3 { - t.Fatalf("got %d resources, want 3 -- one per day the provider answered with", len(resources)) + if len(resources) != 1 { + t.Fatalf("got %d resources, want 1 -- the one the request selected", len(resources)) + } + + only, _ := resources[0].(map[string]any) + if only["id"] != selectedResourceID { + t.Errorf("resource id = %v, want the requested %q", only["id"], selectedResourceID) } - // --- the first day, in full --------------------------------------------- - first, _ := resources[0].(map[string]any) - if first["id"] != "res:mausamgram:forecast:2026-08-26" { - t.Errorf("resource id = %v, want it derived from the forecast date", first["id"]) + // The offer's references must resolve against the resources actually + // returned. This is the assertion the previous shape could not satisfy. + offer, _ := commitment["offer"].(map[string]any) + referenced, _ := offer["resourceIds"].([]any) + if len(referenced) != 1 || referenced[0] != selectedResourceID { + t.Errorf("offer.resourceIds = %v, want exactly [%q]", referenced, selectedResourceID) } - attributes, _ := first["resourceAttributes"].(map[string]any) + attributes, _ := only["resourceAttributes"].(map[string]any) if attributes["@type"] != "openagrinet:WeatherObservation" { t.Errorf("@type = %v, want openagrinet:WeatherObservation", attributes["@type"]) } if attributes["observationType"] != "Forecast" { t.Errorf("observationType = %v, want Forecast", attributes["observationType"]) } - if attributes["advisory"] != "Heavy rainfall warning" { - t.Errorf("advisory = %v, want the provider's warning", attributes["advisory"]) - } + // The point, the source and the observation type are the same for every day, + // so they sit once at the top rather than being repeated per day. + // // GeoJSON order, and the provider's own echo of the point: the mapping reads // response.location rather than anything the step resolved. location, _ := attributes["location"].(map[string]any) @@ -215,7 +233,22 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { t.Errorf("coordinates = %v, want [73.7898, 19.9975] in GeoJSON order", coordinates) } - parameters, _ := attributes["parameters"].([]any) + // --- the days, inside the one resource ---------------------------------- + observations, _ := attributes["observations"].([]any) + if len(observations) != 3 { + t.Fatalf("got %d observations, want 3 -- one per day the provider answered with", len(observations)) + } + + first, _ := observations[0].(map[string]any) + validity, _ := first["validity"].(map[string]any) + if validity["startsAt"] != "2026-08-26" { + t.Errorf("first observation starts at %v, want the provider's first forecast date", validity["startsAt"]) + } + if first["advisory"] != "Heavy rainfall warning" { + t.Errorf("advisory = %v, want the provider's warning", first["advisory"]) + } + + parameters, _ := first["parameters"].([]any) if len(parameters) != 6 { t.Errorf("got %d parameters, want 6 for a fully-reported day", len(parameters)) } @@ -226,14 +259,13 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { // --- a day the provider reported only partially -------------------------- // Readings it did not take are absent, not present and empty: a consumer // must be able to tell "no rainfall recorded" from "zero rainfall". - third, _ := resources[2].(map[string]any) - thirdAttributes, _ := third["resourceAttributes"].(map[string]any) - thirdParameters, _ := thirdAttributes["parameters"].([]any) + third, _ := observations[2].(map[string]any) + thirdParameters, _ := third["parameters"].([]any) if len(thirdParameters) != 2 { t.Errorf("got %d parameters for a partly-reported day, want only the 2 taken", len(thirdParameters)) } - if thirdAttributes["advisory"] != nil { - t.Errorf("advisory = %v, want it absent when the provider gave none", thirdAttributes["advisory"]) + if third["advisory"] != nil { + t.Errorf("advisory = %v, want it absent when the provider gave none", third["advisory"]) } } From 343da4ff1d23806cc7d4cdaf5fe5815256548eef Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Tue, 1 Sep 2026 13:04:10 +0530 Subject: [PATCH 033/130] fix: follow the WeatherObservation schema pack [#1] The mapping now produces what openagrinet:WeatherObservation v0.1 requires in Direct mode. The pack lives in OpenAgriNet/network-specs and had not been read when the mapping was first written; four things were missing or wrong. generatedAt was absent, and Direct mode requires it. So was a resource-level validity, which now spans the first forecast day to the last. A warning was a field of its own called "advisory". The pack has no such property, but its parameter enum carries Alert -- so a warning is now a parameter like any other reading, with unit "1", which is what the pack prescribes for a value that has no unit. It inherits the same $exists guard, so a day the provider gave no warning for carries no Alert entry rather than an empty one. The published catalog resource, which carried no informationMode at all, becomes OnDemand: supportedObservationTypes, supportedParameters, forecastHorizon, updateFrequency and geographicGranularities, and deliberately no parameters, which that mode forbids. One @type therefore serves both the catalog and the answer, and informationMode selects which half of the contract applies -- so a discover filtering on the outcome type finds the catalog. TWO FIELDS REMAIN OUTSIDE THE PACK, both deliberate, both recorded in the mapping's header. The pack sets no additionalProperties, so they validate; they are simply not governed. observations The pack carries one validity and one flat parameters array per resource, and every one of its examples is a single period. It cannot express a five-day forecast in the one resource the request selected. Splitting into five resources would return ids the consumer never selected and break the correlation the Contract model rests on, so the days stay inside. aggregation The pack's parameter entry is parameter/value/unit only. This provider reports a minimum AND a maximum for temperature and humidity, indistinguishable without it. @context stays the canonical schemas.openagrinet.global identifier. In JSON-LD that is a name, not a fetch target: it need not resolve today, and substituting a raw git URL that does would put an implementation detail on the wire and break every consumer when the branch is renamed. Mapping file and its test only; no code. Verified end to end against the published mapping: one resource carrying the requested id, three observations, Direct's required fields present, and the warning carried as an Alert parameter. --- .../weather-observation.select.yaml | 55 +++++++++++-- .../mausamgram/mappings_test.go | 79 +++++++++++++++---- 2 files changed, 112 insertions(+), 22 deletions(-) diff --git a/config/mappings/mausamgram/weather-observation.select.yaml b/config/mappings/mausamgram/weather-observation.select.yaml index 2f72054c..8a0d9b7b 100644 --- a/config/mappings/mausamgram/weather-observation.select.yaml +++ b/config/mappings/mausamgram/weather-observation.select.yaml @@ -19,15 +19,42 @@ # reading them back would be a second name for the same data. Where the answer # needs them, it takes them from what the provider echoed. +# The response half follows the openagrinet:WeatherObservation v0.1 schema pack, +# Direct mode. The pack lives in OpenAgriNet/network-specs; it is referred to +# here by name and version rather than by a path, because a path pins a branch +# and a branch moves. +# +# @context is the canonical schemas.openagrinet.global identifier. In JSON-LD +# that is a name, not a fetch target -- it does not have to resolve today, and +# substituting a raw git URL that does would put an implementation detail on the +# wire and break every consumer when the branch is renamed. +# +# Direct mode requires observationType, source, location, generatedAt and +# parameters. informationMode is what selects those requirements: a catalog +# resource advertising this capability is OnDemand instead, and carries +# supportedParameters rather than values. +# # The answer quotes ONE resource, carrying the id the request selected. The -# forecast days are content of that resource, under resourceAttributes. -# observations -- not resources of their own. +# forecast days are content of that resource, under observations. # # Minting a resource per day was the earlier shape, and it broke the offer: the # offer is echoed from the request, so its resourceIds still pointed at the id # the consumer selected while the resources carried freshly invented ones. # Nothing in the answer resolved that reference. Keeping the selected id is what -# makes it stay true, rather than patching resourceIds to match. +# makes it stay true. +# +# TWO FIELDS HERE ARE NOT IN THE PACK, and both are deliberate. The pack sets no +# additionalProperties, so they validate; they are simply not governed by it. +# +# observations The pack carries one validity and one flat parameters array +# per resource, so it cannot express a five-day forecast in the +# one resource the request selected. Its own examples show one +# resource per period. Keeping the selected id was judged worth +# more than splitting into five resources the consumer never +# asked for. Revisit if the packs gain a series field. +# aggregation The pack's parameter entry is parameter/value/unit only. This +# provider reports a minimum AND a maximum for temperature and +# humidity, which are indistinguishable without it. # # Fields that are the same for every day -- the point, the source, the # observation type -- sit once at the top. Only what varies per day repeats. @@ -59,6 +86,17 @@ response: | } }; + /* A warning is a parameter, not a field of its own: the pack has no + advisory property but does have an Alert parameter. Unit "1" is what it + prescribes for a value that has no unit. */ + $alert := function($value) { + $exists($value) ? { + "parameter": "Alert", + "unit": "1", + "value": $value + } + }; + $selected := beckn.message.contract.commitments[0]; { @@ -99,6 +137,11 @@ response: | "type": "Point", "coordinates": [$lon, $lat] }, + "generatedAt": $now(), + "validity": { + "startsAt": $days[0].date, + "endsAt": $days[-1].date + }, "observations": $map($days, function($day) { { "validity": { @@ -111,9 +154,9 @@ response: | $reading("Temperature", "Maximum", "Cel", $day.tmax), $reading("Humidity", "Minimum", "%", $day.rhmin), $reading("Humidity", "Maximum", "%", $day.rhmax), - $reading("WindSpeed", "Average", "m/s", $day.wspd) - ], - "advisory": $day.weather_warning ? $day.weather_warning : $day.cloud_message + $reading("WindSpeed", "Average", "m/s", $day.wspd), + $alert($day.weather_warning ? $day.weather_warning : $day.cloud_message) + ] } }) } diff --git a/pkg/plugin/implementation/mausamgram/mappings_test.go b/pkg/plugin/implementation/mausamgram/mappings_test.go index 1934e59c..0e7bee45 100644 --- a/pkg/plugin/implementation/mausamgram/mappings_test.go +++ b/pkg/plugin/implementation/mausamgram/mappings_test.go @@ -214,12 +214,24 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { t.Errorf("offer.resourceIds = %v, want exactly [%q]", referenced, selectedResourceID) } + // --- the WeatherObservation schema pack, Direct mode --------------------- + // openagrinet:WeatherObservation v0.1 requires all five of these when + // informationMode is Direct. Two of them were missing before the pack was + // read: generatedAt, and a validity for the resource as a whole. attributes, _ := only["resourceAttributes"].(map[string]any) - if attributes["@type"] != "openagrinet:WeatherObservation" { - t.Errorf("@type = %v, want openagrinet:WeatherObservation", attributes["@type"]) + for _, f := range []struct{ key, want string }{ + {"@type", "openagrinet:WeatherObservation"}, + {"informationMode", "Direct"}, + {"observationType", "Forecast"}, + } { + if attributes[f.key] != f.want { + t.Errorf("%s = %v, want %v", f.key, attributes[f.key], f.want) + } } - if attributes["observationType"] != "Forecast" { - t.Errorf("observationType = %v, want Forecast", attributes["observationType"]) + for _, required := range []string{"source", "location", "generatedAt", "validity", "observations"} { + if attributes[required] == nil { + t.Errorf("resourceAttributes carries no %q", required) + } } // The point, the source and the observation type are the same for every day, @@ -233,40 +245,75 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { t.Errorf("coordinates = %v, want [73.7898, 19.9975] in GeoJSON order", coordinates) } - // --- the days, inside the one resource ---------------------------------- + // The resource-level validity spans the whole forecast, first day to last. + validity, _ := attributes["validity"].(map[string]any) + if validity["startsAt"] != "2026-08-26" || validity["endsAt"] != "2026-08-28" { + t.Errorf("validity = %v, want the span of the days the provider answered with", validity) + } + + // --- the days ------------------------------------------------------------ + // observations is NOT a pack field. The pack carries one validity and one + // flat parameters array per resource, so it cannot express a multi-day + // forecast in the resource the request selected. Keeping the selected id + // matters more, so the days go in an extra field -- which validates, because + // the pack sets no additionalProperties, but is not governed by it. observations, _ := attributes["observations"].([]any) if len(observations) != 3 { t.Fatalf("got %d observations, want 3 -- one per day the provider answered with", len(observations)) } first, _ := observations[0].(map[string]any) - validity, _ := first["validity"].(map[string]any) - if validity["startsAt"] != "2026-08-26" { - t.Errorf("first observation starts at %v, want the provider's first forecast date", validity["startsAt"]) - } - if first["advisory"] != "Heavy rainfall warning" { - t.Errorf("advisory = %v, want the provider's warning", first["advisory"]) + dayValidity, _ := first["validity"].(map[string]any) + if dayValidity["startsAt"] != "2026-08-26" { + t.Errorf("first observation starts at %v, want the provider's first forecast date", dayValidity["startsAt"]) } parameters, _ := first["parameters"].([]any) - if len(parameters) != 6 { - t.Errorf("got %d parameters, want 6 for a fully-reported day", len(parameters)) + if len(parameters) != 7 { + t.Errorf("got %d parameters, want 7 for a fully-reported day with a warning", len(parameters)) } assertParameter(t, parameters, "Rainfall", "Total", "mm", 12.4) assertParameter(t, parameters, "Temperature", "Minimum", "Cel", 22.1) assertParameter(t, parameters, "WindSpeed", "Average", "m/s", 4.2) + // A warning is a parameter, not a field of its own: the pack has no advisory + // property but does have an Alert parameter, and unit "1" is what it + // prescribes for a value that has no unit. + assertAlert(t, parameters, "Heavy rainfall warning") + // --- a day the provider reported only partially -------------------------- // Readings it did not take are absent, not present and empty: a consumer - // must be able to tell "no rainfall recorded" from "zero rainfall". + // must be able to tell "no rainfall recorded" from "zero rainfall". A day + // with no warning carries no Alert parameter at all. third, _ := observations[2].(map[string]any) thirdParameters, _ := third["parameters"].([]any) if len(thirdParameters) != 2 { t.Errorf("got %d parameters for a partly-reported day, want only the 2 taken", len(thirdParameters)) } - if third["advisory"] != nil { - t.Errorf("advisory = %v, want it absent when the provider gave none", third["advisory"]) + for _, entry := range thirdParameters { + if p, _ := entry.(map[string]any); p["parameter"] == "Alert" { + t.Error("a day the provider gave no warning for must carry no Alert parameter") + } + } +} + +// assertAlert finds the Alert parameter and checks its value and unit. +func assertAlert(t *testing.T, parameters []any, want string) { + t.Helper() + for _, entry := range parameters { + p, _ := entry.(map[string]any) + if p["parameter"] != "Alert" { + continue + } + if p["value"] != want { + t.Errorf("Alert value = %v, want %q", p["value"], want) + } + if p["unit"] != "1" { + t.Errorf("Alert unit = %v, want \"1\" -- the pack's code for a unitless value", p["unit"]) + } + return } + t.Errorf("no Alert parameter; want one carrying %q", want) } // The shipped file's request half is deliberately empty: this provider takes its From 72c3f094008d3e40e49b6d5b48755a456983abd4 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Tue, 1 Sep 2026 23:11:46 +0530 Subject: [PATCH 034/130] refactor: extract the provider's request in the mapping [#1] The step used to read the coordinates out of the payload itself and send them. That put the choice of which payload fields reach the provider in Go, so a provider wanting one more query parameter meant editing a struct, rebuilding and redeploying. It is the mapping's decision now: whatever the request half produces IS the request. resolvePoint, the point struct and flatPair are gone. What replaced them reads one field: if location.Type == geometryPoint { return nil } verifyGeometry exists only because a mapping cannot refuse. A Polygon's coordinates are nested, so JSONata would build a query parameter that is not a scalar and fail with an error naming neither the geometry nor the reason. The guard turns that into a 400 that says "request carries Polygon; this capability needs a Point". It reads the geometry's type and nothing else, so which fields reach the provider stays entirely configuration. One behaviour change: an empty request half now means an empty request -- no query parameters, no body. Nothing is substituted, because the step no longer holds anything to substitute. TestRunGuardsGeometryWithoutReadingCoordinates pins the boundary. It sends a Point whose coordinates are [1.0, 2.0] and a mapping producing {"station":"NASHIK-1"}, then asserts the query carries station and NOT lat or lon. Reintroducing extraction in Go fails that test. Verified end to end. Adding a date range to the published mapping -- two lines, no rebuild, no restart, no registry change -- reached the provider as ?from=2026-08-30&lat=19.9975&lon=73.7898&to=2026-09-03, and was reverted after. The geometry guard still answers 400 for Polygon, LineString, MultiPoint and an absent location, and 200 for a Point. --- .../weather-observation.select.yaml | 34 +++- .../mausamgram/mappings_test.go | 34 +++- .../implementation/mausamgram/mausamgram.go | 114 +++++------ .../mausamgram/mausamgram_test.go | 186 +++++++++++++++++- 4 files changed, 283 insertions(+), 85 deletions(-) diff --git a/config/mappings/mausamgram/weather-observation.select.yaml b/config/mappings/mausamgram/weather-observation.select.yaml index 8a0d9b7b..1dd2db8b 100644 --- a/config/mappings/mausamgram/weather-observation.select.yaml +++ b/config/mappings/mausamgram/weather-observation.select.yaml @@ -59,11 +59,35 @@ # Fields that are the same for every day -- the point, the source, the # observation type -- sit once at the top. Only what varies per day repeats. -# An empty request half means there is no request document to build. This -# provider takes its parameters in the query string and the step resolved them -# already, so it sends those directly. For a method that takes a body, an empty -# half would mean exactly what it says: no body. -request: "" +# The request half decides what the provider is asked for. Whatever it produces +# IS the request: query parameters for a method with no body, a body for one that +# takes it. +# +# This is where the extraction lives, deliberately. The step reads only the +# geometry's type -- enough to refuse a Polygon with a clear error, because a +# mapping cannot refuse -- and nothing else. So when this provider wants another +# parameter, it is an edit here and nothing else: no Go, no rebuild, live on the +# next cache expiry. +# +# A date range, for instance, is already in the payload and would be two lines: +# +# "from": $ra.validity.startsAt, +# "to": $ra.validity.endsAt +# +# $ra is bound once so the rest reads as plain field access rather than four +# repetitions of the same path. +# +# GeoJSON is [lon, lat] -- longitude first. Reading them the other way round +# gives a point in the wrong hemisphere that is still a valid request, so it +# fails as wrong data rather than as an error. +request: | + ( + $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; + { + "lat": $ra.location.coordinates[1], + "lon": $ra.location.coordinates[0] + } + ) # Keyed by direction, not by the action it produces: a select is answered by an # on_select over the same HTTP round trip, so the callback is this half rather diff --git a/pkg/plugin/implementation/mausamgram/mappings_test.go b/pkg/plugin/implementation/mausamgram/mappings_test.go index 0e7bee45..83b3aa69 100644 --- a/pkg/plugin/implementation/mausamgram/mappings_test.go +++ b/pkg/plugin/implementation/mausamgram/mappings_test.go @@ -316,10 +316,10 @@ func assertAlert(t *testing.T, parameters []any, want string) { t.Errorf("no Alert parameter; want one carrying %q", want) } -// The shipped file's request half is deliberately empty: this provider takes its -// parameters in the query string. That has to produce nothing rather than an -// empty document, so the step builds the query itself. -func TestShippedMappingsProduceNoRequestDocument(t *testing.T) { +// The shipped file's request half extracts what the provider is asked for. That +// is the point of it living in the mapping: when this provider wants another +// parameter -- a date range, say -- it is an edit here and nothing else. +func TestShippedMappingsExtractTheQueryFromThePayload(t *testing.T) { mappings := serveMappings(t) defer mappings.Close() @@ -329,15 +329,29 @@ func TestShippedMappingsProduceNoRequestDocument(t *testing.T) { } defer closeMapper() - ref := mappings.URL + "/" + shippedMapping - input := map[string]any{"beckn": map[string]any{"context": map[string]any{"action": "select"}}} + var beckn any + if err := json.Unmarshal([]byte(selectRequest), &beckn); err != nil { + t.Fatalf("failed to decode the request: %v", err) + } - got, err := mapper.Transform(context.Background(), ref, definition.DirectionRequest, input) + got, err := mapper.Transform(context.Background(), mappings.URL+"/"+shippedMapping, + definition.DirectionRequest, map[string]any{"beckn": beckn}) if err != nil { - t.Errorf("the empty request half must not be an error, got %v", err) + t.Fatalf("the request half returned an unexpected error: %v", err) + } + + var query map[string]any + if err := json.Unmarshal(got, &query); err != nil { + t.Fatalf("the request half produced something that is not an object: %v", err) + } + + // GeoJSON is [lon, lat]. Reading them the other way round yields a point in + // the wrong hemisphere that is still a valid request. + if query["lat"] != 19.9975 { + t.Errorf("lat = %v, want 19.9975 taken from coordinates[1]", query["lat"]) } - if len(got) != 0 { - t.Errorf("the request half produced %q, want nothing", got) + if query["lon"] != 73.7898 { + t.Errorf("lon = %v, want 73.7898 taken from coordinates[0]", query["lon"]) } } diff --git a/pkg/plugin/implementation/mausamgram/mausamgram.go b/pkg/plugin/implementation/mausamgram/mausamgram.go index 204839a1..019ece7d 100644 --- a/pkg/plugin/implementation/mausamgram/mausamgram.go +++ b/pkg/plugin/implementation/mausamgram/mausamgram.go @@ -195,8 +195,7 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { plan.BindingKey, action, strings.Join(servedActions(plan), ", "))) } - local, err := resolvePoint(ctx.Body) - if err != nil { + if err := verifyGeometry(ctx.Body); err != nil { return err } @@ -205,7 +204,7 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { return err } - upstreamRequest, err := s.buildRequest(ctx, call, beckn, local) + upstreamRequest, err := s.buildRequest(ctx, call, beckn) if err != nil { return err } @@ -223,10 +222,7 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { // The same mapping reference as the request, other half: one file carries // both directions for this action. // - // The mapping is handed what each party sent and nothing else. The values - // resolved above are not passed in: this step holds them and used them to - // make the call, so handing them to the mapping would be a second name for - // the same data. + // The mapping is handed what each party sent and nothing else. becknResponse, err := s.mapper.Transform(ctx, call.Mappings, definition.DirectionResponse, map[string]any{ "beckn": beckn, "response": answer, @@ -261,69 +257,53 @@ func servedActions(plan *model.ProviderRecord) []string { // buildRequest produces what the provider is sent. // -// The mapping is handed the inbound payload and nothing else, and it produces a -// document or it produces nothing. Nothing is the ordinary case here: this -// provider takes its parameters in the query string, this step resolved them -// already, and putting them through a fetch and a compile to arrive at the same -// two fields would buy nothing. -// -// What nothing means depends on the method, and both readings are deliberate: +// Whatever the mapping produces IS the request: a body for a method that takes +// one, query parameters for a method that does not. Nothing is substituted when +// it produces nothing, so an empty request half means an empty request. // -// - a method with no body -- the resolved values ARE the parameters. This step -// knows this provider, so it does not need the mapping's help to call it. -// - a method with a body -- there is no body. An empty mapping means an empty -// request, not the resolved values dressed up as one; a body is the -// mapping's business and it supplied none. -// -// A half with no transform and a transform that matched nothing are treated -// alike here, deliberately: for the request leg there is no document either way, -// and inventing one would send the provider something nobody asked for. -func (s *Step) buildRequest(ctx context.Context, call model.ActionPlan, beckn any, local point) ([]byte, error) { +// This step used to extract a point from the payload and fall back to sending +// that. It meant the choice of which payload fields reach the provider lived in +// Go, so adding a parameter -- a date range, say -- was a rebuild. Now it is a +// mapping edit and nothing else. +func (s *Step) buildRequest(ctx context.Context, call model.ActionPlan, beckn any) ([]byte, error) { mapped, err := s.mapper.Transform(ctx, call.Mappings, definition.DirectionRequest, map[string]any{ "beckn": beckn, }) if err != nil { return nil, err } - if len(mapped) > 0 { - return mapped, nil - } - - if hasBody(call.Method) { - log.Debugf(ctx, "mausamgram: the request half of %s produced nothing; sending no body", call.Mappings) - return nil, nil - } - - log.Debugf(ctx, "mausamgram: the request half of %s produced nothing; sending the resolved point", call.Mappings) - parameters, err := json.Marshal(local) - if err != nil { - return nil, fmt.Errorf("mausamgram: could not encode the resolved point: %w", err) + if len(mapped) == 0 { + log.Debugf(ctx, "mausamgram: the request half of %s produced nothing; sending an empty request", call.Mappings) } - return parameters, nil + return mapped, nil } -// point is what this provider needs beyond the Beckn payload: a latitude and a -// longitude, as separate numbers. -type point struct { - Lat float64 `json:"lat"` - Lon float64 `json:"lon"` -} +// geometryPoint is the GeoJSON type this capability can serve. IMD takes one +// latitude and one longitude, so an area or a set of points has no single answer +// -- and picking a centroid would invent a location the caller did not ask about. +const geometryPoint = "Point" -// resolvePoint reads the coordinates the request is asking about. +// verifyGeometry refuses a request whose location this capability cannot serve. // -// This is the prerequisite step: the work a provider needs done before it can be -// called, which in the old per-provider services was tangled together with -// building the response. Here it produces values and nothing else, and the -// mapping decides what they are called upstream. -func resolvePoint(body []byte) (point, error) { +// It reads the geometry's type and NOTHING ELSE. Coordinates, dates and every +// other field are the mapping's to extract, which is what keeps "the provider +// wants another parameter" a mapping edit rather than a rebuild. This exists +// only because a mapping cannot refuse: JSONata would turn a Polygon's nested +// coordinates into a query parameter that is not a scalar, and surface as an +// error naming neither the geometry nor the reason. +// +// A location that is absent is refused too, and separately: that is a different +// fact from a geometry this provider does not serve, and a caller can act on the +// difference. +func verifyGeometry(body []byte) error { var payload struct { Message struct { Contract struct { Commitments []struct { Resources []struct { ResourceAttributes struct { - Location struct { - Coordinates []float64 `json:"coordinates"` + Location *struct { + Type string `json:"type"` } `json:"location"` } `json:"resourceAttributes"` } `json:"resources"` @@ -332,24 +312,34 @@ func resolvePoint(body []byte) (point, error) { } `json:"message"` } if err := json.Unmarshal(body, &payload); err != nil { - return point{}, fmt.Errorf("mausamgram: request could not be read: %w", err) + return model.NewBadReqErr("", + fmt.Errorf("mausamgram: request could not be read: %w", err)) } + var unsupported string for _, commitment := range payload.Message.Contract.Commitments { for _, resource := range commitment.Resources { - coordinates := resource.ResourceAttributes.Location.Coordinates - if len(coordinates) < 2 { + location := resource.ResourceAttributes.Location + if location == nil { continue } - // GeoJSON order: longitude first, then latitude. Reading these the - // other way round yields a point in the wrong hemisphere that is - // still a valid request, so it fails as wrong data rather than as an - // error. - return point{Lon: coordinates[0], Lat: coordinates[1]}, nil + if location.Type == geometryPoint { + return nil + } + unsupported = location.Type + if unsupported == "" { + unsupported = "a geometry with no type" + } } } - return point{}, model.NewBadReqErr("", - errors.New("mausamgram: request carries no location coordinates")) + + if unsupported != "" { + return model.NewBadReqErr("", fmt.Errorf( + "mausamgram: request carries %s; this capability needs a %s", + unsupported, geometryPoint)) + } + return model.NewBadReqErr("", + errors.New("mausamgram: request carries no location")) } // extractAction reads the Beckn action a request is for. diff --git a/pkg/plugin/implementation/mausamgram/mausamgram_test.go b/pkg/plugin/implementation/mausamgram/mausamgram_test.go index 06b699ad..340e1ae1 100644 --- a/pkg/plugin/implementation/mausamgram/mausamgram_test.go +++ b/pkg/plugin/implementation/mausamgram/mausamgram_test.go @@ -328,11 +328,14 @@ func TestRunSendsTheMappedBodyForAMethodThatTakesOne(t *testing.T) { } } -// A provider taking query parameters needs no request document built. The -// mapping produces nothing, and the values this step resolved become the -// parameters -- the step knows this provider, so it does not need the mapping's -// help to call it. -func TestRunSendsResolvedValuesWhenTheMappingProducesNothing(t *testing.T) { +// The mapping decides what the provider is asked, so what it produces IS the +// request. Nothing is substituted when it produces nothing: an empty request +// half means an empty request, on a method with a body or without one. +// +// This step used to extract a point itself and fall back to sending it. That put +// the choice of which payload fields reach the provider in Go, so adding a +// parameter meant a rebuild. It is the mapping's now. +func TestRunSendsWhatTheMappingProducedAsQueryParameters(t *testing.T) { t.Parallel() var gotQuery string @@ -342,18 +345,43 @@ func TestRunSendsResolvedValuesWhenTheMappingProducesNothing(t *testing.T) { })) defer upstream.Close() - mapper := &stubMapper{requestResult: nil, responseResult: []byte(`{}`)} + // Four fields, none of them known to this step: whatever the mapping named. + mapper := &stubMapper{ + requestResult: []byte(`{"lat":19.9975,"lon":73.7898,"from":"2026-08-30","to":"2026-09-03"}`), + responseResult: []byte(`{}`), + } if _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), selectBody); err != nil { t.Fatalf("Run() returned an unexpected error: %v", err) } - for _, want := range []string{"lat=19.9975", "lon=73.7898"} { + for _, want := range []string{"lat=19.9975", "lon=73.7898", "from=2026-08-30", "to=2026-09-03"} { if !strings.Contains(gotQuery, want) { - t.Errorf("query %q is missing %q -- the resolved point was not sent", gotQuery, want) + t.Errorf("query %q is missing %q", gotQuery, want) } } } +// An empty request half on a method with no body means no query parameters. The +// step has nothing of its own to send in their place. +func TestRunSendsNoQueryWhenTheMappingProducesNothing(t *testing.T) { + t.Parallel() + + var gotQuery string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotQuery = r.URL.RawQuery + fmt.Fprint(w, `{}`) + })) + defer upstream.Close() + + mapper := &stubMapper{requestResult: nil, responseResult: []byte(`{}`)} + if _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), selectBody); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + if gotQuery != "" { + t.Errorf("query = %q, want none -- nothing is substituted for an empty mapping", gotQuery) + } +} + // A method that takes a body, and a mapping that produces nothing, means no // body -- not the resolved values dressed up as one. Query parameters are the // step's own doing; a body is the mapping's, and there is nothing to send. @@ -402,6 +430,148 @@ func TestRunRefusesWhenTheResponseMappingProducesNothing(t *testing.T) { } } +// --- the geometry the request carries --------------------------------------- + +// selectWithLocation renders a select payload whose resource carries the given +// GeoJSON geometry verbatim, or none at all when geometry is empty. +func selectWithLocation(t *testing.T, geometry string) string { + t.Helper() + location := "" + if geometry != "" { + location = `"location": ` + geometry + `,` + } + return `{ + "context": { "version": "2.0.0", "action": "select", "transactionId": "txn-1" }, + "message": { "contract": { "commitments": [ { + "resources": [ { "id": "res:x", "resourceAttributes": { + ` + location + ` + "@type": "openagrinet:WeatherObservation" + } } ], + "offer": { "id": "offer:x", "provider": { "id": "mausamgram" } } + } ] } } +}` +} + +// This capability needs one point. A request carrying any other geometry is the +// caller sending something this provider cannot serve, so it has to come back as +// a bad request naming what was sent -- not as a 500, which says the fault is +// ours and tells the caller nothing. +// +// Before this was fixed, a Polygon reached json.Unmarshal as a three-deep array +// where a flat pair was expected, failed there, and surfaced as +// NET_INTERNAL_ERROR. +func TestRunRefusesAGeometryThatIsNotAPoint(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + geometry string + }{ + {"a polygon", `{"type":"Polygon","coordinates":[[[73.0,19.0],[74.0,19.0],[74.0,20.0],[73.0,19.0]]]}`}, + {"a line string", `{"type":"LineString","coordinates":[[73.0,19.0],[74.0,20.0]]}`}, + {"several points", `{"type":"MultiPoint","coordinates":[[73.7898,19.9975],[75.0,21.0]]}`}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Error("the provider must not be called for a geometry this capability cannot serve") + })) + defer upstream.Close() + + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), + selectWithLocation(t, tc.geometry)) + if err == nil { + t.Fatal("expected an unsupported geometry to be refused") + } + + var coded *model.CodedErr + if !errors.As(err, &coded) { + t.Fatalf("error is %T, want a coded error so it NACKs as a bad request: %v", err, err) + } + if coded.HTTPStatus() != http.StatusBadRequest { + t.Errorf("status = %d, want 400 -- the payload is the caller's, not our fault", coded.HTTPStatus()) + } + // Naming the geometry that arrived is what makes this actionable. + if !strings.Contains(err.Error(), "Point") { + t.Errorf("error %q should say a Point is what this capability needs", err) + } + }) + } +} + +// A Point passes the guard and reaches the mapping. +func TestRunAcceptsAPoint(t *testing.T) { + t.Parallel() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{}`) + })) + defer upstream.Close() + + mapper := &stubMapper{requestResult: []byte(`{"lat":19.9975}`), responseResult: []byte(`{}`)} + if _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), + selectWithLocation(t, `{"type":"Point","coordinates":[73.7898,19.9975]}`)); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + if mapper.requestInput == nil { + t.Error("the request mapping was never called for a Point") + } +} + +// The guard checks the geometry and nothing else. It does not read coordinates, +// so which payload fields reach the provider stays entirely the mapping's -- +// adding a parameter is a mapping edit, never a rebuild. +func TestRunGuardsGeometryWithoutReadingCoordinates(t *testing.T) { + t.Parallel() + + var gotQuery string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotQuery = r.URL.RawQuery + fmt.Fprint(w, `{}`) + })) + defer upstream.Close() + + // A Point whose coordinates the step never looks at, and a mapping that + // names something else entirely. + mapper := &stubMapper{requestResult: []byte(`{"station":"NASHIK-1"}`), responseResult: []byte(`{}`)} + if _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), + selectWithLocation(t, `{"type":"Point","coordinates":[1.0,2.0]}`)); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + if !strings.Contains(gotQuery, "station=NASHIK-1") { + t.Errorf("query = %q, want the mapping's own field", gotQuery) + } + if strings.Contains(gotQuery, "lat=") || strings.Contains(gotQuery, "lon=") { + t.Errorf("query = %q -- the step is still injecting coordinates of its own", gotQuery) + } +} + +// An absent location was already a bad request, and stays one. Kept alongside the +// geometry cases so the two failures are visibly the same kind of thing. +func TestRunRefusesAMissingLocation(t *testing.T) { + t.Parallel() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Error("the provider must not be called without a location") + })) + defer upstream.Close() + + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), + selectWithLocation(t, "")) + if err == nil { + t.Fatal("expected a request with no location to be refused") + } + var coded *model.CodedErr + if !errors.As(err, &coded) || coded.HTTPStatus() != http.StatusBadRequest { + t.Errorf("expected a 400, got %v", err) + } +} + // A mapping that failed is a failure, and must not be papered over by sending // the resolved values instead. func TestRunDoesNotSubstituteForARealMappingFailure(t *testing.T) { From 7da3fc43c3bd73c567010b61b63fd37f1303c80a Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 01:03:31 +0530 Subject: [PATCH 035/130] feat: let a mapping declare what it requires of a payload [#1] A mapping decides what a provider is asked for, but it could not decide that a request cannot be served at all. That judgement stayed in Go, so a capability with its own rule needed its own build -- and the rule sat in a different file from the extraction it guarded. Mappings now carry an optional block, checked before either half runs: required: - check: | ( $ra := beckn.message...resourceAttributes; $exists($ra.location) and $ra.location.type = "Point" ) message: "this capability needs a Point location" Named check/message so each field says what it is. "test" says nothing about which way the predicate must answer, and "otherwise" reads like an alternative value rather than an error. Verify is a method of its own rather than folded into Transform. Transform returns early for a half with no transform, so a mapping with an empty request half would have skipped its own preconditions -- a trap that cannot arise when asking is a separate call. Four ways this refuses rather than passing quietly: a false predicate returns its message as a 400; a predicate answering anything but true or false is a mapping fault, NOT permission, because a typo yielding nothing would otherwise wave through every request the check existed to stop; a predicate with no message is a fault, since refusing without saying why is what this avoids; and a predicate that will not compile reports itself without taking the halves down. verifyGeometry and geometryPoint are gone from the provider step, which now asks and propagates. The consequence is worth stating: nothing in the adapter enforces a payload rule any more. Whatever a mapping does not require, it accepts. That is the point, and it is why the responsibility sits in the published file. The shipped mapping also stops naming five forecast days. The provider answers fcstday1..fcstdayN and N is whatever the forecast ran to, so five truncated a ten-day answer. Two things that testing caught and reasoning would not have: the keys sort lexically as fcstday1, fcstday10, fcstday2, so the days are sorted on the numeric suffix; and JSONata collapses a one-element sequence to a bare value, so the list is forced to an array or a single-day forecast answers with an object where every other N answers with a list. That second bug was present in the hardcoded version too, masked because the mock always sent three. Verified end to end. A Point is served; a Polygon, a LineString, a MultiPoint and an absent location are each refused with the mapping's own message. With the provider sending one, three and five days the answer carries one, three and five observations, and the resource validity window follows. --- .../weather-observation.select.yaml | 54 ++++- pkg/plugin/definition/mapper.go | 12 + .../implementation/jsonmapper/README.md | 50 ++++- .../implementation/jsonmapper/jsonmapper.go | 147 +++++++++++- .../jsonmapper/jsonmapper_test.go | 211 ++++++++++++++++++ .../mausamgram/cmd/plugin_test.go | 2 + .../mausamgram/dispatch_test.go | 2 + .../mausamgram/mappings_test.go | 127 +++++++++++ .../implementation/mausamgram/mausamgram.go | 74 +----- .../mausamgram/mausamgram_test.go | 140 +++++------- 10 files changed, 647 insertions(+), 172 deletions(-) diff --git a/config/mappings/mausamgram/weather-observation.select.yaml b/config/mappings/mausamgram/weather-observation.select.yaml index 1dd2db8b..6170a2c6 100644 --- a/config/mappings/mausamgram/weather-observation.select.yaml +++ b/config/mappings/mausamgram/weather-observation.select.yaml @@ -59,6 +59,36 @@ # Fields that are the same for every day -- the point, the source, the # observation type -- sit once at the top. Only what varies per day repeats. +# What this capability requires of a payload, checked before either half runs. +# A predicate that is false refuses the request with the message beside it, so +# the caller is told what is wrong with their payload rather than that an +# expression somewhere returned false. +# +# This rule used to be Go: the step read the geometry and required a Point, which +# meant a capability with a different rule needed a different build. It is here +# now, beside the extraction it guards. +# +# NOTE the consequence: nothing in the adapter enforces a geometry any more. A +# mapping that declares no preconditions accepts whatever arrives and hands it to +# the request half, which is exactly the configurability that was asked for -- +# and exactly why the responsibility sits in this file. +# +# One check, because there is one thing to say. $exists guards the type test, so +# a request carrying no location and a request carrying a Polygon both land here +# and both learn what this capability needs -- splitting them would be two +# entries repeating the same sentence. +# +# Each check is its own expression and binds $ra for itself; there is no shared +# scope with the halves below. Where several checks say genuinely different +# things, they are separate entries and the first failure is the one reported. +required: + - check: | + ( + $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; + $exists($ra.location) and $ra.location.type = "Point" + ) + message: "this capability needs a Point location; the provider forecasts one point at a time" + # The request half decides what the provider is asked for. Whatever it produces # IS the request: query parameters for a method with no body, a body for one that # takes it. @@ -96,10 +126,19 @@ response: | ( $lat := response.location.lat; $lon := response.location.lon; - $days := [ - response.fcstday1, response.fcstday2, response.fcstday3, - response.fcstday4, response.fcstday5 - ]; + /* However many days the provider sent. It answers fcstday1..fcstdayN and N + is whatever the forecast ran to, so naming five would truncate a ten-day + answer and mis-handle a one-day one. + + Sorted on the numeric suffix, not the key: the keys sort lexically as + fcstday1, fcstday10, fcstday2, and a ten-day forecast delivered in that + order would be wrong in a way nothing downstream could detect. */ + $days := $each(response, function($v, $k) { + $contains($k, "fcstday") ? { + "n": $number($substringAfter($k, "fcstday")), + "day": $v + } + })^(n).day; $reading := function($name, $aggregation, $unit, $value) { $exists($value) ? { @@ -166,7 +205,10 @@ response: | "startsAt": $days[0].date, "endsAt": $days[-1].date }, - "observations": $map($days, function($day) { + /* Wrapped: JSONata collapses a one-element sequence to a + bare value, so a single-day forecast would answer with an + object where every other N answers with a list. */ + "observations": [$map($days, function($day) { { "validity": { "startsAt": $day.date, @@ -182,7 +224,7 @@ response: | $alert($day.weather_warning ? $day.weather_warning : $day.cloud_message) ] } - }) + })] } } ] diff --git a/pkg/plugin/definition/mapper.go b/pkg/plugin/definition/mapper.go index 369966fa..2ff68f83 100644 --- a/pkg/plugin/definition/mapper.go +++ b/pkg/plugin/definition/mapper.go @@ -41,6 +41,18 @@ type Mapper interface { // What nothing means belongs to the caller: on the request leg it means there // is no document to send. Transform(ctx context.Context, mappingRef string, direction Direction, input any) ([]byte, error) + + // Verify checks the preconditions the mapping at mappingRef declares, and + // returns an error carrying the mapping's own explanation when one fails. + // + // It exists because a mapping otherwise cannot refuse. Without it, every + // judgement about whether a payload can be served at all lives in Go, so a + // provider with its own rule needs its own build -- and the rule and the + // extraction it guards end up in different places. + // + // A mapping declaring no preconditions imposes none. That is what lets the + // facility be adopted per provider rather than all at once. + Verify(ctx context.Context, mappingRef string, input any) error } // MapperProvider initializes a new Mapper. diff --git a/pkg/plugin/implementation/jsonmapper/README.md b/pkg/plugin/implementation/jsonmapper/README.md index c1fa59b4..0b2ac9c7 100644 --- a/pkg/plugin/implementation/jsonmapper/README.md +++ b/pkg/plugin/implementation/jsonmapper/README.md @@ -25,10 +25,17 @@ generic. ## Mapping files -One file per binding-action, carrying **both directions**: +One file per binding-action, carrying **both directions** and what the capability +requires of a payload: ```yaml # mappings/mausamgram/weather-observation.select.yaml +required: + - check: | + ( $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; + $ra.location.type = "Point" ) + message: "this capability needs a Point location" + request: | { "lat": beckn.message.contract.commitments[0].resources[0].resourceAttributes.location.coordinates[1] @@ -41,6 +48,8 @@ response: | } ``` +The three keys are checked and run in the order they appear. + **One file rather than two because both legs of one upstream call are one unit of configuration.** They are published, reviewed and retired together, and a reference to one is a reference to the other. It also means the response leg is @@ -56,6 +65,45 @@ A broken half takes down only itself: a typo in the response mapping is no reaso to stop making the call, and finding out on the way back beats finding out before the call was made. +### `required` + +A list of preconditions, each a JSONata predicate and the message to refuse with. +Named `check`/`message` so each field says what it is: `check` is what has to +hold, `message` is what the caller is told when it does not. A neutral name like +`test` says nothing about which way the predicate must answer, and `otherwise` +reads like an alternative value rather than an error. + +**It exists because a mapping otherwise cannot refuse.** Without it, every +judgement about whether a payload can be served at all lives in Go — so a +provider with its own rule needs its own build, and the rule ends up in a +different place from the extraction it guards. + +`Verify` evaluates them in order and **the first failure is the one reported**. +Reporting the last, or all of them, buries the thing to fix. + +Four ways this refuses rather than passing quietly: + +- a predicate that is false → the caller gets its `message`, as a `400` +- a predicate answering anything but `true`/`false` → a mapping fault, **not** + permission. A typo yielding nothing would otherwise wave through every request + the check existed to stop. +- a predicate with no `message` → a mapping fault. Refusing without saying why + is the failure this key exists to avoid. +- a predicate that will not compile → its own error, and it does not take the + halves down with it + +Each predicate is a separate expression, so each binds its own variables — there +is no shared scope with the halves. + +**A mapping declaring no `required` imposes nothing.** That is deliberate: it lets +one provider adopt preconditions while others have not, so a second provider +arriving with its own rules does not force every existing mapping to be rewritten. + +The consequence is worth stating plainly: **nothing in the adapter enforces a +payload rule any more.** Whatever a mapping does not require, it accepts. That is +the point — the rule is configuration — and it is why the responsibility sits in +the published file. + Which action a file serves is settled by the registry entry pointing at it, so nothing inside names it and the filename carries no meaning to this plugin. (The registry contract does require the filename's action segment to match the action diff --git a/pkg/plugin/implementation/jsonmapper/jsonmapper.go b/pkg/plugin/implementation/jsonmapper/jsonmapper.go index a4bf6af8..3088c73f 100644 --- a/pkg/plugin/implementation/jsonmapper/jsonmapper.go +++ b/pkg/plugin/implementation/jsonmapper/jsonmapper.go @@ -54,8 +54,27 @@ const codeAdaptationFailed = "SCH_SCHEMA_ADAPTATION_FAILED" // there is no transform for that direction, and what that means belongs to the // caller. type mappingFile struct { - Request string `yaml:"request"` - Response string `yaml:"response"` + // Required are the preconditions this binding-action imposes on a payload, + // verified before either half runs. Absent means none. + Required []requirement `yaml:"required"` + Request string `yaml:"request"` + Response string `yaml:"response"` +} + +// requirement is one precondition: what must hold, and what to tell the caller +// when it does not. +// +// Named check/message so each field says what it is: check is the predicate +// that has to hold, message is what the caller is told when it does not. A +// neutral name like "test" says nothing about which way the predicate must +// answer, and "otherwise" reads like an alternative value rather than an error. +// +// The message is required. A precondition that refuses without saying why is +// the failure this whole facility exists to avoid -- the caller is left with +// "rejected" and no way to act on it. +type requirement struct { + Check string `yaml:"check"` + Message string `yaml:"message"` } // Config holds configuration parameters for the mapper. @@ -101,12 +120,25 @@ type cacheEntry struct { // and compiled as a whole, so both are ready after the first request for // either. directions map[definition.Direction]*compiledMapping + // checks are the file's preconditions, in the order it declared them. + checks []*compiledRequirement // err is a failure that applies to the whole file -- it could not be // fetched, or not parsed -- as opposed to one action failing to compile. err error expiresAt time.Time } +// compiledRequirement is one precondition, or the failure that stopped it +// compiling. Held per requirement for the same reason a half is: a broken +// precondition is the mapping's fault and should be reported as one, without +// taking the halves down with it. +type compiledRequirement struct { + expression jsonata.Expression + evaluating *sync.Mutex + message string + err error +} + // compiledMapping is one half of a mapping, or the failure that stopped it // compiling. Failures are held per half deliberately: a typo in the response // mapping is no reason for the request half to stop working, and finding out on @@ -213,6 +245,68 @@ func (m *Mapper) Transform(ctx context.Context, mappingRef string, direction def return m.evaluate(ctx, mapping, mappingRef, direction, input) } +// Verify checks the preconditions the mapping declares, in the order declared. +// +// The first failure is the one reported. Reporting the last, or all of them, +// buries the thing to fix. +func (m *Mapper) Verify(ctx context.Context, mappingRef string, input any) error { + entry, err := m.compiled(ctx, mappingRef) + if err != nil { + return err + } + if len(entry.checks) == 0 { + return nil + } + + document, err := marshalInput(input) + if err != nil { + return fmt.Errorf("jsonmapper: mapping %q: %w", mappingRef, err) + } + + for _, precondition := range entry.checks { + if precondition.err != nil { + return precondition.err + } + + // See compiledMapping: Evaluate mutates the expression it is called on. + precondition.evaluating.Lock() + result, evalErr := precondition.expression.Evaluate(document, nil) + precondition.evaluating.Unlock() + if evalErr != nil { + log.Errorf(ctx, evalErr, "JSON mapping %s precondition failed to evaluate: %v", mappingRef, evalErr) + return model.NewBadReqErr(codeAdaptationFailed, fmt.Errorf( + "mapping %q precondition could not be applied: %w", mappingRef, evalErr)) + } + + holds, answered := asBool(result) + if !answered { + // Not read as permission. A typo yielding nothing would otherwise + // wave every request through the check meant to stop it. + return fmt.Errorf("jsonmapper: mapping %q precondition answered %q, want true or false", + mappingRef, result) + } + if !holds { + // The mapping's own words: the caller is told what is wrong with + // their payload, not that an expression somewhere returned false. + return model.NewBadReqErr("", errors.New(precondition.message)) + } + } + return nil +} + +// asBool reads a precondition's answer. JSONata yields JSON, so a predicate +// answers with the two literals and nothing else counts. +func asBool(result []byte) (bool, bool) { + switch strings.TrimSpace(string(result)) { + case "true": + return true, true + case "false": + return false, true + default: + return false, false + } +} + // compiled returns the compiled mapping for a reference, fetching and compiling // it on first use. A failure is cached too, for a shorter time. func (m *Mapper) compiled(ctx context.Context, mappingRef string) (cacheEntry, error) { @@ -220,8 +314,8 @@ func (m *Mapper) compiled(ctx context.Context, mappingRef string) (cacheEntry, e return entry, entry.err } - directions, err := m.fetchAndCompile(ctx, mappingRef) - return m.remember(mappingRef, directions, err), err + directions, checks, err := m.fetchAndCompile(ctx, mappingRef) + return m.remember(mappingRef, directions, checks, err), err } // cached returns a live cache entry, if there is one. @@ -239,13 +333,15 @@ func (m *Mapper) cached(mappingRef string) (cacheEntry, bool) { // remember caches a compiled mapping, or the failure that stopped it compiling. // A failure gets the shorter TTL: it should stop hammering a broken reference // without outlasting the fix. -func (m *Mapper) remember(mappingRef string, directions map[definition.Direction]*compiledMapping, err error) cacheEntry { +func (m *Mapper) remember(mappingRef string, directions map[definition.Direction]*compiledMapping, + checks []*compiledRequirement, err error) cacheEntry { ttl := m.config.CacheTTL if err != nil { ttl = m.config.NegativeTTL } entry := cacheEntry{ directions: directions, + checks: checks, err: err, expiresAt: time.Now().Add(ttl), } @@ -276,14 +372,22 @@ func (m *Mapper) cachedCount() int { } // fetchAndCompile retrieves a mapping and turns it into a runnable expression. -func (m *Mapper) fetchAndCompile(ctx context.Context, mappingRef string) (map[definition.Direction]*compiledMapping, error) { +func (m *Mapper) fetchAndCompile(ctx context.Context, mappingRef string) ( + map[definition.Direction]*compiledMapping, []*compiledRequirement, error) { body, err := m.fetch(ctx, mappingRef) if err != nil { - return nil, err + return nil, nil, err } file, err := parseMapping(body) if err != nil { - return nil, fmt.Errorf("jsonmapper: mapping %q: %w", mappingRef, err) + return nil, nil, fmt.Errorf("jsonmapper: mapping %q: %w", mappingRef, err) + } + + // Preconditions compile with the halves, so one round trip leaves the whole + // file ready and a precondition costs no extra fetch. + checks := make([]*compiledRequirement, 0, len(file.Required)) + for _, declared := range file.Required { + checks = append(checks, m.compileRequirement(ctx, mappingRef, declared)) } // Both halves are compiled now rather than on first use, so one fetch leaves @@ -292,8 +396,31 @@ func (m *Mapper) fetchAndCompile(ctx context.Context, mappingRef string) (map[de directions := make(map[definition.Direction]*compiledMapping, 2) directions[definition.DirectionRequest] = m.compileMapping(ctx, mappingRef, definition.DirectionRequest, file.Request) directions[definition.DirectionResponse] = m.compileMapping(ctx, mappingRef, definition.DirectionResponse, file.Response) - log.Debugf(ctx, "JSON mapper compiled mapping: %s", mappingRef) - return directions, nil + log.Debugf(ctx, "JSON mapper compiled mapping: %s (%d precondition(s))", mappingRef, len(checks)) + return directions, checks, nil +} + +// compileRequirement compiles one precondition, keeping any failure local to it. +func (m *Mapper) compileRequirement(ctx context.Context, mappingRef string, declared requirement) *compiledRequirement { + if strings.TrimSpace(declared.Message) == "" { + return &compiledRequirement{err: fmt.Errorf( + "jsonmapper: mapping %q declares a precondition with no message", mappingRef)} + } + if strings.TrimSpace(declared.Check) == "" { + return &compiledRequirement{err: fmt.Errorf( + "jsonmapper: mapping %q declares a precondition with no check", mappingRef)} + } + expression, err := m.instance.Compile(declared.Check, false) + if err != nil { + log.Errorf(ctx, err, "JSON mapper could not compile a precondition of %s: %v", mappingRef, err) + return &compiledRequirement{err: fmt.Errorf( + "jsonmapper: mapping %q precondition %q failed to compile: %w", mappingRef, declared.Check, err)} + } + return &compiledRequirement{ + expression: expression, + evaluating: &sync.Mutex{}, + message: declared.Message, + } } // compileMapping compiles one half, keeping any failure local to it. diff --git a/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go b/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go index f9a9f1f3..04aebd82 100644 --- a/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go +++ b/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go @@ -3,6 +3,7 @@ package jsonmapper import ( "context" "encoding/json" + "errors" "fmt" "net/http" "net/http/httptest" @@ -11,6 +12,7 @@ import ( "testing" "time" + "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" ) @@ -237,6 +239,215 @@ response: | } } +// --- preconditions ---------------------------------------------------------- +// +// A mapping decides what a provider is asked for. It has to be able to decide +// that a request cannot be served at all, or that judgement stays in Go and +// every provider with its own rule needs its own build. +// +// required is a list of predicates over the same payload the request half reads. +// A predicate that is false refuses the request, carrying the message the +// mapping supplied -- so the caller gets a sentence about their payload rather +// than a mapping error about ours. + +const withChecks = `required: + - check: beckn.message.location.type = "Point" + message: "this capability needs a Point location" + - check: $exists(beckn.message.validity) + message: "this capability needs a validity window" + +request: | + { "txn": beckn.context.transactionId } + +response: | + { "txn": beckn.context.transactionId } +` + +// verifyInput is a payload that satisfies withChecks. The beckn wrapper is +// what the caller passes, so a precondition reads the payload by the same name +// the request half does. +func verifyInput() map[string]any { + return map[string]any{ + "beckn": map[string]any{ + "context": map[string]any{"transactionId": "txn-123"}, + "message": map[string]any{ + "location": map[string]any{"type": "Point", "coordinates": []any{73.7898, 19.9975}}, + "validity": map[string]any{"startsAt": "2026-09-01"}, + }, + }, + } +} + +// becknOf reaches into the wrapper, so a test can spoil one field. +func becknOf(input map[string]any) map[string]any { + return input["beckn"].(map[string]any) +} + +func TestVerifyPassesWhenEveryPredicateHolds(t *testing.T) { + t.Parallel() + + srv := newMappingServer(t, withChecks, nil) + defer srv.Close() + + if err := newTestMapper(t).Verify(context.Background(), ref(srv.URL), verifyInput()); err != nil { + t.Errorf("Verify() refused a payload that satisfies every predicate: %v", err) + } +} + +// The message belongs to the mapping, so the caller is told what is wrong with +// their payload rather than that an expression failed. +func TestVerifyRefusesWithTheMappingsOwnMessage(t *testing.T) { + t.Parallel() + + srv := newMappingServer(t, withChecks, nil) + defer srv.Close() + + input := verifyInput() + becknOf(input)["message"].(map[string]any)["location"] = map[string]any{"type": "Polygon"} + + err := newTestMapper(t).Verify(context.Background(), ref(srv.URL), input) + if err == nil { + t.Fatal("expected a payload failing a predicate to be refused") + } + if !strings.Contains(err.Error(), "needs a Point location") { + t.Errorf("error %q should carry the mapping's own message", err) + } + + // A bad request: the payload is the caller's, so this must not read as a + // fault of this adapter. + var coded *model.CodedErr + if !errors.As(err, &coded) || coded.HTTPStatus() != http.StatusBadRequest { + t.Errorf("error is %T, want a 400 so the caller is not blamed for our fault: %v", err, err) + } +} + +// Predicates are checked in order and the first failure is the one reported. +// Reporting the last, or all of them, buries the thing to fix. +func TestVerifyReportsTheFirstFailure(t *testing.T) { + t.Parallel() + + srv := newMappingServer(t, withChecks, nil) + defer srv.Close() + + // Both predicates fail. + input := verifyInput() + becknOf(input)["message"] = map[string]any{"location": map[string]any{"type": "Polygon"}} + + err := newTestMapper(t).Verify(context.Background(), ref(srv.URL), input) + if err == nil { + t.Fatal("expected a refusal") + } + if !strings.Contains(err.Error(), "needs a Point location") { + t.Errorf("error %q should report the first failing predicate", err) + } + if strings.Contains(err.Error(), "validity window") { + t.Error("only the first failure should be reported") + } +} + +// A mapping that declares no preconditions imposes none. That is what lets one +// provider adopt the key while others have not, so a second provider arriving +// with its own rules does not force every existing mapping to be rewritten. +func TestVerifyAllowsAMappingWithNoChecks(t *testing.T) { + t.Parallel() + + srv := newMappingServer(t, bothDirections, nil) + defer srv.Close() + + if err := newTestMapper(t).Verify(context.Background(), ref(srv.URL), requestInput()); err != nil { + t.Errorf("a mapping with no required block must impose nothing: %v", err) + } +} + +// A predicate has to answer true or false. Anything else -- a string, a number, +// nothing at all -- is a mapping fault, and must not be read as permission: a +// typo that yields undefined would otherwise wave every request through. +func TestVerifyRefusesAPredicateThatIsNotABoolean(t *testing.T) { + t.Parallel() + + testCases := []struct{ name, check string }{ + {"a string", `"yes"`}, + {"a number", `1`}, + {"a field that does not exist", `beckn.message.nothing.here`}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + mapping := "required:\n - check: " + tc.check + "\n message: \"nope\"\n\nrequest: |\n { \"a\": 1 }\n" + srv := newMappingServer(t, mapping, nil) + defer srv.Close() + + if err := newTestMapper(t).Verify(context.Background(), ref(srv.URL), verifyInput()); err == nil { + t.Error("a predicate that is not a boolean must be refused, not treated as permission") + } + }) + } +} + +// A predicate that will not compile is the mapping's fault, and is reported as +// one -- but it must not take the halves down with it, exactly as a broken half +// does not take its sibling down. +func TestVerifyIsolatesAPredicateThatWillNotCompile(t *testing.T) { + t.Parallel() + + mapping := `required: + - check: "{{{" + message: "unreachable" + +request: | + { "txn": beckn.context.transactionId } +` + srv := newMappingServer(t, mapping, nil) + defer srv.Close() + mapper := newTestMapper(t) + + if err := mapper.Verify(context.Background(), ref(srv.URL), verifyInput()); err == nil { + t.Error("expected an uncompilable predicate to be reported") + } + // The request half still works: a broken precondition is not a broken file. + if _, err := mapper.Transform(context.Background(), ref(srv.URL), + definition.DirectionRequest, verifyInput()); err != nil { + t.Errorf("the request half must still be served: %v", err) + } +} + +// An entry with no message is a mapping that refuses without saying why, which +// is the failure this whole key exists to avoid. +func TestVerifyRefusesAPredicateWithNoMessage(t *testing.T) { + t.Parallel() + + srv := newMappingServer(t, "required:\n - check: 'false'\n\nrequest: |\n { \"a\": 1 }\n", nil) + defer srv.Close() + + if err := newTestMapper(t).Verify(context.Background(), ref(srv.URL), verifyInput()); err == nil { + t.Error("a predicate with no reject message must be refused as a mapping fault") + } +} + +// Preconditions are fetched and compiled with the halves: one round trip leaves +// the whole file ready. +func TestVerifyCompilesWithTheRestOfTheFile(t *testing.T) { + t.Parallel() + + var fetches atomic.Int32 + srv := newMappingServer(t, withChecks, &fetches) + defer srv.Close() + mapper := newTestMapper(t) + + if err := mapper.Verify(context.Background(), ref(srv.URL), verifyInput()); err != nil { + t.Fatalf("Verify() returned an unexpected error: %v", err) + } + if _, err := mapper.Transform(context.Background(), ref(srv.URL), + definition.DirectionRequest, verifyInput()); err != nil { + t.Fatalf("Transform() returned an unexpected error: %v", err) + } + if got := fetches.Load(); got != 1 { + t.Errorf("fetched %d times, want 1 -- preconditions refetched the file", got) + } +} + // --- direction validation --------------------------------------------------- // A direction outside the two is a caller bug, not a mapping problem, and must diff --git a/pkg/plugin/implementation/mausamgram/cmd/plugin_test.go b/pkg/plugin/implementation/mausamgram/cmd/plugin_test.go index f932a559..81698e48 100644 --- a/pkg/plugin/implementation/mausamgram/cmd/plugin_test.go +++ b/pkg/plugin/implementation/mausamgram/cmd/plugin_test.go @@ -20,6 +20,8 @@ func (stubRegistry) ProviderRecord(context.Context, string) (*model.ProviderReco type stubMapper struct{} +func (stubMapper) Verify(context.Context, string, any) error { return nil } + func (stubMapper) Transform(context.Context, string, definition.Direction, any) ([]byte, error) { return nil, nil } diff --git a/pkg/plugin/implementation/mausamgram/dispatch_test.go b/pkg/plugin/implementation/mausamgram/dispatch_test.go index d13bc333..52f92c3c 100644 --- a/pkg/plugin/implementation/mausamgram/dispatch_test.go +++ b/pkg/plugin/implementation/mausamgram/dispatch_test.go @@ -21,6 +21,8 @@ const dispatchMappingRef = "https://m.example.com/mausamgram/weather-observation // nothing else. type fixedMapper struct{ answer string } +func (m fixedMapper) Verify(context.Context, string, any) error { return nil } + func (m fixedMapper) Transform(_ context.Context, mappingRef string, _ definition.Direction, _ any) ([]byte, error) { if strings.Contains(mappingRef, "request") { return []byte(`{}`), nil diff --git a/pkg/plugin/implementation/mausamgram/mappings_test.go b/pkg/plugin/implementation/mausamgram/mappings_test.go index 83b3aa69..92b40f22 100644 --- a/pkg/plugin/implementation/mausamgram/mappings_test.go +++ b/pkg/plugin/implementation/mausamgram/mappings_test.go @@ -355,6 +355,133 @@ func TestShippedMappingsExtractTheQueryFromThePayload(t *testing.T) { } } +// The shipped mapping's own preconditions, against the published file. This is +// where "which geometries does this capability serve" is now answered -- in +// configuration, not in Go. +func TestShippedMappingsPreconditions(t *testing.T) { + mappings := serveMappings(t) + defer mappings.Close() + + mapper, closeMapper, err := jsonmapper.New(context.Background(), &jsonmapper.Config{}) + if err != nil { + t.Fatalf("failed to build the mapper: %v", err) + } + defer closeMapper() + ref := mappings.URL + "/" + shippedMapping + + payload := func(t *testing.T, geometry string) map[string]any { + t.Helper() + location := "" + if geometry != "" { + location = `"location": ` + geometry + `,` + } + body := `{"context":{"action":"select"},"message":{"contract":{"commitments":[{"resources":[{"resourceAttributes":{` + + location + `"@type":"openagrinet:WeatherObservation"}}]}]}}}` + var beckn any + if err := json.Unmarshal([]byte(body), &beckn); err != nil { + t.Fatalf("failed to build the payload: %v", err) + } + return map[string]any{"beckn": beckn} + } + + t.Run("a Point is served", func(t *testing.T) { + if err := mapper.Verify(context.Background(), ref, + payload(t, `{"type":"Point","coordinates":[73.7898,19.9975]}`)); err != nil { + t.Errorf("a Point must be served: %v", err) + } + }) + + for _, tc := range []struct{ name, geometry string }{ + {"a polygon", `{"type":"Polygon","coordinates":[[[73.0,19.0],[74.0,19.0],[74.0,20.0],[73.0,19.0]]]}`}, + {"a line string", `{"type":"LineString","coordinates":[[73.0,19.0],[74.0,20.0]]}`}, + {"several points", `{"type":"MultiPoint","coordinates":[[73.7898,19.9975]]}`}, + {"no location at all", ``}, + } { + t.Run(tc.name+" is refused", func(t *testing.T) { + err := mapper.Verify(context.Background(), ref, payload(t, tc.geometry)) + if err == nil { + t.Fatalf("expected %s to be refused", tc.name) + } + // The message is the mapping's, and has to name what is needed. + if !strings.Contains(err.Error(), "Point") { + t.Errorf("error %q should say a Point is what this capability needs", err) + } + }) + } +} + +// How many days the provider answers with is the provider's business, not the +// mapping's. It returns fcstday1..fcstdayN and N is whatever the forecast ran +// to, so a mapping naming five would truncate a ten-day answer and pad a +// three-day one. +// +// The ordering matters as much as the count: the keys sort lexically as +// fcstday1, fcstday10, fcstday2, so the mapping sorts on the numeric suffix. A +// ten-day forecast delivered in that order would be wrong in a way nothing +// downstream could detect. +func TestShippedMappingsTakeHoweverManyDaysTheProviderSent(t *testing.T) { + mappings := serveMappings(t) + defer mappings.Close() + + mapper, closeMapper, err := jsonmapper.New(context.Background(), &jsonmapper.Config{}) + if err != nil { + t.Fatalf("failed to build the mapper: %v", err) + } + defer closeMapper() + + var beckn any + if err := json.Unmarshal([]byte(selectRequest), &beckn); err != nil { + t.Fatalf("failed to decode the request: %v", err) + } + + for _, days := range []int{1, 3, 10} { + t.Run(fmt.Sprintf("%d days", days), func(t *testing.T) { + provider := map[string]any{"location": map[string]any{"lat": 19.9975, "lon": 73.7898}} + for i := 1; i <= days; i++ { + provider[fmt.Sprintf("fcstday%d", i)] = map[string]any{ + "date": fmt.Sprintf("2026-09-%02d", i), + "rain": float64(i), + } + } + + got, err := mapper.Transform(context.Background(), mappings.URL+"/"+shippedMapping, + definition.DirectionResponse, + map[string]any{"beckn": beckn, "response": provider}) + if err != nil { + t.Fatalf("Transform() returned an unexpected error: %v", err) + } + + var answer map[string]any + if err := json.Unmarshal(got, &answer); err != nil { + t.Fatalf("failed to decode the answer: %v", err) + } + attributes := firstCommitment(t, answer)["resources"].([]any)[0].(map[string]any)["resourceAttributes"].(map[string]any) + observations, _ := attributes["observations"].([]any) + + if len(observations) != days { + t.Fatalf("got %d observations, want %d -- the mapping is not reading the provider's own count", + len(observations), days) + } + + // In the provider's order, not the keys' lexical order. + for i, entry := range observations { + validity := entry.(map[string]any)["validity"].(map[string]any) + want := fmt.Sprintf("2026-09-%02d", i+1) + if validity["startsAt"] != want { + t.Errorf("observation %d covers %v, want %s -- days are out of order", + i, validity["startsAt"], want) + } + } + + // The resource-level window still spans first to last. + window, _ := attributes["validity"].(map[string]any) + if window["endsAt"] != fmt.Sprintf("2026-09-%02d", days) { + t.Errorf("validity ends at %v, want the last day the provider sent", window["endsAt"]) + } + }) + } +} + func firstCommitment(t *testing.T, answer map[string]any) map[string]any { t.Helper() message, _ := answer["message"].(map[string]any) diff --git a/pkg/plugin/implementation/mausamgram/mausamgram.go b/pkg/plugin/implementation/mausamgram/mausamgram.go index 019ece7d..7337dbe3 100644 --- a/pkg/plugin/implementation/mausamgram/mausamgram.go +++ b/pkg/plugin/implementation/mausamgram/mausamgram.go @@ -195,12 +195,16 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { plan.BindingKey, action, strings.Join(servedActions(plan), ", "))) } - if err := verifyGeometry(ctx.Body); err != nil { + beckn, err := decodeBody(ctx.Body) + if err != nil { return err } - beckn, err := decodeBody(ctx.Body) - if err != nil { + // What this provider requires of a payload is declared by its mapping, not + // by this step. A capability with a different rule is a different mapping + // file rather than a different build -- and the rule sits beside the + // extraction it guards. + if err := s.mapper.Verify(ctx, call.Mappings, map[string]any{"beckn": beckn}); err != nil { return err } @@ -278,70 +282,6 @@ func (s *Step) buildRequest(ctx context.Context, call model.ActionPlan, beckn an return mapped, nil } -// geometryPoint is the GeoJSON type this capability can serve. IMD takes one -// latitude and one longitude, so an area or a set of points has no single answer -// -- and picking a centroid would invent a location the caller did not ask about. -const geometryPoint = "Point" - -// verifyGeometry refuses a request whose location this capability cannot serve. -// -// It reads the geometry's type and NOTHING ELSE. Coordinates, dates and every -// other field are the mapping's to extract, which is what keeps "the provider -// wants another parameter" a mapping edit rather than a rebuild. This exists -// only because a mapping cannot refuse: JSONata would turn a Polygon's nested -// coordinates into a query parameter that is not a scalar, and surface as an -// error naming neither the geometry nor the reason. -// -// A location that is absent is refused too, and separately: that is a different -// fact from a geometry this provider does not serve, and a caller can act on the -// difference. -func verifyGeometry(body []byte) error { - var payload struct { - Message struct { - Contract struct { - Commitments []struct { - Resources []struct { - ResourceAttributes struct { - Location *struct { - Type string `json:"type"` - } `json:"location"` - } `json:"resourceAttributes"` - } `json:"resources"` - } `json:"commitments"` - } `json:"contract"` - } `json:"message"` - } - if err := json.Unmarshal(body, &payload); err != nil { - return model.NewBadReqErr("", - fmt.Errorf("mausamgram: request could not be read: %w", err)) - } - - var unsupported string - for _, commitment := range payload.Message.Contract.Commitments { - for _, resource := range commitment.Resources { - location := resource.ResourceAttributes.Location - if location == nil { - continue - } - if location.Type == geometryPoint { - return nil - } - unsupported = location.Type - if unsupported == "" { - unsupported = "a geometry with no type" - } - } - } - - if unsupported != "" { - return model.NewBadReqErr("", fmt.Errorf( - "mausamgram: request carries %s; this capability needs a %s", - unsupported, geometryPoint)) - } - return model.NewBadReqErr("", - errors.New("mausamgram: request carries no location")) -} - // extractAction reads the Beckn action a request is for. func extractAction(body []byte) string { var payload struct { diff --git a/pkg/plugin/implementation/mausamgram/mausamgram_test.go b/pkg/plugin/implementation/mausamgram/mausamgram_test.go index 340e1ae1..44169b32 100644 --- a/pkg/plugin/implementation/mausamgram/mausamgram_test.go +++ b/pkg/plugin/implementation/mausamgram/mausamgram_test.go @@ -49,12 +49,22 @@ type stubMapper struct { err error requestErr error + verifyErr error + verified bool + requestInput any responseInput any directions []definition.Direction refs []string } +// verifyErr is what Verify answers with, so a test can stand in for a mapping +// whose precondition refused. +func (s *stubMapper) Verify(_ context.Context, mappingRef string, input any) error { + s.verified = true + return s.verifyErr +} + func (s *stubMapper) Transform(_ context.Context, mappingRef string, direction definition.Direction, input any) ([]byte, error) { s.directions = append(s.directions, direction) s.refs = append(s.refs, mappingRef) @@ -430,7 +440,7 @@ func TestRunRefusesWhenTheResponseMappingProducesNothing(t *testing.T) { } } -// --- the geometry the request carries --------------------------------------- +// --- preconditions ---------------------------------------------------------- // selectWithLocation renders a select payload whose resource carries the given // GeoJSON geometry verbatim, or none at all when geometry is empty. @@ -452,123 +462,77 @@ func selectWithLocation(t *testing.T, geometry string) string { }` } -// This capability needs one point. A request carrying any other geometry is the -// caller sending something this provider cannot serve, so it has to come back as -// a bad request naming what was sent -- not as a 500, which says the fault is -// ours and tells the caller nothing. +// What a payload must satisfy is the mapping's rule, not this step's. The step's +// job is to ask, and to stop when the answer is no -- without calling the +// provider. // -// Before this was fixed, a Polygon reached json.Unmarshal as a three-deep array -// where a flat pair was expected, failed there, and surfaced as -// NET_INTERNAL_ERROR. -func TestRunRefusesAGeometryThatIsNotAPoint(t *testing.T) { - t.Parallel() - - testCases := []struct { - name string - geometry string - }{ - {"a polygon", `{"type":"Polygon","coordinates":[[[73.0,19.0],[74.0,19.0],[74.0,20.0],[73.0,19.0]]]}`}, - {"a line string", `{"type":"LineString","coordinates":[[73.0,19.0],[74.0,20.0]]}`}, - {"several points", `{"type":"MultiPoint","coordinates":[[73.7898,19.9975],[75.0,21.0]]}`}, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - t.Error("the provider must not be called for a geometry this capability cannot serve") - })) - defer upstream.Close() - - mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} - _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), - selectWithLocation(t, tc.geometry)) - if err == nil { - t.Fatal("expected an unsupported geometry to be refused") - } - - var coded *model.CodedErr - if !errors.As(err, &coded) { - t.Fatalf("error is %T, want a coded error so it NACKs as a bad request: %v", err, err) - } - if coded.HTTPStatus() != http.StatusBadRequest { - t.Errorf("status = %d, want 400 -- the payload is the caller's, not our fault", coded.HTTPStatus()) - } - // Naming the geometry that arrived is what makes this actionable. - if !strings.Contains(err.Error(), "Point") { - t.Errorf("error %q should say a Point is what this capability needs", err) - } - }) - } -} - -// A Point passes the guard and reaches the mapping. -func TestRunAcceptsAPoint(t *testing.T) { +// This step used to hold the rule itself: it read the geometry and required a +// Point. That meant a capability with a different rule needed a different build. +// Which geometries the shipped mapping accepts is now asserted in +// mappings_test.go, against the published file. +func TestRunRefusesWhenTheMappingsPreconditionFails(t *testing.T) { t.Parallel() upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{}`) + t.Error("the provider must not be called when a precondition failed") })) defer upstream.Close() - mapper := &stubMapper{requestResult: []byte(`{"lat":19.9975}`), responseResult: []byte(`{}`)} - if _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), - selectWithLocation(t, `{"type":"Point","coordinates":[73.7898,19.9975]}`)); err != nil { - t.Fatalf("Run() returned an unexpected error: %v", err) + refusal := model.NewBadReqErr("", errors.New("this capability needs a Point location")) + mapper := &stubMapper{verifyErr: refusal, requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + + _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), + selectWithLocation(t, `{"type":"Polygon","coordinates":[[[73.0,19.0],[74.0,19.0],[74.0,20.0],[73.0,19.0]]]}`)) + if err == nil { + t.Fatal("expected a failed precondition to be refused") } - if mapper.requestInput == nil { - t.Error("the request mapping was never called for a Point") + // Propagated verbatim: the mapping's message is what the caller reads. + if !errors.Is(err, refusal) { + t.Errorf("error = %v, want the mapping's own refusal propagated", err) } } -// The guard checks the geometry and nothing else. It does not read coordinates, -// so which payload fields reach the provider stays entirely the mapping's -- -// adding a parameter is a mapping edit, never a rebuild. -func TestRunGuardsGeometryWithoutReadingCoordinates(t *testing.T) { +// Preconditions are checked before the request is built, so a mapping can refuse +// a payload its own request half could not have read. +func TestRunChecksPreconditionsBeforeBuildingTheRequest(t *testing.T) { t.Parallel() - var gotQuery string upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - gotQuery = r.URL.RawQuery - fmt.Fprint(w, `{}`) + t.Error("the provider must not be called when a precondition failed") })) defer upstream.Close() - // A Point whose coordinates the step never looks at, and a mapping that - // names something else entirely. - mapper := &stubMapper{requestResult: []byte(`{"station":"NASHIK-1"}`), responseResult: []byte(`{}`)} - if _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), - selectWithLocation(t, `{"type":"Point","coordinates":[1.0,2.0]}`)); err != nil { - t.Fatalf("Run() returned an unexpected error: %v", err) + mapper := &stubMapper{ + verifyErr: model.NewBadReqErr("", errors.New("nope")), + requestResult: []byte(`{}`), + responseResult: []byte(`{}`), } - if !strings.Contains(gotQuery, "station=NASHIK-1") { - t.Errorf("query = %q, want the mapping's own field", gotQuery) + if _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), + selectBody); err == nil { + t.Fatal("expected a refusal") } - if strings.Contains(gotQuery, "lat=") || strings.Contains(gotQuery, "lon=") { - t.Errorf("query = %q -- the step is still injecting coordinates of its own", gotQuery) + if mapper.requestInput != nil { + t.Error("the request half ran despite a failed precondition") } } -// An absent location was already a bad request, and stays one. Kept alongside the -// geometry cases so the two failures are visibly the same kind of thing. -func TestRunRefusesAMissingLocation(t *testing.T) { +// A mapping declaring no preconditions imposes none, and the step asks anyway -- +// so adopting the facility is per provider, not all at once. +func TestRunProceedsWhenTheMappingImposesNothing(t *testing.T) { t.Parallel() upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - t.Error("the provider must not be called without a location") + fmt.Fprint(w, `{}`) })) defer upstream.Close() mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} - _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), - selectWithLocation(t, "")) - if err == nil { - t.Fatal("expected a request with no location to be refused") + if _, err := runStep(t, newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper), + selectWithLocation(t, `{"type":"Polygon","coordinates":[[[1.0,2.0]]]}`)); err != nil { + t.Fatalf("a mapping imposing nothing must let a request through: %v", err) } - var coded *model.CodedErr - if !errors.As(err, &coded) || coded.HTTPStatus() != http.StatusBadRequest { - t.Errorf("expected a 400, got %v", err) + if !mapper.verified { + t.Error("the step did not ask the mapping at all") } } From bf7dcdee9e6fe2decc948855d2dc858c62f759ba Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 01:39:14 +0530 Subject: [PATCH 036/130] feat: let one provider step serve several capabilities [#1] A provider can serve more than one capability. The registry contract says so outright: a provider serving two capabilities is one Participant and two ProviderSchema rows. The adapter could not. bindingKey was a single string, so a second capability meant a second providerSteps entry with the same plugin id -- and those collide in the handler's id-keyed step map. The second silently overwrote the first, the step list could only name it once, and a capability disappeared with no error at startup or at request time. bindingKeys is a list now, and the step checks membership. Comma-separated, because a plugin config value is a string: the convention reqpreprocessor and schemav2validator already use, and unambiguous here because a binding key separates its own halves with a pipe. Nothing else was needed -- what differs per capability is the endpoint, the mapping and the budget, and all three come from the registry. Configuring the same provider step id twice is now refused at startup rather than quietly losing one. The message says where the capabilities belong instead, because the mistake is easy to make and impossible to see: provider step "mausamgram" is configured more than once; a step serving several capabilities lists them in its own config Widening the config must not widen the dispatch, so a test pins that a capability the step is NOT configured for still passes through untouched. That is the mechanism several provider steps depend on to coexist. Verified end to end, including the duplicate-id case: with two entries sharing an id the adapter refuses to start, where before it started and served one capability fewer. --- core/module/handler/responsebody_test.go | 28 +++++ core/module/handler/stdHandler.go | 8 ++ .../implementation/mausamgram/cmd/plugin.go | 22 +++- .../mausamgram/cmd/plugin_test.go | 27 ++++- .../mausamgram/dispatch_test.go | 6 +- .../implementation/mausamgram/mausamgram.go | 40 +++++-- .../mausamgram/mausamgram_test.go | 103 ++++++++++++++++++ 7 files changed, 220 insertions(+), 14 deletions(-) diff --git a/core/module/handler/responsebody_test.go b/core/module/handler/responsebody_test.go index 59121226..55ec8f0d 100644 --- a/core/module/handler/responsebody_test.go +++ b/core/module/handler/responsebody_test.go @@ -1,6 +1,7 @@ package handler import ( + "context" "encoding/json" "errors" "net/http" @@ -10,6 +11,7 @@ import ( "testing" "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" ) @@ -166,6 +168,32 @@ func TestServeHTTPTreatsAnEmptyAnswerAsNoAnswer(t *testing.T) { } } +// Provider steps land in the same id-keyed map as plain steps, so two entries +// sharing an id would leave one silently overwritten -- a capability lost with +// no error anywhere. Refused at startup instead. +// +// With binding keys a list, one entry serves several capabilities, so a repeated +// id is now a mistake rather than the way to configure a second one. +func TestInitStepsRefusesTwoProviderStepsWithTheSameID(t *testing.T) { + h := &stdHandler{moduleName: "test-module"} + cfg := &Config{ + Plugins: PluginCfg{ + ProviderSteps: []plugin.Config{ + {ID: "mausamgram", Config: map[string]string{}}, + {ID: "mausamgram", Config: map[string]string{}}, + }, + }, + } + + err := h.initSteps(context.Background(), noopPluginManager{}, cfg) + if err == nil { + t.Fatal("expected two provider steps with the same id to be refused") + } + if !strings.Contains(err.Error(), "mausamgram") { + t.Errorf("error %q should name the id that repeats", err) + } +} + // --- an unanswered request in a provider module ------------------------------ // silentStep is the dispatch no-op: a provider step recognising the request as diff --git a/core/module/handler/stdHandler.go b/core/module/handler/stdHandler.go index 8969ea6c..39c02063 100644 --- a/core/module/handler/stdHandler.go +++ b/core/module/handler/stdHandler.go @@ -678,6 +678,14 @@ func (h *stdHandler) initSteps(ctx context.Context, mgr PluginManager, cfg *Conf // plugin steps cannot receive. They land in the same id-keyed map, so a step // list names them exactly like any other plugin step. for _, c := range cfg.Plugins.ProviderSteps { + // The same map as plain steps, so a repeated id would leave one entry + // silently overwritten -- a capability lost with no error anywhere. A + // provider step serving several capabilities says so in its own config + // rather than by appearing twice. + if _, taken := steps[c.ID]; taken { + return fmt.Errorf("provider step %q is configured more than once; "+ + "a step serving several capabilities lists them in its own config", c.ID) + } step, err := h.loadProviderStep(ctx, mgr, &c) if err != nil { return err diff --git a/pkg/plugin/implementation/mausamgram/cmd/plugin.go b/pkg/plugin/implementation/mausamgram/cmd/plugin.go index bfc2b8e5..c07bdcc6 100644 --- a/pkg/plugin/implementation/mausamgram/cmd/plugin.go +++ b/pkg/plugin/implementation/mausamgram/cmd/plugin.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "strconv" + "strings" "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" @@ -22,7 +23,7 @@ var newStepFunc = mausamgram.New // scheme, so those rules live in one place. func (p mausamgramProvider) parseConfig(config map[string]string) (*mausamgram.Config, error) { cfg := &mausamgram.Config{ - BindingKey: config["bindingKey"], + BindingKeys: splitList(config["bindingKeys"]), AuthScheme: config["authScheme"], UsernameEnv: config["usernameEnv"], PasswordEnv: config["passwordEnv"], @@ -67,6 +68,25 @@ func (p mausamgramProvider) New(ctx context.Context, registry definition.Provide } // Provider is the exported plugin instance. +// splitList reads a comma-separated config value, which is how a list reaches a +// plugin -- the config is map[string]string. Blanks are dropped and spaces +// trimmed, so a trailing comma or a wrapped line is not a config error. +// +// A comma is unambiguous here: a binding key separates its own halves with a +// pipe. +func splitList(raw string) []string { + if strings.TrimSpace(raw) == "" { + return nil + } + var out []string + for _, part := range strings.Split(raw, ",") { + if trimmed := strings.TrimSpace(part); trimmed != "" { + out = append(out, trimmed) + } + } + return out +} + var Provider = mausamgramProvider{} // Compile-time proof the provider satisfies the interface the manager asserts diff --git a/pkg/plugin/implementation/mausamgram/cmd/plugin_test.go b/pkg/plugin/implementation/mausamgram/cmd/plugin_test.go index 81698e48..878301f4 100644 --- a/pkg/plugin/implementation/mausamgram/cmd/plugin_test.go +++ b/pkg/plugin/implementation/mausamgram/cmd/plugin_test.go @@ -45,7 +45,7 @@ func TestParseConfig(t *testing.T) { { name: "reads every supported setting", config: map[string]string{ - "bindingKey": "other|capability", + "bindingKeys": "other|capability", "authScheme": "basic", "usernameEnv": "U", "passwordEnv": "P", @@ -54,7 +54,7 @@ func TestParseConfig(t *testing.T) { "maxResponseBytes": "2048", }, expected: &mausamgram.Config{ - BindingKey: "other|capability", + BindingKeys: []string{"other|capability"}, AuthScheme: "basic", UsernameEnv: "U", PasswordEnv: "P", @@ -101,6 +101,29 @@ func TestParseConfig(t *testing.T) { } } +// A plugin config is map[string]string, so a list arrives comma-separated -- +// the convention reqpreprocessor and schemav2validator already use. Binding keys +// separate their own halves with a pipe, so a comma is unambiguous. +func TestParseConfigReadsSeveralBindingKeys(t *testing.T) { + t.Parallel() + + cfg, err := mausamgramProvider{}.parseConfig(map[string]string{ + "bindingKeys": "a|openagrinet:One, b|openagrinet:Two ,, ", + }) + if err != nil { + t.Fatalf("parseConfig() returned an unexpected error: %v", err) + } + want := []string{"a|openagrinet:One", "b|openagrinet:Two"} + if len(cfg.BindingKeys) != len(want) { + t.Fatalf("binding keys = %v, want %v -- blanks should be dropped and spaces trimmed", cfg.BindingKeys, want) + } + for i, key := range want { + if cfg.BindingKeys[i] != key { + t.Errorf("binding key %d = %q, want %q", i, cfg.BindingKeys[i], key) + } + } +} + func TestNew(t *testing.T) { t.Parallel() diff --git a/pkg/plugin/implementation/mausamgram/dispatch_test.go b/pkg/plugin/implementation/mausamgram/dispatch_test.go index 52f92c3c..d53f5666 100644 --- a/pkg/plugin/implementation/mausamgram/dispatch_test.go +++ b/pkg/plugin/implementation/mausamgram/dispatch_test.go @@ -49,8 +49,8 @@ func TestTwoProviderStepsDispatchByBindingKey(t *testing.T) { newProviderStep := func(t *testing.T, bindingKey, upstreamURL, answer string) definition.Step { t.Helper() plan := &model.ProviderRecord{ - BindingKey: bindingKey, BaseURL: upstreamURL, - + BindingKey: bindingKey, + BaseURL: upstreamURL, Actions: map[string]model.ActionPlan{ "select": {Method: http.MethodGet, Path: "/x", Mappings: dispatchMappingRef, RetryMax: 1}, }, @@ -58,7 +58,7 @@ func TestTwoProviderStepsDispatchByBindingKey(t *testing.T) { step, closer, err := mausamgram.New(context.Background(), &stubRegistry{plan: plan}, fixedMapper{answer: answer}, - &mausamgram.Config{BindingKey: bindingKey}) + &mausamgram.Config{BindingKeys: []string{bindingKey}}) if err != nil { t.Fatal(err) } diff --git a/pkg/plugin/implementation/mausamgram/mausamgram.go b/pkg/plugin/implementation/mausamgram/mausamgram.go index 7337dbe3..723ccd20 100644 --- a/pkg/plugin/implementation/mausamgram/mausamgram.go +++ b/pkg/plugin/implementation/mausamgram/mausamgram.go @@ -17,6 +17,7 @@ import ( "net/http" "net/url" "os" + "slices" "sort" "strconv" "strings" @@ -61,9 +62,18 @@ const codeUpstreamUnavailable = "NET_DOWNSTREAM_UNAVAILABLE" // Config holds configuration parameters for the step. type Config struct { - // BindingKey is the capability this step answers to. A request for anything - // else passes through untouched. - BindingKey string `yaml:"bindingKey" json:"bindingKey"` + // BindingKeys are the capabilities this step answers to. A request for + // anything else passes through untouched. + // + // A list because a provider can serve more than one: the registry contract + // says a provider serving two capabilities is one Participant and two + // ProviderSchema rows. Configuring a second entry with the same plugin id + // instead would collide in the handler's id-keyed step map, and one + // capability would be lost with no error anywhere. + // + // What differs per capability -- the endpoint, the mapping, the budget -- + // comes from the registry, so one step serving several needs nothing else. + BindingKeys []string `yaml:"bindingKeys" json:"bindingKeys"` // AuthScheme is how credentials are presented upstream: none, basic or // header. Providers differ here -- basic auth, a raw token header, a field @@ -122,14 +132,19 @@ func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper d return nil } - log.Infof(ctx, "Mausamgram step created for binding %s", cfg.BindingKey) + log.Infof(ctx, "Mausamgram step created for %s", strings.Join(cfg.BindingKeys, ", ")) return step, closer, nil } // applyDefaults fills in what was left out and rejects what cannot be defaulted. func applyDefaults(cfg *Config) error { - if cfg.BindingKey == "" { - cfg.BindingKey = DefaultBindingKey + if len(cfg.BindingKeys) == 0 { + cfg.BindingKeys = []string{DefaultBindingKey} + } + for _, key := range cfg.BindingKeys { + if strings.TrimSpace(key) == "" { + return errors.New("mausamgram: bindingKeys carries an empty entry") + } } if cfg.AuthScheme == "" { cfg.AuthScheme = AuthSchemeNone @@ -168,8 +183,8 @@ func (s *Step) Run(ctx *model.StepContext) error { if err != nil { return err } - if binding.Key() != s.config.BindingKey { - log.Debugf(ctx, "mausamgram: %s is not this step's capability, passing through", binding.Key()) + if !s.serves(binding.Key()) { + log.Debugf(ctx, "mausamgram: %s is not one of this step's capabilities, passing through", binding.Key()) return nil } @@ -181,6 +196,15 @@ func (s *Step) Run(ctx *model.StepContext) error { return s.serve(ctx, plan) } +// serves reports whether a binding key is one this step answers to. +// +// A slice rather than a set: a step serves a handful of capabilities at most, so +// the scan costs less than the map would, and the config order is preserved in +// the log line above. +func (s *Step) serves(key string) bool { + return slices.Contains(s.config.BindingKeys, key) +} + // serve runs the exchange this step exists for: resolve, map out, call, map back. func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { action := extractAction(ctx.Body) diff --git a/pkg/plugin/implementation/mausamgram/mausamgram_test.go b/pkg/plugin/implementation/mausamgram/mausamgram_test.go index 44169b32..0a10b129 100644 --- a/pkg/plugin/implementation/mausamgram/mausamgram_test.go +++ b/pkg/plugin/implementation/mausamgram/mausamgram_test.go @@ -440,6 +440,109 @@ func TestRunRefusesWhenTheResponseMappingProducesNothing(t *testing.T) { } } +// --- several capabilities, one step ------------------------------------------- +// +// A provider can serve more than one capability -- the registry contract says so +// outright: "a provider serving two capabilities is one Participant and two +// ProviderSchema rows". The step has to be able to answer to all of them. +// +// It used to hold a single binding key, so a second capability meant a second +// providerSteps entry with the same plugin id. Those collide in the handler's +// id-keyed step map and the second silently wins, which loses a capability with +// no error anywhere. + +func TestRunServesEveryBindingKeyItIsConfiguredFor(t *testing.T) { + t.Parallel() + + for _, capability := range []string{ + "openagrinet:WeatherObservation", + "openagrinet:WeatherAdvisory", + } { + t.Run(capability, func(t *testing.T) { + t.Parallel() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{}`) + })) + defer upstream.Close() + + key := "mausamgram|" + capability + plan := testPlan(upstream.URL, http.MethodGet) + plan.BindingKey = key + + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + step := newStep(t, &stubRegistry{plan: plan}, mapper, func(c *Config) { + c.BindingKeys = []string{ + "mausamgram|openagrinet:WeatherObservation", + "mausamgram|openagrinet:WeatherAdvisory", + } + }) + + body := strings.Replace(selectBody, "openagrinet:WeatherObservation", capability, 1) + ctx, err := runStep(t, step, body) + if err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + if len(ctx.ResponseBody) == 0 { + t.Errorf("%s was not served, though the step is configured for it", key) + } + }) + } +} + +// A capability the step is not configured for still passes through untouched -- +// that is the dispatch mechanism, and widening to a list must not widen it into +// answering for everything. +func TestRunStillPassesThroughACapabilityItIsNotConfiguredFor(t *testing.T) { + t.Parallel() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Error("the provider must not be called for another capability") + })) + defer upstream.Close() + + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper, + func(c *Config) { c.BindingKeys = []string{"someone-else|openagrinet:MandiPrice"} }) + + ctx, err := runStep(t, step, selectBody) + if err != nil { + t.Fatalf("passing through must not be an error: %v", err) + } + if len(ctx.ResponseBody) != 0 { + t.Error("the step answered for a capability it is not configured for") + } +} + +// Configured for nothing means the default capability, so an operator who names +// no binding key gets the one this plugin was written for rather than a step +// that answers to nothing. +func TestNewDefaultsToTheCapabilityThePluginIsFor(t *testing.T) { + t.Parallel() + + step, closer, err := New(context.Background(), &stubRegistry{}, &stubMapper{}, &Config{}) + if err != nil { + t.Fatalf("New() returned an unexpected error: %v", err) + } + t.Cleanup(func() { _ = closer() }) + + if got := step.config.BindingKeys; len(got) != 1 || got[0] != DefaultBindingKey { + t.Errorf("binding keys = %v, want just the default %q", got, DefaultBindingKey) + } +} + +// A binding key naming no capability is a config mistake that would otherwise +// make the step answer to a key nothing can produce. +func TestNewRefusesAnEmptyBindingKey(t *testing.T) { + t.Parallel() + + _, _, err := New(context.Background(), &stubRegistry{}, &stubMapper{}, + &Config{BindingKeys: []string{"mausamgram|openagrinet:WeatherObservation", " "}}) + if err == nil { + t.Error("expected an empty binding key to be refused") + } +} + // --- preconditions ---------------------------------------------------------- // selectWithLocation renders a select payload whose resource carries the given From 5c033d83112ad5bc1d6dbcddedec32ae06c3e0c5 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 02:38:15 +0530 Subject: [PATCH 037/130] refactor: split the provider step into machinery and domain [#1] mausamgram held two things that had grown apart: the machinery for calling an API that has never heard of Beckn, and the fact that it was IMD's weather forecast. By the time preconditions and extraction had moved into the mapping, the second was nothing but a package name -- every remaining line was generic. internal/upstream is the machinery. "upstream" is the registry's own word for such an API, a Participant of type upstream as against a node that speaks Beckn, so the package says what it does in the network's vocabulary rather than a new one. It recognises a capability, resolves the call plan, authenticates, calls with the registry's budget, and translates in both directions. None of that differs by domain. weather is the domain package, and it is 56 lines. One package per schema pack family, so which plugin owns a capability is readable from its binding key: openagrinet:WeatherObservation and openagrinet:WeatherAdvisory are weather's, openagrinet:MandiPrice will not be. A market or knowledge plugin is now a package of the same size and a cmd directory. What a domain package owns is its name and its prerequisites -- the work a mapping cannot express, keyed by binding key. The map is empty, deliberately: every capability so far is served by reading the payload, which the mapping does. An entry is needed only for real I/O, a station id from a spatial lookup or a token from an exchange, because no expression language should be able to do those. Adding one is a function and a line, and nothing else in the package moves. So _local returns, and this time it earns the name: it carries whatever a prerequisite produced, and an empty map when there is none. A mapping reading _local.stationId on a capability without one finds a missing field rather than failing. There is no default binding key any more. A package serving a family cannot guess which of its capabilities a deployment has providers for, so naming one would be wrong for every other domain built on the same machinery -- and silently wrong. It is required, and refused at startup. The mausamgram name survives where it belongs: as a provider id in fixtures, in resource ids, and as the directory its mapping lives in. That is the provider, not the plugin. Verified end to end: the step loads as "weather", serves the capability its config names, and the mapping's own preconditions still refuse a Polygon and an absent location with the message the mapping supplies. --- config/oan-provider-adapter.yaml | 6 +- install/build-plugins.sh | 2 +- .../upstream}/dispatch_test.go | 31 +++- .../upstream/upstream.go} | 139 ++++++++++++------ .../upstream/upstream_test.go} | 80 ++++++---- .../{mausamgram => weather}/cmd/plugin.go | 26 ++-- .../cmd/plugin_test.go | 41 ++++-- .../{mausamgram => weather}/mappings_test.go | 13 +- .../implementation/weather/prerequisites.go | 24 +++ pkg/plugin/implementation/weather/weather.go | 32 ++++ 10 files changed, 277 insertions(+), 117 deletions(-) rename pkg/plugin/implementation/{mausamgram => internal/upstream}/dispatch_test.go (69%) rename pkg/plugin/implementation/{mausamgram/mausamgram.go => internal/upstream/upstream.go} (73%) rename pkg/plugin/implementation/{mausamgram/mausamgram_test.go => internal/upstream/upstream_test.go} (93%) rename pkg/plugin/implementation/{mausamgram => weather}/cmd/plugin.go (70%) rename pkg/plugin/implementation/{mausamgram => weather}/cmd/plugin_test.go (74%) rename pkg/plugin/implementation/{mausamgram => weather}/mappings_test.go (97%) create mode 100644 pkg/plugin/implementation/weather/prerequisites.go create mode 100644 pkg/plugin/implementation/weather/weather.go diff --git a/config/oan-provider-adapter.yaml b/config/oan-provider-adapter.yaml index 7c289742..8ac94201 100644 --- a/config/oan-provider-adapter.yaml +++ b/config/oan-provider-adapter.yaml @@ -75,14 +75,14 @@ modules: # mechanism. Credentials are named, never held: authScheme says how to # present them, and the *Env keys name environment variables. providerSteps: - - id: mausamgram + - id: weather config: - bindingKey: "mausamgram|openagrinet:WeatherObservation" + bindingKeys: "mausamgram|openagrinet:WeatherObservation" authScheme: basic usernameEnv: MAUSAMGRAM_USER passwordEnv: MAUSAMGRAM_X_API_KEY steps: - validateSign # the sender's key, from the registry - - mausamgram # resolve, map out, call, map back + - weather # resolve, map out, call, map back - signAck # signs whatever the step answered with diff --git a/install/build-plugins.sh b/install/build-plugins.sh index a185f29a..3bf07a01 100755 --- a/install/build-plugins.sh +++ b/install/build-plugins.sh @@ -32,7 +32,7 @@ plugins=( "dediregistry" "oanregistry" "jsonmapper" - "mausamgram" + "weather" "manifestloader" "reqpreprocessor" "otelsetup" diff --git a/pkg/plugin/implementation/mausamgram/dispatch_test.go b/pkg/plugin/implementation/internal/upstream/dispatch_test.go similarity index 69% rename from pkg/plugin/implementation/mausamgram/dispatch_test.go rename to pkg/plugin/implementation/internal/upstream/dispatch_test.go index d53f5666..09cceb3e 100644 --- a/pkg/plugin/implementation/mausamgram/dispatch_test.go +++ b/pkg/plugin/implementation/internal/upstream/dispatch_test.go @@ -1,4 +1,4 @@ -package mausamgram_test +package upstream_test import ( "context" @@ -10,13 +10,33 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/mausamgram" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" ) // dispatchMappingRef stands in for the one reference an action carries. This // test is about dispatch, so what is behind it never matters. const dispatchMappingRef = "https://m.example.com/mausamgram/weather-observation.select.yaml" +// dispatchRequest names a provider and a capability, which is all dispatch reads. +// Kept minimal on purpose: this test is about which step claims a request, not +// about what any of them would do with it. +const dispatchRequest = `{ + "context": { "version": "2.0.0", "action": "select" }, + "message": { "contract": { "commitments": [ { + "resources": [ { "resourceAttributes": { "@type": "openagrinet:WeatherObservation" } } ], + "offer": { "provider": { "id": "mausamgram" } } + } ] } } +}` + +// stubRegistry answers with one call plan, whatever is asked. This file's own, +// because it is an external test: it exercises the package exactly as the +// adapter does, through the exported surface and nothing else. +type stubRegistry struct{ plan *model.ProviderRecord } + +func (s *stubRegistry) ProviderRecord(context.Context, string) (*model.ProviderRecord, error) { + return s.plan, nil +} + // fixedMapper returns canned results, so this test is about dispatch and // nothing else. type fixedMapper struct{ answer string } @@ -55,10 +75,11 @@ func TestTwoProviderStepsDispatchByBindingKey(t *testing.T) { "select": {Method: http.MethodGet, Path: "/x", Mappings: dispatchMappingRef, RetryMax: 1}, }, } - step, closer, err := mausamgram.New(context.Background(), + step, closer, err := upstream.New(context.Background(), &stubRegistry{plan: plan}, fixedMapper{answer: answer}, - &mausamgram.Config{BindingKeys: []string{bindingKey}}) + nil, + &upstream.Config{BindingKeys: []string{bindingKey}}) if err != nil { t.Fatal(err) } @@ -73,7 +94,7 @@ func TestTwoProviderStepsDispatchByBindingKey(t *testing.T) { // A request for the FIRST capability, run through both steps in order, as a // pipeline would. - ctx := &model.StepContext{Context: t.Context(), Body: []byte(selectRequest)} + ctx := &model.StepContext{Context: t.Context(), Body: []byte(dispatchRequest)} for i, step := range steps { if err := step.Run(ctx); err != nil { t.Fatalf("step %d: %v", i, err) diff --git a/pkg/plugin/implementation/mausamgram/mausamgram.go b/pkg/plugin/implementation/internal/upstream/upstream.go similarity index 73% rename from pkg/plugin/implementation/mausamgram/mausamgram.go rename to pkg/plugin/implementation/internal/upstream/upstream.go index 723ccd20..f5bec720 100644 --- a/pkg/plugin/implementation/mausamgram/mausamgram.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -1,11 +1,17 @@ -// Package mausamgram serves the IMD Mausamgram point-forecast capability. +// Package upstream serves a Beckn capability by calling an ordinary API that has +// never heard of Beckn. // -// It is the first provider step, and the shape every other one follows: it -// recognises its own capability, resolves what the provider needs beyond the -// Beckn payload, calls it, and lets the mapper translate in both directions. -// Nothing about weather forecasts appears outside this package, and nothing -// about mapping appears inside it. -package mausamgram +// "upstream" is the registry's own word for such an API -- a Participant of type +// upstream, as against a node that speaks Beckn. This package is the machinery +// for calling one: recognise the capability, resolve the call plan, translate +// out, call, translate back. +// +// It holds nothing about any provider or any domain. What varies per capability +// comes from the registry (endpoint, method, budget, which mapping) and from the +// mapping itself (what the payload must satisfy, what to send, what to return). +// A domain package wraps this, supplying only its name and whatever prerequisite +// work a mapping cannot express. +package upstream import ( "bytes" @@ -31,10 +37,6 @@ import ( // Defaults applied when the registry or the operator leaves a setting out. const ( - // DefaultBindingKey is the capability this step serves. It is configurable - // so a deployment can rename the participant without a rebuild, but it has a - // default because a step that serves nothing is never what an operator meant. - DefaultBindingKey = "mausamgram|openagrinet:WeatherObservation" // DefaultTimeout and DefaultRetryMax are the registry contract's defaults // for an action that leaves timeoutMs or retryMax out. Zero retries is // deliberate: a provider that failed is retried only where the operator @@ -60,6 +62,18 @@ const ( // answered with a failure. It is not this adapter's fault and not the caller's. const codeUpstreamUnavailable = "NET_DOWNSTREAM_UNAVAILABLE" +// Prerequisites are the values a capability needs that its payload does not +// carry, keyed by binding key. +// +// A mapping cannot produce them: a station id comes from a spatial lookup, a +// session token from an exchange, a market code from a table. That is real I/O, +// and no expression language should be able to do it. +// +// Whatever a function returns is handed to the mapping as _local, so the mapping +// still decides what the provider is finally asked for. A capability with no +// entry needs nothing, which is the common case. +type Prerequisites map[string]func(context.Context, any) (map[string]any, error) + // Config holds configuration parameters for the step. type Config struct { // BindingKeys are the capabilities this step answers to. A request for @@ -96,19 +110,21 @@ type Config struct { // Step serves the Mausamgram capability. It is safe for concurrent use. type Step struct { - config *Config - registry definition.ProviderRecordLookup - mapper definition.Mapper - httpClient *http.Client + config *Config + prerequisites Prerequisites + registry definition.ProviderRecordLookup + mapper definition.Mapper + httpClient *http.Client } // New creates the step. -func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper definition.Mapper, cfg *Config) (*Step, func() error, error) { +func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper definition.Mapper, + prerequisites Prerequisites, cfg *Config) (*Step, func() error, error) { if registry == nil { - return nil, nil, errors.New("mausamgram: a provider record lookup is required") + return nil, nil, errors.New("upstream: a provider record lookup is required") } if mapper == nil { - return nil, nil, errors.New("mausamgram: a mapper is required") + return nil, nil, errors.New("upstream: a mapper is required") } if cfg == nil { cfg = &Config{} @@ -118,32 +134,36 @@ func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper d } step := &Step{ - config: cfg, - registry: registry, - mapper: mapper, + config: cfg, + prerequisites: prerequisites, + registry: registry, + mapper: mapper, // Timeout is set per request from the registry's own budget, so the // client carries none of its own. httpClient: &http.Client{}, } closer := func() error { - log.Debugf(ctx, "Cleaning up mausamgram step resources") + log.Debugf(ctx, "Cleaning up upstream step resources") step.httpClient.CloseIdleConnections() return nil } - log.Infof(ctx, "Mausamgram step created for %s", strings.Join(cfg.BindingKeys, ", ")) + log.Infof(ctx, "Upstream step created for %s", strings.Join(cfg.BindingKeys, ", ")) return step, closer, nil } // applyDefaults fills in what was left out and rejects what cannot be defaulted. func applyDefaults(cfg *Config) error { + // No default. This package serves whatever a domain package configures it + // for, so a default would have to name one provider's capability -- wrong + // for every other domain built on it, and silently wrong rather than loudly. if len(cfg.BindingKeys) == 0 { - cfg.BindingKeys = []string{DefaultBindingKey} + return errors.New("upstream: bindingKeys is required: it is what this step answers to") } for _, key := range cfg.BindingKeys { if strings.TrimSpace(key) == "" { - return errors.New("mausamgram: bindingKeys carries an empty entry") + return errors.New("upstream: bindingKeys carries an empty entry") } } if cfg.AuthScheme == "" { @@ -157,14 +177,14 @@ func applyDefaults(cfg *Config) error { case AuthSchemeNone: case AuthSchemeBasic: if cfg.UsernameEnv == "" || cfg.PasswordEnv == "" { - return errors.New("mausamgram: authScheme basic requires usernameEnv and passwordEnv") + return errors.New("upstream: authScheme basic requires usernameEnv and passwordEnv") } case AuthSchemeHeader: if cfg.HeaderName == "" || cfg.HeaderValueEnv == "" { - return errors.New("mausamgram: authScheme header requires headerName and headerValueEnv") + return errors.New("upstream: authScheme header requires headerName and headerValueEnv") } default: - return fmt.Errorf("mausamgram: unknown authScheme %q: must be none, basic or header", cfg.AuthScheme) + return fmt.Errorf("upstream: unknown authScheme %q: must be none, basic or header", cfg.AuthScheme) } return nil } @@ -184,18 +204,38 @@ func (s *Step) Run(ctx *model.StepContext) error { return err } if !s.serves(binding.Key()) { - log.Debugf(ctx, "mausamgram: %s is not one of this step's capabilities, passing through", binding.Key()) + log.Debugf(ctx, "upstream: %s is not one of this step's capabilities, passing through", binding.Key()) return nil } plan, err := s.registry.ProviderRecord(ctx, binding.Key()) if err != nil { - return fmt.Errorf("mausamgram: no call plan for %s: %w", binding.Key(), err) + return fmt.Errorf("upstream: no call plan for %s: %w", binding.Key(), err) } return s.serve(ctx, plan) } +// resolve runs whatever prerequisite work this capability needs, and returns the +// values for the mapping to read under _local. +// +// Empty rather than nil when there is nothing: a mapping referring to _local on +// a capability that resolves nothing should read a missing field, not fail. +func (s *Step) resolve(ctx context.Context, bindingKey string, beckn any) (map[string]any, error) { + prerequisite, needed := s.prerequisites[bindingKey] + if !needed { + return map[string]any{}, nil + } + local, err := prerequisite(ctx, beckn) + if err != nil { + return nil, fmt.Errorf("upstream: %s could not resolve what it needs before the call: %w", bindingKey, err) + } + if local == nil { + return map[string]any{}, nil + } + return local, nil +} + // serves reports whether a binding key is one this step answers to. // // A slice rather than a set: a step serves a handful of capabilities at most, so @@ -215,7 +255,7 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { // happened to be on the record -- naming what it does serve turns a // registry mistake into a one-line fix. return model.NewBadReqErr("", fmt.Errorf( - "mausamgram: %s does not serve action %q; it serves %s", + "upstream: %s does not serve action %q; it serves %s", plan.BindingKey, action, strings.Join(servedActions(plan), ", "))) } @@ -232,7 +272,14 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { return err } - upstreamRequest, err := s.buildRequest(ctx, call, beckn) + // Whatever this capability needs that its payload does not carry. Empty for + // most: the mapping reads the payload directly and needs nothing resolved. + local, err := s.resolve(ctx, plan.BindingKey, beckn) + if err != nil { + return err + } + + upstreamRequest, err := s.buildRequest(ctx, call, beckn, local) if err != nil { return err } @@ -244,7 +291,7 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { answer, err := decodeBody(upstreamResponse) if err != nil { - return fmt.Errorf("mausamgram: provider answered with something that is not JSON: %w", err) + return fmt.Errorf("upstream: provider answered with something that is not JSON: %w", err) } // The same mapping reference as the request, other half: one file carries @@ -253,6 +300,7 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { // The mapping is handed what each party sent and nothing else. becknResponse, err := s.mapper.Transform(ctx, call.Mappings, definition.DirectionResponse, map[string]any{ "beckn": beckn, + "_local": local, "response": answer, }) if err != nil { @@ -263,12 +311,12 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { // in this answer. Both leave no Beckn response to return, and returning // the provider's own shape instead would be worse than failing. The // message says what was observed rather than guessing which it was. - return fmt.Errorf("mausamgram: the response half of %s produced nothing, so %s cannot be answered", + return fmt.Errorf("upstream: the response half of %s produced nothing, so %s cannot be answered", call.Mappings, plan.BindingKey) } ctx.ResponseBody = becknResponse - log.Infof(ctx, "mausamgram: served %s in %d bytes", plan.BindingKey, len(becknResponse)) + log.Infof(ctx, "upstream: served %s in %d bytes", plan.BindingKey, len(becknResponse)) return nil } @@ -293,15 +341,16 @@ func servedActions(plan *model.ProviderRecord) []string { // that. It meant the choice of which payload fields reach the provider lived in // Go, so adding a parameter -- a date range, say -- was a rebuild. Now it is a // mapping edit and nothing else. -func (s *Step) buildRequest(ctx context.Context, call model.ActionPlan, beckn any) ([]byte, error) { +func (s *Step) buildRequest(ctx context.Context, call model.ActionPlan, beckn any, local map[string]any) ([]byte, error) { mapped, err := s.mapper.Transform(ctx, call.Mappings, definition.DirectionRequest, map[string]any{ - "beckn": beckn, + "beckn": beckn, + "_local": local, }) if err != nil { return nil, err } if len(mapped) == 0 { - log.Debugf(ctx, "mausamgram: the request half of %s produced nothing; sending an empty request", call.Mappings) + log.Debugf(ctx, "upstream: the request half of %s produced nothing; sending an empty request", call.Mappings) } return mapped, nil } @@ -323,7 +372,7 @@ func extractAction(body []byte) string { func decodeBody(body []byte) (any, error) { var decoded any if err := json.Unmarshal(body, &decoded); err != nil { - return nil, fmt.Errorf("mausamgram: could not read JSON: %w", err) + return nil, fmt.Errorf("upstream: could not read JSON: %w", err) } return decoded, nil } @@ -355,10 +404,10 @@ func (s *Step) call(ctx context.Context, baseURL string, call model.ActionPlan, return body, nil } lastErr = err - log.Warnf(ctx, "mausamgram: attempt %d/%d failed: %v", attempt, attempts, err) + log.Warnf(ctx, "upstream: attempt %d/%d failed: %v", attempt, attempts, err) } return nil, model.NewCodedErr(http.StatusBadGateway, codeUpstreamUnavailable, - fmt.Errorf("mausamgram: provider did not answer after %d attempts: %w", attempts, lastErr)) + fmt.Errorf("upstream: provider did not answer after %d attempts: %w", attempts, lastErr)) } // attempt makes one upstream request. @@ -403,14 +452,14 @@ func (s *Step) authenticate(req *http.Request) error { case AuthSchemeBasic: username, password := os.Getenv(s.config.UsernameEnv), os.Getenv(s.config.PasswordEnv) if username == "" || password == "" { - return fmt.Errorf("mausamgram: %s and %s must both be set for basic auth", + return fmt.Errorf("upstream: %s and %s must both be set for basic auth", s.config.UsernameEnv, s.config.PasswordEnv) } req.SetBasicAuth(username, password) case AuthSchemeHeader: value := os.Getenv(s.config.HeaderValueEnv) if value == "" { - return fmt.Errorf("mausamgram: %s must be set for header auth", s.config.HeaderValueEnv) + return fmt.Errorf("upstream: %s must be set for header auth", s.config.HeaderValueEnv) } req.Header.Set(s.config.HeaderName, value) } @@ -450,14 +499,14 @@ func asQuery(mapped []byte) (string, error) { } var fields map[string]any if err := json.Unmarshal(mapped, &fields); err != nil { - return "", fmt.Errorf("mausamgram: mapped request is not an object, so it cannot become a query: %w", err) + return "", fmt.Errorf("upstream: mapped request is not an object, so it cannot become a query: %w", err) } values := url.Values{} for name, value := range fields { rendered, ok := renderScalar(value) if !ok { - return "", fmt.Errorf("mausamgram: mapped field %q is not a scalar and cannot become a query parameter", name) + return "", fmt.Errorf("upstream: mapped field %q is not a scalar and cannot become a query parameter", name) } values.Set(name, rendered) } diff --git a/pkg/plugin/implementation/mausamgram/mausamgram_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go similarity index 93% rename from pkg/plugin/implementation/mausamgram/mausamgram_test.go rename to pkg/plugin/implementation/internal/upstream/upstream_test.go index 0a10b129..1bb47b2a 100644 --- a/pkg/plugin/implementation/mausamgram/mausamgram_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -1,4 +1,4 @@ -package mausamgram +package upstream import ( "context" @@ -40,6 +40,10 @@ func (s *stubRegistry) ProviderRecord(context.Context, string) (*model.ProviderR // stubMapper records what it was asked and returns canned results, so a test can // testMappingRef is the one reference an action carries: the URL of a single // published file holding both halves. +// testBindingKey is what these tests configure the step for. There is no +// default any more: this package serves whatever a domain package points it at. +const testBindingKey = "mausamgram|openagrinet:WeatherObservation" + const testMappingRef = "https://mappings.example.com/mausamgram/weather-observation.select.yaml" // assert what reached the mapping without writing one. @@ -84,7 +88,7 @@ func (s *stubMapper) Transform(_ context.Context, mappingRef string, direction d func testPlan(baseURL, method string) *model.ProviderRecord { return &model.ProviderRecord{ - BindingKey: DefaultBindingKey, + BindingKey: testBindingKey, ParticipantID: "mausamgram", CapabilityCode: "openagrinet:WeatherObservation", BaseURL: baseURL, @@ -98,11 +102,11 @@ func testPlan(baseURL, method string) *model.ProviderRecord { func newStep(t *testing.T, registry definition.ProviderRecordLookup, mapper definition.Mapper, tweak ...func(*Config)) *Step { t.Helper() - cfg := &Config{} + cfg := &Config{BindingKeys: []string{testBindingKey}} for _, apply := range tweak { apply(cfg) } - step, closer, err := New(context.Background(), registry, mapper, cfg) + step, closer, err := New(context.Background(), registry, mapper, nil, cfg) if err != nil { t.Fatalf("New() returned an unexpected error: %v", err) } @@ -121,10 +125,10 @@ func runStep(t *testing.T, step *Step, body string) (*model.StepContext, error) func TestNewRequiresItsDependencies(t *testing.T) { t.Parallel() - if _, _, err := New(context.Background(), nil, &stubMapper{}, &Config{}); err == nil { + if _, _, err := New(context.Background(), nil, &stubMapper{}, nil, minimalConfig()); err == nil { t.Error("expected a missing registry to be refused") } - if _, _, err := New(context.Background(), &stubRegistry{}, nil, &Config{}); err == nil { + if _, _, err := New(context.Background(), &stubRegistry{}, nil, nil, minimalConfig()); err == nil { t.Error("expected a missing mapper to be refused") } } @@ -150,7 +154,8 @@ func TestNewValidatesTheAuthScheme(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - _, _, err := New(context.Background(), &stubRegistry{}, &stubMapper{}, tc.config) + tc.config.BindingKeys = []string{testBindingKey} + _, _, err := New(context.Background(), &stubRegistry{}, &stubMapper{}, nil, tc.config) if tc.valid && err != nil { t.Errorf("expected the config to be accepted, got %v", err) } @@ -260,19 +265,24 @@ func TestRunKeepsResolvedValuesInScopeForTheResponse(t *testing.T) { if !ok { t.Fatalf("response input = %T, want a map", mapper.responseInput) } - for _, key := range []string{"beckn", "response"} { + for _, key := range []string{"beckn", "_local", "response"} { if _, present := input[key]; !present { t.Errorf("response mapping cannot see %q", key) } } + if len(input) != 3 { + t.Errorf("response input carries %v, want beckn, _local and response", keysOf(input)) + } - // A mapping is handed only what a party sent. Values this step resolved - // before the call stay in the step: it holds them already and uses them - // directly, so passing them through the mapping would be a detour and a - // second name for the same data. - if len(input) != 2 { - t.Errorf("response input carries %d keys (%v), want exactly beckn and response", - len(input), keysOf(input)) + // _local is empty when nothing was resolved, not absent: a mapping reading + // it on a capability with no prerequisites should find a missing field + // rather than fail. + local, ok := input["_local"].(map[string]any) + if !ok { + t.Fatalf("_local = %T, want a map", input["_local"]) + } + if len(local) != 0 { + t.Errorf("_local = %v, want empty -- this capability resolves nothing", local) } } @@ -304,11 +314,13 @@ func TestRunGivesTheRequestMappingOnlyTheInboundPayload(t *testing.T) { if !ok { t.Fatalf("request input = %T, want a map", mapper.requestInput) } - if len(input) != 1 { - t.Errorf("request input carries %v, want only beckn", keysOf(input)) + if len(input) != 2 { + t.Errorf("request input carries %v, want beckn and _local", keysOf(input)) } - if _, present := input["beckn"]; !present { - t.Error("request mapping cannot see beckn") + for _, key := range []string{"beckn", "_local"} { + if _, present := input[key]; !present { + t.Errorf("request mapping cannot see %q", key) + } } } @@ -514,20 +526,24 @@ func TestRunStillPassesThroughACapabilityItIsNotConfiguredFor(t *testing.T) { } } -// Configured for nothing means the default capability, so an operator who names -// no binding key gets the one this plugin was written for rather than a step -// that answers to nothing. -func TestNewDefaultsToTheCapabilityThePluginIsFor(t *testing.T) { +// minimalConfig is the least a step needs: what it answers to. +func minimalConfig() *Config { + return &Config{BindingKeys: []string{testBindingKey}} +} + +// There is no default capability, and there cannot be a sensible one: this +// package serves whatever a domain package points it at, so a default would name +// one provider's capability and be wrong for every other domain built on it. +// Refused at startup, where an operator is watching. +func TestNewRequiresBindingKeys(t *testing.T) { t.Parallel() - step, closer, err := New(context.Background(), &stubRegistry{}, &stubMapper{}, &Config{}) - if err != nil { - t.Fatalf("New() returned an unexpected error: %v", err) + _, _, err := New(context.Background(), &stubRegistry{}, &stubMapper{}, nil, &Config{}) + if err == nil { + t.Fatal("expected a step configured for no capability to be refused") } - t.Cleanup(func() { _ = closer() }) - - if got := step.config.BindingKeys; len(got) != 1 || got[0] != DefaultBindingKey { - t.Errorf("binding keys = %v, want just the default %q", got, DefaultBindingKey) + if !strings.Contains(err.Error(), "bindingKeys") { + t.Errorf("error %q should name the setting that is missing", err) } } @@ -536,8 +552,8 @@ func TestNewDefaultsToTheCapabilityThePluginIsFor(t *testing.T) { func TestNewRefusesAnEmptyBindingKey(t *testing.T) { t.Parallel() - _, _, err := New(context.Background(), &stubRegistry{}, &stubMapper{}, - &Config{BindingKeys: []string{"mausamgram|openagrinet:WeatherObservation", " "}}) + _, _, err := New(context.Background(), &stubRegistry{}, &stubMapper{}, nil, + &Config{BindingKeys: []string{testBindingKey, " "}}) if err == nil { t.Error("expected an empty binding key to be refused") } diff --git a/pkg/plugin/implementation/mausamgram/cmd/plugin.go b/pkg/plugin/implementation/weather/cmd/plugin.go similarity index 70% rename from pkg/plugin/implementation/mausamgram/cmd/plugin.go rename to pkg/plugin/implementation/weather/cmd/plugin.go index c07bdcc6..f5462650 100644 --- a/pkg/plugin/implementation/mausamgram/cmd/plugin.go +++ b/pkg/plugin/implementation/weather/cmd/plugin.go @@ -9,20 +9,20 @@ import ( "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/mausamgram" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/weather" ) -// mausamgramProvider implements definition.ProviderStepProvider. -type mausamgramProvider struct{} +// weatherProvider implements definition.ProviderStepProvider. +type weatherProvider struct{} // newStepFunc creates a new step. Indirected for tests. -var newStepFunc = mausamgram.New +var newStepFunc = weather.New // parseConfig turns the plugin config map into a typed Config. Anything absent -// is left zero: mausamgram.New applies the defaults and validates the auth +// is left zero: weather.New applies the defaults and validates the auth // scheme, so those rules live in one place. -func (p mausamgramProvider) parseConfig(config map[string]string) (*mausamgram.Config, error) { - cfg := &mausamgram.Config{ +func (p weatherProvider) parseConfig(config map[string]string) (*weather.Config, error) { + cfg := &weather.Config{ BindingKeys: splitList(config["bindingKeys"]), AuthScheme: config["authScheme"], UsernameEnv: config["usernameEnv"], @@ -45,21 +45,21 @@ func (p mausamgramProvider) parseConfig(config map[string]string) (*mausamgram.C return cfg, nil } -// New creates a new mausamgram provider step instance. -func (p mausamgramProvider) New(ctx context.Context, registry definition.ProviderRecordLookup, mapper definition.Mapper, config map[string]string) (definition.Step, func() error, error) { +// New creates a new weather provider step instance. +func (p weatherProvider) New(ctx context.Context, registry definition.ProviderRecordLookup, mapper definition.Mapper, config map[string]string) (definition.Step, func() error, error) { if ctx == nil { return nil, nil, errors.New("context cannot be nil") } cfg, err := p.parseConfig(config) if err != nil { - log.Errorf(ctx, err, "Failed to parse mausamgram configuration") - return nil, nil, fmt.Errorf("failed to parse mausamgram configuration: %w", err) + log.Errorf(ctx, err, "Failed to parse weather configuration") + return nil, nil, fmt.Errorf("failed to parse weather configuration: %w", err) } step, closer, err := newStepFunc(ctx, registry, mapper, cfg) if err != nil { - log.Errorf(ctx, err, "Failed to create mausamgram step") + log.Errorf(ctx, err, "Failed to create weather step") return nil, nil, err } @@ -87,7 +87,7 @@ func splitList(raw string) []string { return out } -var Provider = mausamgramProvider{} +var Provider = weatherProvider{} // Compile-time proof the provider satisfies the interface the manager asserts // against. A mismatch is otherwise a runtime cast failure at startup. diff --git a/pkg/plugin/implementation/mausamgram/cmd/plugin_test.go b/pkg/plugin/implementation/weather/cmd/plugin_test.go similarity index 74% rename from pkg/plugin/implementation/mausamgram/cmd/plugin_test.go rename to pkg/plugin/implementation/weather/cmd/plugin_test.go index 878301f4..86c9a7c5 100644 --- a/pkg/plugin/implementation/mausamgram/cmd/plugin_test.go +++ b/pkg/plugin/implementation/weather/cmd/plugin_test.go @@ -9,7 +9,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/mausamgram" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/weather" ) type stubRegistry struct{} @@ -32,15 +32,15 @@ func TestParseConfig(t *testing.T) { testCases := []struct { name string config map[string]string - expected *mausamgram.Config + expected *weather.Config expectedErr string }{ { - // Everything absent is left zero: mausamgram.New defaults it, so the + // Everything absent is left zero: weather.New defaults it, so the // rules are defined in exactly one place. name: "leaves everything unset for New to default", config: map[string]string{}, - expected: &mausamgram.Config{}, + expected: &weather.Config{}, }, { name: "reads every supported setting", @@ -53,7 +53,7 @@ func TestParseConfig(t *testing.T) { "headerValueEnv": "V", "maxResponseBytes": "2048", }, - expected: &mausamgram.Config{ + expected: &weather.Config{ BindingKeys: []string{"other|capability"}, AuthScheme: "basic", UsernameEnv: "U", @@ -79,7 +79,7 @@ func TestParseConfig(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - got, err := mausamgramProvider{}.parseConfig(tc.config) + got, err := weatherProvider{}.parseConfig(tc.config) if tc.expectedErr != "" { if err == nil { @@ -107,7 +107,7 @@ func TestParseConfig(t *testing.T) { func TestParseConfigReadsSeveralBindingKeys(t *testing.T) { t.Parallel() - cfg, err := mausamgramProvider{}.parseConfig(map[string]string{ + cfg, err := weatherProvider{}.parseConfig(map[string]string{ "bindingKeys": "a|openagrinet:One, b|openagrinet:Two ,, ", }) if err != nil { @@ -131,7 +131,7 @@ func TestNew(t *testing.T) { t.Parallel() //nolint:staticcheck // deliberately passing a nil context to assert the guard. - _, _, err := mausamgramProvider{}.New(nil, stubRegistry{}, stubMapper{}, map[string]string{}) + _, _, err := weatherProvider{}.New(nil, stubRegistry{}, stubMapper{}, map[string]string{}) if err == nil { t.Fatal("expected an error for a nil context, got none") } @@ -140,7 +140,7 @@ func TestNew(t *testing.T) { t.Run("rejects an unparseable config", func(t *testing.T) { t.Parallel() - _, _, err := mausamgramProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, + _, _, err := weatherProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, map[string]string{"maxResponseBytes": "lots"}) if err == nil { t.Fatal("expected an error for an invalid cap, got none") @@ -150,17 +150,30 @@ func TestNew(t *testing.T) { t.Run("propagates an invalid auth scheme from New", func(t *testing.T) { t.Parallel() - _, _, err := mausamgramProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, + _, _, err := weatherProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, map[string]string{"authScheme": "oauth"}) if err == nil { t.Fatal("expected an unknown auth scheme to be refused") } }) - t.Run("builds a step from an empty config", func(t *testing.T) { + // A domain plugin serves a family of capabilities, so it cannot guess which + // of them a deployment has providers for. Refused at startup rather than + // answering to nothing. + t.Run("refuses a config naming no capability", func(t *testing.T) { t.Parallel() - step, closer, err := mausamgramProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, map[string]string{}) + if _, _, err := (weatherProvider{}).New(context.Background(), stubRegistry{}, stubMapper{}, + map[string]string{}); err == nil { + t.Fatal("expected a config with no bindingKeys to be refused") + } + }) + + t.Run("builds a step from the capabilities it is given", func(t *testing.T) { + t.Parallel() + + step, closer, err := weatherProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, + map[string]string{"bindingKeys": "imd|openagrinet:WeatherObservation"}) if err != nil { t.Fatalf("expected no error, got: %v", err) } @@ -178,11 +191,11 @@ func TestNew(t *testing.T) { t.Cleanup(func() { newStepFunc = original }) wantErr := errors.New("boom") - newStepFunc = func(context.Context, definition.ProviderRecordLookup, definition.Mapper, *mausamgram.Config) (*mausamgram.Step, func() error, error) { + newStepFunc = func(context.Context, definition.ProviderRecordLookup, definition.Mapper, *weather.Config) (definition.Step, func() error, error) { return nil, nil, wantErr } - _, _, err := mausamgramProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, map[string]string{}) + _, _, err := weatherProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, map[string]string{}) if !errors.Is(err, wantErr) { t.Errorf("expected the construction error to propagate, got %v", err) } diff --git a/pkg/plugin/implementation/mausamgram/mappings_test.go b/pkg/plugin/implementation/weather/mappings_test.go similarity index 97% rename from pkg/plugin/implementation/mausamgram/mappings_test.go rename to pkg/plugin/implementation/weather/mappings_test.go index 92b40f22..5c54ad79 100644 --- a/pkg/plugin/implementation/mausamgram/mappings_test.go +++ b/pkg/plugin/implementation/weather/mappings_test.go @@ -1,4 +1,4 @@ -package mausamgram_test +package weather_test // mappings_test.go runs the shipped mapping files through the real mapper and // the real provider step. It is the only test that proves the three pieces fit: @@ -22,7 +22,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/mausamgram" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/weather" ) // mappingsDir is where the shipped mappings live, relative to this package. @@ -36,6 +36,10 @@ const selectedResourceID = "res:mausamgram:point-forecast" // directions. The registry carries its full URL; the action segment of the name // must match the action that registry entry declares -- a mismatch would apply a // correct mapping to the wrong call, silently. +// shippedBindingKey is the capability these tests exercise. Named here because +// the package has no default: it serves whatever a deployment configures. +const shippedBindingKey = "mausamgram|openagrinet:WeatherObservation" + const shippedMapping = "weather-observation.select.yaml" // selectRequest is the verbatim /select captured from the OAN network. @@ -127,7 +131,7 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { defer closeMapper() registry := &stubRegistry{plan: &model.ProviderRecord{ - BindingKey: mausamgram.DefaultBindingKey, + BindingKey: shippedBindingKey, ParticipantID: "mausamgram", CapabilityCode: "openagrinet:WeatherObservation", BaseURL: upstream.URL, @@ -137,7 +141,8 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { }, }} - step, closeStep, err := mausamgram.New(context.Background(), registry, mapper, &mausamgram.Config{}) + step, closeStep, err := weather.New(context.Background(), registry, mapper, + &weather.Config{BindingKeys: []string{shippedBindingKey}}) if err != nil { t.Fatalf("failed to build the step: %v", err) } diff --git a/pkg/plugin/implementation/weather/prerequisites.go b/pkg/plugin/implementation/weather/prerequisites.go new file mode 100644 index 00000000..0c7b947a --- /dev/null +++ b/pkg/plugin/implementation/weather/prerequisites.go @@ -0,0 +1,24 @@ +package weather + +import "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" + +// prerequisites is what a weather capability needs that its payload does not +// carry, keyed by binding key. +// +// Empty, and that is the point: every weather capability so far is served by +// reading the payload, which the mapping does. An entry is needed only for real +// I/O -- a station id from a spatial lookup, a session token from an exchange -- +// because no expression language should be able to do those. +// +// Adding one is a function and a line here. Nothing else in the package moves, +// and capabilities that need nothing are untouched. +// +// Example, when a provider needs a station id: +// +// "imd-city|openagrinet:WeatherObservation": resolveStation, +// +// whose result the mapping then reads: +// +// request: | +// { "station": _local.stationId } +var prerequisites = upstream.Prerequisites{} diff --git a/pkg/plugin/implementation/weather/weather.go b/pkg/plugin/implementation/weather/weather.go new file mode 100644 index 00000000..b899e45b --- /dev/null +++ b/pkg/plugin/implementation/weather/weather.go @@ -0,0 +1,32 @@ +// Package weather serves the network's weather capabilities. +// +// One package per schema pack family, so which plugin owns a capability is +// readable from its binding key: openagrinet:WeatherObservation and +// openagrinet:WeatherAdvisory are weather's, openagrinet:MandiPrice is not. +// +// Almost nothing lives here. Recognising a capability, resolving the call plan, +// authenticating, calling with the registry's budget and translating in both +// directions are all internal/upstream's, because none of them differ by domain. +// What this package owns is its name, and prerequisites -- the work a mapping +// cannot express, which is domain knowledge by definition. +package weather + +import ( + "context" + + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" +) + +// Config is upstream's, unchanged. Aliased here so a domain plugin's cmd package +// need not know where the machinery lives. +type Config = upstream.Config + +// New creates the weather step. +// +// Which capabilities it answers to is configuration, with no default: a package +// serving a family cannot guess which of them a deployment has providers for. +func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper definition.Mapper, + cfg *Config) (definition.Step, func() error, error) { + return upstream.New(ctx, registry, mapper, prerequisites, cfg) +} From 395a701442112eb2b1108ddb0e4d0b9b3fc5211f Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 03:05:09 +0530 Subject: [PATCH 038/130] feat: let a deployment say where a binding key lives [#1] Where the two halves of a binding key sit in a payload was a typed struct, so a Beckn shape change meant editing Go, rebuilding and redeploying every adapter on the network at once. oanbinding.BecknV2 is that shape as data, and From walks it. A deployment can override both paths, which exists for one situation: the spec moves a field and someone needs to track it without waiting for a release. It is a DEFAULT, not a setting, and the distinction matters. Where a binding key lives is a network convention -- every participant has to agree, or two adapters disagree about what a binding key even is and requests silently fail to match, with no error anywhere to say why. So absent means correct, and overriding is something an operator types deliberately. Both halves or neither, refused at startup: overriding one and leaving the other on the convention matches nothing, and would do so silently on every request. The walk understands two things: dotted segments, and [] meaning "this is an array, look in each element". No wildcards, no filters, no indices. Each of those would be another way to write something subtly wrong in configuration nobody reviews, to buy an expressiveness no payload shape has needed. It is an escape hatch, not a query language -- and it is 40 lines rather than a dependency, which for an escape hatch is the right trade. From now takes the paths and walks a generic document rather than unmarshalling a typed one. Every existing test runs against BecknV2, which is the regression guard: the default has to give exactly the answers the typed walk gave. Verified live. With the paths pointed somewhere the payload does not use them, the request stops matching and falls out of the pipeline as a 404 -- the unanswered-request guard catching the consequence, which is what proves the override is genuinely in effect rather than ignored. --- .../internal/oanbinding/oanbinding.go | 51 ++---- .../internal/oanbinding/oanbinding_test.go | 153 +++++++++++++++++- .../internal/oanbinding/paths.go | 99 ++++++++++++ .../internal/upstream/upstream.go | 43 ++++- .../internal/upstream/upstream_test.go | 62 +++++++ .../implementation/weather/cmd/plugin.go | 16 +- .../implementation/weather/cmd/plugin_test.go | 31 ++++ 7 files changed, 409 insertions(+), 46 deletions(-) create mode 100644 pkg/plugin/implementation/internal/oanbinding/paths.go diff --git a/pkg/plugin/implementation/internal/oanbinding/oanbinding.go b/pkg/plugin/implementation/internal/oanbinding/oanbinding.go index cf8a43a7..0f148409 100644 --- a/pkg/plugin/implementation/internal/oanbinding/oanbinding.go +++ b/pkg/plugin/implementation/internal/oanbinding/oanbinding.go @@ -32,31 +32,6 @@ func (b Binding) Key() string { return b.ParticipantID + separator + b.CapabilityCode } -// selectPayload is the part of a Beckn v2 payload a binding is derived from. -// -// Both commitments and resources are arrays, and both are read as such. The -// provider is named once per commitment and the type once per resource, so a -// single request can in principle carry several -- see From for what happens -// when it does. -type selectPayload struct { - Message struct { - Contract struct { - Commitments []struct { - Offer struct { - Provider struct { - ID string `json:"id"` - } `json:"provider"` - } `json:"offer"` - Resources []struct { - ResourceAttributes struct { - Type string `json:"@type"` - } `json:"resourceAttributes"` - } `json:"resources"` - } `json:"commitments"` - } `json:"contract"` - } `json:"message"` -} - // From derives the capability binding a payload is asking for. // // Returns ErrNoBinding when the payload names no provider or no type, which is @@ -66,19 +41,17 @@ type selectPayload struct { // than resolved to its first: the two halves index one registry row describing // one upstream call, so a request spanning several is asking for something this // design cannot express. Guessing would silently serve part of it. -func From(body []byte) (Binding, error) { - var payload selectPayload +// +// Where the halves live is BecknV2 unless a deployment says otherwise -- see +// Paths for why that is a default and not a setting. +func From(paths Paths, body []byte) (Binding, error) { + var payload any if err := json.Unmarshal(body, &payload); err != nil { return Binding{}, fmt.Errorf("oanbinding: payload could not be read: %w", err) } - var providers, types []string - for _, commitment := range payload.Message.Contract.Commitments { - providers = appendDistinct(providers, commitment.Offer.Provider.ID) - for _, resource := range commitment.Resources { - types = appendDistinct(types, resource.ResourceAttributes.Type) - } - } + providers := distinct(valuesAt(payload, paths.ProviderID)) + types := distinct(valuesAt(payload, paths.CapabilityCode)) if len(providers) == 0 || len(types) == 0 { return Binding{}, ErrNoBinding @@ -95,6 +68,16 @@ func From(body []byte) (Binding, error) { return Binding{ParticipantID: providers[0], CapabilityCode: types[0]}, nil } +// distinct drops blanks and repeats, keeping the order they were found in so a +// refusal names them the way the payload did. +func distinct(values []string) []string { + var out []string + for _, value := range values { + out = appendDistinct(out, value) + } + return out +} + // appendDistinct adds value if it is neither empty nor already present. func appendDistinct(values []string, value string) []string { if value == "" { diff --git a/pkg/plugin/implementation/internal/oanbinding/oanbinding_test.go b/pkg/plugin/implementation/internal/oanbinding/oanbinding_test.go index d1f1ea41..09b60469 100644 --- a/pkg/plugin/implementation/internal/oanbinding/oanbinding_test.go +++ b/pkg/plugin/implementation/internal/oanbinding/oanbinding_test.go @@ -42,7 +42,7 @@ const realSelectPayload = `{ func TestFromReadsARealSelectPayload(t *testing.T) { t.Parallel() - got, err := From([]byte(realSelectPayload)) + got, err := From(BecknV2, []byte(realSelectPayload)) if err != nil { t.Fatalf("From() returned an unexpected error: %v", err) } @@ -79,7 +79,7 @@ func TestFromReportsAPayloadWithNoBinding(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - if _, err := From([]byte(tc.body)); !errors.Is(err, ErrNoBinding) { + if _, err := From(BecknV2, []byte(tc.body)); !errors.Is(err, ErrNoBinding) { t.Errorf("expected ErrNoBinding, got %v", err) } }) @@ -114,7 +114,7 @@ func TestFromRefusesAnAmbiguousPayload(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - _, err := From([]byte(tc.body)) + _, err := From(BecknV2, []byte(tc.body)) if err == nil { t.Fatal("expected an ambiguous payload to be refused") } @@ -137,7 +137,7 @@ func TestFromAcceptsRepetitionOfTheSameBinding(t *testing.T) { {"offer":{"provider":{"id":"p"}},"resources":[{"resourceAttributes":{"@type":"t"}}]}, {"offer":{"provider":{"id":"p"}},"resources":[{"resourceAttributes":{"@type":"t"}}]}]}}}` - got, err := From([]byte(body)) + got, err := From(BecknV2, []byte(body)) if err != nil { t.Fatalf("From() returned an unexpected error: %v", err) } @@ -149,7 +149,7 @@ func TestFromAcceptsRepetitionOfTheSameBinding(t *testing.T) { func TestFromReportsAnUnreadablePayload(t *testing.T) { t.Parallel() - _, err := From([]byte(`{"message":`)) + _, err := From(BecknV2, []byte(`{"message":`)) if err == nil { t.Fatal("expected unreadable JSON to be reported") } @@ -157,3 +157,146 @@ func TestFromReportsAnUnreadablePayload(t *testing.T) { t.Error("a broken payload is not an absent binding") } } + +// --- where the binding key lives --------------------------------------------- +// +// The two halves of a binding key sit at a fixed place in a Beckn v2 payload. +// That is a NETWORK convention: every participant must agree, or two adapters +// disagree about what a binding key even is and requests silently fail to match. +// +// So it is a default, not a setting. BecknV2 is what every deployment uses. An +// override exists only so a spec change can be tracked without waiting for a +// release, and it has to be typed deliberately -- absent means correct. + +func TestBecknV2IsTheDefault(t *testing.T) { + t.Parallel() + + if BecknV2.ProviderID == "" || BecknV2.CapabilityCode == "" { + t.Fatal("the default paths must be set") + } + got, err := From(BecknV2, []byte(realSelectPayload)) + if err != nil { + t.Fatalf("From() returned an unexpected error: %v", err) + } + if got.ParticipantID != "mausamgram" || got.CapabilityCode != "openagrinet:WeatherObservation" { + t.Errorf("binding = %+v, want the Beckn v2 convention's answer", got) + } +} + +// An override reads the halves from somewhere else entirely, which is what makes +// a spec change survivable without a build. +func TestFromReadsAnOverriddenPath(t *testing.T) { + t.Parallel() + + body := `{"who":{"provider":"agmarknet"},"what":[{"type":"openagrinet:MandiPrice"}]}` + got, err := From(Paths{ + ProviderID: "who.provider", + CapabilityCode: "what[].type", + }, []byte(body)) + if err != nil { + t.Fatalf("From() returned an unexpected error: %v", err) + } + if got.Key() != "agmarknet|openagrinet:MandiPrice" { + t.Errorf("binding key = %q, want it read from the overridden paths", got.Key()) + } +} + +// A path that matches nothing is a request this step is not meant to serve -- +// the ordinary case, and the same answer the typed walk gave. +func TestFromReportsNoBindingWhenAPathMatchesNothing(t *testing.T) { + t.Parallel() + + _, err := From(Paths{ProviderID: "nowhere.at.all", CapabilityCode: "what[].type"}, + []byte(`{"what":[{"type":"x"}]}`)) + if !errors.Is(err, ErrNoBinding) { + t.Errorf("expected ErrNoBinding, got %v", err) + } +} + +// Several distinct values stays a refusal whatever path found them: one request +// maps to one call, and guessing which would serve part of it silently. +func TestFromStillRefusesSeveralValuesUnderAnOverride(t *testing.T) { + t.Parallel() + + _, err := From(Paths{ProviderID: "who[].provider", CapabilityCode: "what[].type"}, + []byte(`{"who":[{"provider":"a"},{"provider":"b"}],"what":[{"type":"x"}]}`)) + if err == nil || errors.Is(err, ErrNoBinding) { + t.Errorf("expected a refusal naming both providers, got %v", err) + } +} + +// The walk is deliberately small: dotted segments, and [] to flatten an array. +// No wildcards, no filters, no indices -- it is an escape hatch, not a query +// language, and every one of those would be a way to write something subtly +// wrong in config nobody reviews. +func TestPathWalk(t *testing.T) { + t.Parallel() + + doc := map[string]any{ + "a": map[string]any{"b": "flat"}, + "list": []any{ + map[string]any{"v": "one"}, + map[string]any{"v": "two"}, + }, + "nested": []any{ + map[string]any{"inner": []any{map[string]any{"v": "deep"}}}, + }, + "number": 42, + } + + testCases := []struct { + name string + path string + want []string + }{ + {"a flat field", "a.b", []string{"flat"}}, + {"through an array", "list[].v", []string{"one", "two"}}, + {"through two arrays", "nested[].inner[].v", []string{"deep"}}, + {"a path that is not there", "a.missing", nil}, + {"a value that is not a string", "number", nil}, + {"an array not marked", "list.v", nil}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got := valuesAt(doc, tc.path) + if len(got) != len(tc.want) { + t.Fatalf("valuesAt(%q) = %v, want %v", tc.path, got, tc.want) + } + for i, want := range tc.want { + if got[i] != want { + t.Errorf("valuesAt(%q)[%d] = %q, want %q", tc.path, i, got[i], want) + } + } + }) + } +} + +// An override that names no path at all would match nothing and make every +// request unservable, silently. Refused where it is configured instead. +func TestPathsValidate(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + paths Paths + }{ + {"no provider path", Paths{CapabilityCode: "a.b"}}, + {"no capability path", Paths{ProviderID: "a.b"}}, + {"a blank segment", Paths{ProviderID: "a..b", CapabilityCode: "a.b"}}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + if err := tc.paths.Validate(); err == nil { + t.Error("expected an unusable path pair to be refused") + } + }) + } + + if err := BecknV2.Validate(); err != nil { + t.Errorf("the default paths must validate: %v", err) + } +} diff --git a/pkg/plugin/implementation/internal/oanbinding/paths.go b/pkg/plugin/implementation/internal/oanbinding/paths.go new file mode 100644 index 00000000..9120f7cb --- /dev/null +++ b/pkg/plugin/implementation/internal/oanbinding/paths.go @@ -0,0 +1,99 @@ +package oanbinding + +import ( + "fmt" + "strings" +) + +// Paths says where the two halves of a binding key live in a payload. +// +// This is a NETWORK convention, not a deployment's preference: every participant +// has to agree, or two adapters disagree about what a binding key is and +// requests silently fail to match. So BecknV2 is the answer, and overriding is +// something an operator has to type deliberately -- absent means correct. +// +// The override exists for one situation: the spec moves a field and a deployment +// needs to track it without waiting for a release. It is deliberately not +// something to reach for otherwise. +type Paths struct { + ProviderID string + CapabilityCode string +} + +// BecknV2 is where core-v2.0.0-lts puts them. +var BecknV2 = Paths{ + ProviderID: "message.contract.commitments[].offer.provider.id", + CapabilityCode: "message.contract.commitments[].resources[].resourceAttributes.@type", +} + +// arrayMarker flattens an array at that segment. It is the only operator the +// walk understands. +const arrayMarker = "[]" + +// Validate refuses a pair that could never match, so a mistake surfaces where it +// was configured rather than as every request quietly going unserved. +func (p Paths) Validate() error { + for name, path := range map[string]string{ + "providerIdAt": p.ProviderID, + "capabilityCodeAt": p.CapabilityCode, + } { + if strings.TrimSpace(path) == "" { + return fmt.Errorf("oanbinding: %s is empty", name) + } + for _, segment := range strings.Split(path, ".") { + if strings.TrimSpace(strings.TrimSuffix(segment, arrayMarker)) == "" { + return fmt.Errorf("oanbinding: %s (%q) has a blank segment", name, path) + } + } + } + return nil +} + +// valuesAt collects every string the path reaches. +// +// The grammar is two things: segments separated by ".", and a "[]" suffix +// meaning "this is an array, look in each element". No wildcards, no filters, no +// indices. Each of those would be another way to write something subtly wrong in +// config nobody reviews, to buy an expressiveness a payload shape has never +// needed. +func valuesAt(node any, path string) []string { + return walk(node, strings.Split(path, ".")) +} + +func walk(node any, segments []string) []string { + if len(segments) == 0 { + // The leaf. Only strings are binding-key material; a number or an + // object here means the path landed somewhere unintended. + if value, ok := node.(string); ok { + return []string{value} + } + return nil + } + + segment := segments[0] + rest := segments[1:] + + fields, ok := node.(map[string]any) + if !ok { + return nil + } + child, present := fields[strings.TrimSuffix(segment, arrayMarker)] + if !present { + return nil + } + + if !strings.HasSuffix(segment, arrayMarker) { + return walk(child, rest) + } + + // An array segment: every element contributes. + elements, ok := child.([]any) + if !ok { + return nil + } + var found []string + for _, element := range elements { + found = append(found, walk(element, rest)...) + } + return found +} diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index f5bec720..c50b5fb0 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -89,6 +89,18 @@ type Config struct { // comes from the registry, so one step serving several needs nothing else. BindingKeys []string `yaml:"bindingKeys" json:"bindingKeys"` + // ProviderIDAt and CapabilityCodeAt override where the two halves of a + // binding key sit in a payload. Absent means the Beckn v2 convention, which + // is what every deployment should be using. + // + // This is a network convention rather than a deployment's preference -- + // every participant must agree, or two adapters disagree about what a + // binding key is and requests silently fail to match. It is configurable + // only so that a spec change can be tracked without waiting for a release, + // and both must be given together. + ProviderIDAt string `yaml:"providerIdAt" json:"providerIdAt"` + CapabilityCodeAt string `yaml:"capabilityCodeAt" json:"capabilityCodeAt"` + // AuthScheme is how credentials are presented upstream: none, basic or // header. Providers differ here -- basic auth, a raw token header, a field // in the body -- which is why it is configuration and not an assumption. @@ -111,6 +123,7 @@ type Config struct { // Step serves the Mausamgram capability. It is safe for concurrent use. type Step struct { config *Config + paths oanbinding.Paths prerequisites Prerequisites registry definition.ProviderRecordLookup mapper definition.Mapper @@ -133,8 +146,14 @@ func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper d return nil, nil, err } + paths, err := bindingPaths(cfg) + if err != nil { + return nil, nil, err + } + step := &Step{ config: cfg, + paths: paths, prerequisites: prerequisites, registry: registry, mapper: mapper, @@ -153,6 +172,28 @@ func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper d return step, closer, nil } +// bindingPaths resolves where this step reads a binding key from. +// +// Both halves or neither: overriding one and leaving the other on the default +// is a half-configured deployment that would match nothing, and it would do so +// silently on every request rather than once at startup. +func bindingPaths(cfg *Config) (oanbinding.Paths, error) { + if cfg.ProviderIDAt == "" && cfg.CapabilityCodeAt == "" { + return oanbinding.BecknV2, nil + } + if cfg.ProviderIDAt == "" { + return oanbinding.Paths{}, errors.New("upstream: capabilityCodeAt is set without providerIdAt") + } + if cfg.CapabilityCodeAt == "" { + return oanbinding.Paths{}, errors.New("upstream: providerIdAt is set without capabilityCodeAt") + } + paths := oanbinding.Paths{ProviderID: cfg.ProviderIDAt, CapabilityCode: cfg.CapabilityCodeAt} + if err := paths.Validate(); err != nil { + return oanbinding.Paths{}, err + } + return paths, nil +} + // applyDefaults fills in what was left out and rejects what cannot be defaulted. func applyDefaults(cfg *Config) error { // No default. This package serves whatever a domain package configures it @@ -196,7 +237,7 @@ func applyDefaults(cfg *Config) error { // pipeline and each recognises its own work, so adding a provider is one more // entry rather than a change to a routing table. func (s *Step) Run(ctx *model.StepContext) error { - binding, err := oanbinding.From(ctx.Body) + binding, err := oanbinding.From(s.paths, ctx.Body) if errors.Is(err, oanbinding.ErrNoBinding) { return nil } diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go index 1bb47b2a..440fde11 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -14,6 +14,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/oanbinding" ) const selectBody = `{ @@ -452,6 +453,67 @@ func TestRunRefusesWhenTheResponseMappingProducesNothing(t *testing.T) { } } +// --- where the binding key lives ---------------------------------------------- +// +// A default, not a setting: every participant must agree where the halves of a +// binding key sit, or two adapters disagree about what a binding key is and +// requests silently fail to match. The override exists so a spec change can be +// tracked without waiting for a release, and has to be typed deliberately. + +func TestNewUsesTheBecknConventionByDefault(t *testing.T) { + t.Parallel() + + step := newStep(t, &stubRegistry{}, &stubMapper{}) + if step.paths != oanbinding.BecknV2 { + t.Errorf("paths = %+v, want the Beckn v2 convention", step.paths) + } +} + +// An override reads the halves from somewhere else, end to end through the step. +func TestRunReadsTheBindingKeyFromOverriddenPaths(t *testing.T) { + t.Parallel() + + upstreamServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{}`) + })) + defer upstreamServer.Close() + + plan := testPlan(upstreamServer.URL, http.MethodGet) + plan.BindingKey = "agmarknet|openagrinet:MandiPrice" + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + + step := newStep(t, &stubRegistry{plan: plan}, mapper, func(c *Config) { + c.BindingKeys = []string{"agmarknet|openagrinet:MandiPrice"} + c.ProviderIDAt = "who.provider" + c.CapabilityCodeAt = "what[].type" + }) + + ctx, err := runStep(t, step, `{"context":{"action":"select"},"who":{"provider":"agmarknet"},"what":[{"type":"openagrinet:MandiPrice"}]}`) + if err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + if len(ctx.ResponseBody) == 0 { + t.Error("the step did not recognise a binding key at the overridden paths") + } +} + +// Overriding one half and not the other is a half-configured deployment that +// would match nothing. Refused at startup rather than at every request. +func TestNewRefusesAHalfConfiguredOverride(t *testing.T) { + t.Parallel() + + _, _, err := New(context.Background(), &stubRegistry{}, &stubMapper{}, nil, &Config{ + BindingKeys: []string{testBindingKey}, + ProviderIDAt: "who.provider", + }) + if err == nil { + t.Fatal("expected one path without the other to be refused") + } + if !strings.Contains(err.Error(), "capabilityCodeAt") { + t.Errorf("error %q should name the path that is missing", err) + } +} + // --- several capabilities, one step ------------------------------------------- // // A provider can serve more than one capability -- the registry contract says so diff --git a/pkg/plugin/implementation/weather/cmd/plugin.go b/pkg/plugin/implementation/weather/cmd/plugin.go index f5462650..4bb82112 100644 --- a/pkg/plugin/implementation/weather/cmd/plugin.go +++ b/pkg/plugin/implementation/weather/cmd/plugin.go @@ -23,12 +23,16 @@ var newStepFunc = weather.New // scheme, so those rules live in one place. func (p weatherProvider) parseConfig(config map[string]string) (*weather.Config, error) { cfg := &weather.Config{ - BindingKeys: splitList(config["bindingKeys"]), - AuthScheme: config["authScheme"], - UsernameEnv: config["usernameEnv"], - PasswordEnv: config["passwordEnv"], - HeaderName: config["headerName"], - HeaderValueEnv: config["headerValueEnv"], + BindingKeys: splitList(config["bindingKeys"]), + // Absent means the Beckn v2 convention. See upstream.Config for why + // this is a default rather than something to set. + ProviderIDAt: config["providerIdAt"], + CapabilityCodeAt: config["capabilityCodeAt"], + AuthScheme: config["authScheme"], + UsernameEnv: config["usernameEnv"], + PasswordEnv: config["passwordEnv"], + HeaderName: config["headerName"], + HeaderValueEnv: config["headerValueEnv"], } if raw, exists := config["maxResponseBytes"]; exists && raw != "" { diff --git a/pkg/plugin/implementation/weather/cmd/plugin_test.go b/pkg/plugin/implementation/weather/cmd/plugin_test.go index 86c9a7c5..e04b9191 100644 --- a/pkg/plugin/implementation/weather/cmd/plugin_test.go +++ b/pkg/plugin/implementation/weather/cmd/plugin_test.go @@ -124,6 +124,37 @@ func TestParseConfigReadsSeveralBindingKeys(t *testing.T) { } } +// The override is two keys, both or neither. Absent leaves the step on the +// Beckn v2 convention, which is what every deployment should be running. +func TestParseConfigReadsTheBindingKeyOverride(t *testing.T) { + t.Parallel() + + cfg, err := weatherProvider{}.parseConfig(map[string]string{ + "bindingKeys": "a|openagrinet:One", + "providerIdAt": "who.provider", + "capabilityCodeAt": "what[].type", + }) + if err != nil { + t.Fatalf("parseConfig() returned an unexpected error: %v", err) + } + if cfg.ProviderIDAt != "who.provider" || cfg.CapabilityCodeAt != "what[].type" { + t.Errorf("override = %q / %q, want the configured paths", cfg.ProviderIDAt, cfg.CapabilityCodeAt) + } +} + +// Absent leaves them empty, and upstream reads that as "use the convention". +func TestParseConfigLeavesTheOverrideUnsetByDefault(t *testing.T) { + t.Parallel() + + cfg, err := weatherProvider{}.parseConfig(map[string]string{"bindingKeys": "a|openagrinet:One"}) + if err != nil { + t.Fatalf("parseConfig() returned an unexpected error: %v", err) + } + if cfg.ProviderIDAt != "" || cfg.CapabilityCodeAt != "" { + t.Errorf("override = %q / %q, want both empty", cfg.ProviderIDAt, cfg.CapabilityCodeAt) + } +} + func TestNew(t *testing.T) { t.Parallel() From 92cd52984ea331320b1c3533862c586c245ca5e1 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 12:15:38 +0530 Subject: [PATCH 039/130] fix: quote one resource per forecast day [#1] The answer carried one resource holding every day under an "observations" field of its own invention. The WeatherObservation pack has no form for that: each of its examples carries a single validity and a flat parameters array, so a period is a resource. Minting a resource per day was tried before and reverted because it left the offer dangling -- the offer is echoed from the request, so its resourceIds still named the id that was asked for while the resources carried freshly derived ones. Rewriting those references is what makes the split safe, so the offer is $merge-ed rather than echoed and points at the days actually returned. Ids derive from the forecast date. They are new by design: the request names an abstract point forecast, the answer returns the concrete days that satisfy it. --- .../weather-observation.select.yaml | 99 ++++++++------- .../implementation/weather/mappings_test.go | 118 +++++++++--------- 2 files changed, 116 insertions(+), 101 deletions(-) diff --git a/config/mappings/mausamgram/weather-observation.select.yaml b/config/mappings/mausamgram/weather-observation.select.yaml index 6170a2c6..ca8fb12f 100644 --- a/config/mappings/mausamgram/weather-observation.select.yaml +++ b/config/mappings/mausamgram/weather-observation.select.yaml @@ -34,24 +34,20 @@ # resource advertising this capability is OnDemand instead, and carries # supportedParameters rather than values. # -# The answer quotes ONE resource, carrying the id the request selected. The -# forecast days are content of that resource, under observations. -# -# Minting a resource per day was the earlier shape, and it broke the offer: the -# offer is echoed from the request, so its resourceIds still pointed at the id -# the consumer selected while the resources carried freshly invented ones. -# Nothing in the answer resolved that reference. Keeping the selected id is what -# makes it stay true. -# -# TWO FIELDS HERE ARE NOT IN THE PACK, and both are deliberate. The pack sets no -# additionalProperties, so they validate; they are simply not governed by it. -# -# observations The pack carries one validity and one flat parameters array -# per resource, so it cannot express a five-day forecast in the -# one resource the request selected. Its own examples show one -# resource per period. Keeping the selected id was judged worth -# more than splitting into five resources the consumer never -# asked for. Revisit if the packs gain a series field. +# The answer returns ONE RESOURCE PER FORECAST DAY, each with its own id derived +# from its date. That is the shape the pack describes: every WeatherObservation +# example carries a single validity and a flat parameters array, so a period is a +# resource and there is no form for several in one. +# +# The ids are therefore new -- the request named an abstract point forecast, the +# answer returns the concrete days that satisfy it. Which is why the offer's +# resourceIds are rewritten below rather than echoed: the offer arrives naming +# the id that was asked for, and leaving it would point the offer at something +# that appears nowhere in the answer. +# +# ONE FIELD HERE IS NOT IN THE PACK, deliberately. The pack sets no +# additionalProperties, so it validates; it is simply not governed. +# # aggregation The pack's parameter entry is parameter/value/unit only. This # provider reports a minimum AND a maximum for temperature and # humidity, which are indistinguishable without it. @@ -162,6 +158,11 @@ response: | $selected := beckn.message.contract.commitments[0]; + /* Bound once because it is used twice -- for a resource's own id and for the + offer's reference to it. Two copies of the same expression is how a + dangling reference gets reintroduced. */ + $resourceId := function($day) { "res:mausamgram:forecast:" & $day.date }; + { "context": { "version": beckn.context.version, @@ -182,10 +183,29 @@ response: | "status": { "descriptor": { "code": "QUOTED", "name": "Quoted" } }, - "offer": $selected.offer, - "resources": [ + /* The offer is echoed, but its references are not: the request + named the abstract point forecast, and the answer returns the + concrete days. Leaving resourceIds as they arrived would point + the offer at an id that appears nowhere in the answer. + + $merge keeps everything else the request offered -- the id, the + descriptor, the provider -- and replaces one key. */ + "offer": $merge([ + $selected.offer, + { "resourceIds": [$map($days, function($day) { $resourceId($day) })] } + ]), + /* One resource per forecast day, which is what the pack describes: + every WeatherObservation example carries a single validity and a + flat parameters array, so a period is a resource and there is no + form for several in one. + + Wrapped for the same reason as the resourceIds above: JSONata + collapses a one-element sequence to a bare value, so a one-day + forecast would answer with an object where every other N answers + with a list. */ + "resources": [$map($days, function($day) { { - "id": $selected.resources[0].id, + "id": $resourceId($day), "resourceAttributes": { "@context": "https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld", "@type": "openagrinet:WeatherObservation", @@ -201,33 +221,24 @@ response: | "coordinates": [$lon, $lat] }, "generatedAt": $now(), + /* This resource reports one day, so its validity opens and + closes on it. */ "validity": { - "startsAt": $days[0].date, - "endsAt": $days[-1].date + "startsAt": $day.date, + "endsAt": $day.date }, - /* Wrapped: JSONata collapses a one-element sequence to a - bare value, so a single-day forecast would answer with an - object where every other N answers with a list. */ - "observations": [$map($days, function($day) { - { - "validity": { - "startsAt": $day.date, - "endsAt": $day.date - }, - "parameters": [ - $reading("Rainfall", "Total", "mm", $day.rain), - $reading("Temperature", "Minimum", "Cel", $day.tmin), - $reading("Temperature", "Maximum", "Cel", $day.tmax), - $reading("Humidity", "Minimum", "%", $day.rhmin), - $reading("Humidity", "Maximum", "%", $day.rhmax), - $reading("WindSpeed", "Average", "m/s", $day.wspd), - $alert($day.weather_warning ? $day.weather_warning : $day.cloud_message) - ] - } - })] + "parameters": [ + $reading("Rainfall", "Total", "mm", $day.rain), + $reading("Temperature", "Minimum", "Cel", $day.tmin), + $reading("Temperature", "Maximum", "Cel", $day.tmax), + $reading("Humidity", "Minimum", "%", $day.rhmin), + $reading("Humidity", "Maximum", "%", $day.rhmax), + $reading("WindSpeed", "Average", "m/s", $day.wspd), + $alert($day.weather_warning ? $day.weather_warning : $day.cloud_message) + ] } } - ] + })] } ] } diff --git a/pkg/plugin/implementation/weather/mappings_test.go b/pkg/plugin/implementation/weather/mappings_test.go index 5c54ad79..4eacd95e 100644 --- a/pkg/plugin/implementation/weather/mappings_test.go +++ b/pkg/plugin/implementation/weather/mappings_test.go @@ -16,6 +16,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "slices" "strings" "testing" @@ -194,36 +195,56 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { t.Error("the quoted commitment carries no offer") } - // One resource, carrying the id the request selected. The consumer asked for - // a quote on one resource, so that is what is quoted -- the forecast days are - // content of it, not resources of their own. + // One resource per forecast day, each with its own id. That is what the + // openagrinet:WeatherObservation pack describes -- every one of its examples + // carries a single validity and a flat parameters array, so a period is a + // resource and there is no form for several in one. // - // Minting a resource per day would leave offer.resourceIds pointing at an id - // that appears nowhere in the answer, because the offer is echoed from the - // request. Keeping the id is what makes that reference stay true. + // The ids are new: the consumer selected an abstract point forecast and gets + // back the concrete days that answer it. Which means the offer's references + // have to be rewritten, because the offer is echoed from the request and its + // resourceIds still name the id that was asked for. Leaving them was the + // dangling reference this file used to carry. resources, _ := commitment["resources"].([]any) - if len(resources) != 1 { - t.Fatalf("got %d resources, want 1 -- the one the request selected", len(resources)) + if len(resources) != 3 { + t.Fatalf("got %d resources, want 3 -- one per day the provider answered with", len(resources)) } - only, _ := resources[0].(map[string]any) - if only["id"] != selectedResourceID { - t.Errorf("resource id = %v, want the requested %q", only["id"], selectedResourceID) + returned := make([]string, 0, len(resources)) + for _, entry := range resources { + resource, _ := entry.(map[string]any) + id, _ := resource["id"].(string) + if !strings.HasPrefix(id, "res:mausamgram:forecast:") { + t.Errorf("resource id = %q, want one derived from the forecast date", id) + } + returned = append(returned, id) } - // The offer's references must resolve against the resources actually - // returned. This is the assertion the previous shape could not satisfy. + // The offer must reference the resources actually returned, not the one that + // was asked for. This is the assertion that fails the moment the offer is + // echoed unchanged. offer, _ := commitment["offer"].(map[string]any) referenced, _ := offer["resourceIds"].([]any) - if len(referenced) != 1 || referenced[0] != selectedResourceID { - t.Errorf("offer.resourceIds = %v, want exactly [%q]", referenced, selectedResourceID) + if len(referenced) != len(returned) { + t.Fatalf("offer.resourceIds has %d entries, want %d -- one per resource returned", + len(referenced), len(returned)) + } + for _, reference := range referenced { + if !slices.Contains(returned, reference.(string)) { + t.Errorf("offer references %v, which is not among the resources returned", reference) + } + } + // And the descriptor the request offered is still there: only the references + // are rewritten, not the offer. + if offer["id"] != "offer:mausamgram:open-data" { + t.Errorf("offer id = %v, want the one the request offered", offer["id"]) } // --- the WeatherObservation schema pack, Direct mode --------------------- // openagrinet:WeatherObservation v0.1 requires all five of these when - // informationMode is Direct. Two of them were missing before the pack was - // read: generatedAt, and a validity for the resource as a whole. - attributes, _ := only["resourceAttributes"].(map[string]any) + // informationMode is Direct, and each resource is one Direct observation. + first, _ := resources[0].(map[string]any) + attributes, _ := first["resourceAttributes"].(map[string]any) for _, f := range []struct{ key, want string }{ {"@type", "openagrinet:WeatherObservation"}, {"informationMode", "Direct"}, @@ -233,15 +254,12 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { t.Errorf("%s = %v, want %v", f.key, attributes[f.key], f.want) } } - for _, required := range []string{"source", "location", "generatedAt", "validity", "observations"} { + for _, required := range []string{"source", "location", "generatedAt", "validity", "parameters"} { if attributes[required] == nil { t.Errorf("resourceAttributes carries no %q", required) } } - // The point, the source and the observation type are the same for every day, - // so they sit once at the top rather than being repeated per day. - // // GeoJSON order, and the provider's own echo of the point: the mapping reads // response.location rather than anything the step resolved. location, _ := attributes["location"].(map[string]any) @@ -250,30 +268,13 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { t.Errorf("coordinates = %v, want [73.7898, 19.9975] in GeoJSON order", coordinates) } - // The resource-level validity spans the whole forecast, first day to last. + // This resource covers one day, so its validity opens and closes on it. validity, _ := attributes["validity"].(map[string]any) - if validity["startsAt"] != "2026-08-26" || validity["endsAt"] != "2026-08-28" { - t.Errorf("validity = %v, want the span of the days the provider answered with", validity) - } - - // --- the days ------------------------------------------------------------ - // observations is NOT a pack field. The pack carries one validity and one - // flat parameters array per resource, so it cannot express a multi-day - // forecast in the resource the request selected. Keeping the selected id - // matters more, so the days go in an extra field -- which validates, because - // the pack sets no additionalProperties, but is not governed by it. - observations, _ := attributes["observations"].([]any) - if len(observations) != 3 { - t.Fatalf("got %d observations, want 3 -- one per day the provider answered with", len(observations)) - } - - first, _ := observations[0].(map[string]any) - dayValidity, _ := first["validity"].(map[string]any) - if dayValidity["startsAt"] != "2026-08-26" { - t.Errorf("first observation starts at %v, want the provider's first forecast date", dayValidity["startsAt"]) + if validity["startsAt"] != "2026-08-26" || validity["endsAt"] != "2026-08-26" { + t.Errorf("validity = %v, want the single day this resource reports", validity) } - parameters, _ := first["parameters"].([]any) + parameters, _ := attributes["parameters"].([]any) if len(parameters) != 7 { t.Errorf("got %d parameters, want 7 for a fully-reported day with a warning", len(parameters)) } @@ -290,8 +291,9 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { // Readings it did not take are absent, not present and empty: a consumer // must be able to tell "no rainfall recorded" from "zero rainfall". A day // with no warning carries no Alert parameter at all. - third, _ := observations[2].(map[string]any) - thirdParameters, _ := third["parameters"].([]any) + third, _ := resources[2].(map[string]any) + thirdAttributes, _ := third["resourceAttributes"].(map[string]any) + thirdParameters, _ := thirdAttributes["parameters"].([]any) if len(thirdParameters) != 2 { t.Errorf("got %d parameters for a partly-reported day, want only the 2 taken", len(thirdParameters)) } @@ -460,28 +462,30 @@ func TestShippedMappingsTakeHoweverManyDaysTheProviderSent(t *testing.T) { if err := json.Unmarshal(got, &answer); err != nil { t.Fatalf("failed to decode the answer: %v", err) } - attributes := firstCommitment(t, answer)["resources"].([]any)[0].(map[string]any)["resourceAttributes"].(map[string]any) - observations, _ := attributes["observations"].([]any) + commitment := firstCommitment(t, answer) + resources, _ := commitment["resources"].([]any) - if len(observations) != days { - t.Fatalf("got %d observations, want %d -- the mapping is not reading the provider's own count", - len(observations), days) + if len(resources) != days { + t.Fatalf("got %d resources, want %d -- the mapping is not reading the provider's own count", + len(resources), days) } // In the provider's order, not the keys' lexical order. - for i, entry := range observations { - validity := entry.(map[string]any)["validity"].(map[string]any) + for i, entry := range resources { + attributes := entry.(map[string]any)["resourceAttributes"].(map[string]any) + validity := attributes["validity"].(map[string]any) want := fmt.Sprintf("2026-09-%02d", i+1) if validity["startsAt"] != want { - t.Errorf("observation %d covers %v, want %s -- days are out of order", + t.Errorf("resource %d covers %v, want %s -- days are out of order", i, validity["startsAt"], want) } } - // The resource-level window still spans first to last. - window, _ := attributes["validity"].(map[string]any) - if window["endsAt"] != fmt.Sprintf("2026-09-%02d", days) { - t.Errorf("validity ends at %v, want the last day the provider sent", window["endsAt"]) + // However many resources there are, the offer references all of them. + offer, _ := commitment["offer"].(map[string]any) + referenced, _ := offer["resourceIds"].([]any) + if len(referenced) != days { + t.Errorf("offer references %d resources, want %d", len(referenced), days) } }) } From b7187c63c72d5bd2af188ca574f540e9b86129f8 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 12:15:38 +0530 Subject: [PATCH 040/130] fix: refuse a published path with an empty segment [#1] The registry accepted "//get-daily" and the step joined it verbatim, so a typo in a registry row surfaced as a provider 404 three hops away with nothing naming the cause. The registry schema is tightened alongside this, but it is a separate deployable that may not be updated in step, so a row that slipped through has to fail here with something an operator can act on. A trailing slash is deliberately still accepted: "/api/" and "/api" are a distinction some APIs genuinely make, so stripping it would silently change the URL the operator published. An empty segment is the only case that is never deliberate. --- .../internal/upstream/upstream.go | 34 ++++++++ .../internal/upstream/upstream_test.go | 85 +++++++++++++++++++ 2 files changed, 119 insertions(+) diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index c50b5fb0..b64a1d4a 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -510,6 +510,14 @@ func (s *Step) authenticate(req *http.Request) error { // buildEndpoint joins the plan's base URL and path, carrying the mapped request // as query parameters when the method takes no body. func buildEndpoint(baseURL string, call model.ActionPlan, mapped []byte) (string, error) { + if err := verifyPath(call.Path); err != nil { + return "", err + } + + // baseUrl cannot end in a slash and path must begin with one, so exactly one + // separator appears between them. The trim is belt and braces: the registry + // refuses a trailing slash on baseUrl, and this keeps a row that predates + // that from producing a doubled one. endpoint := strings.TrimSuffix(baseURL, "/") + call.Path if hasBody(call.Method) { return endpoint, nil @@ -528,6 +536,32 @@ func buildEndpoint(baseURL string, call model.ActionPlan, mapped []byte) (string return endpoint + "?" + query, nil } +// verifyPath refuses a published path nobody could have meant. +// +// The registry constrains this, but it is a separate deployable that may not be +// updated in step, so a row that slipped through has to fail here with something +// an operator can act on rather than as a provider's 404 three hops away. +// +// An empty segment is the case worth catching: "//get-daily" is never +// deliberate, and plenty of servers answer it differently from "/get-daily". A +// trailing slash is deliberately left alone -- "/api/" and "/api" are a +// distinction some APIs genuinely make, so stripping it would silently change +// the URL the operator published. +func verifyPath(path string) error { + if path == "" { + return model.NewBadReqErr("", errors.New("upstream: the registry publishes no path for this action")) + } + if !strings.HasPrefix(path, "/") { + return model.NewBadReqErr("", fmt.Errorf( + "upstream: path %q does not begin with a slash, so it cannot be joined to a base url", path)) + } + if strings.Contains(path, "//") { + return model.NewBadReqErr("", fmt.Errorf( + "upstream: path %q has an empty segment; write it with single slashes", path)) + } + return nil +} + // asQuery renders a mapped request as query parameters. // // A method with no body still needs the mapping's output somewhere, and the diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go index 440fde11..bf656007 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -453,6 +453,91 @@ func TestRunRefusesWhenTheResponseMappingProducesNothing(t *testing.T) { } } +// --- the endpoint the registry publishes ------------------------------------- +// +// baseUrl and path are joined and sent. The registry constrains both, but it is +// a separate deployable that may not be updated in step, so a row that slipped +// through has to fail here with something an operator can act on rather than as +// a provider's 404 three hops away. +// +// An EMPTY SEGMENT is the case worth catching: "//get-daily" is never what +// anyone meant, and many servers answer it differently from "/get-daily". A +// TRAILING slash is left alone deliberately -- "/api/" and "/api" are a +// distinction some APIs genuinely make, so silently stripping it would change +// the URL the operator asked for. +func TestRunRefusesANonCanonicalPath(t *testing.T) { + t.Parallel() + + for _, path := range []string{"//get-daily", "/v1//get-daily", "/get-daily//"} { + t.Run(path, func(t *testing.T) { + t.Parallel() + + upstreamServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Error("the provider must not be called with a path nobody meant") + })) + defer upstreamServer.Close() + + plan := testPlan(upstreamServer.URL, http.MethodGet) + plan.Actions["select"] = model.ActionPlan{Method: http.MethodGet, Path: path, Mappings: testMappingRef} + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + + _, err := runStep(t, newStep(t, &stubRegistry{plan: plan}, mapper), selectBody) + if err == nil { + t.Fatal("expected a path with an empty segment to be refused") + } + // Naming the path and the binding key is what makes this a one-line + // fix in the registry rather than a hunt. + if !strings.Contains(err.Error(), path) { + t.Errorf("error %q should name the path that is wrong", err) + } + }) + } +} + +// A trailing slash is meaningful, so it goes through untouched. +func TestRunKeepsATrailingSlashOnThePath(t *testing.T) { + t.Parallel() + + var gotPath string + upstreamServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + fmt.Fprint(w, `{}`) + })) + defer upstreamServer.Close() + + plan := testPlan(upstreamServer.URL, http.MethodGet) + plan.Actions["select"] = model.ActionPlan{Method: http.MethodGet, Path: "/get-daily/", Mappings: testMappingRef} + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + + if _, err := runStep(t, newStep(t, &stubRegistry{plan: plan}, mapper), selectBody); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + if gotPath != "/get-daily/" { + t.Errorf("provider was called at %q, want the path exactly as published", gotPath) + } +} + +// The join itself: baseUrl cannot end in a slash and path must begin with one, +// so exactly one separator appears between them. Asserted so a change to either +// side cannot quietly produce a doubled or missing slash. +func TestBuildEndpointJoinsWithOneSlash(t *testing.T) { + t.Parallel() + + for _, tc := range []struct{ base, path, want string }{ + {"http://host:9100", "/get-daily", "http://host:9100/get-daily"}, + {"http://host:9100/api", "/get-daily", "http://host:9100/api/get-daily"}, + {"http://host:9100/", "/get-daily", "http://host:9100/get-daily"}, + } { + got, err := buildEndpoint(tc.base, model.ActionPlan{Method: http.MethodPost, Path: tc.path}, nil) + if err != nil { + t.Fatalf("buildEndpoint(%q, %q) returned an unexpected error: %v", tc.base, tc.path, err) + } + if got != tc.want { + t.Errorf("buildEndpoint(%q, %q) = %q, want %q", tc.base, tc.path, got, tc.want) + } + } +} + // --- where the binding key lives ---------------------------------------------- // // A default, not a setting: every participant must agree where the halves of a From 52768f00c838c15ba37ef132201c3adb64ccaba7 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 12:47:06 +0530 Subject: [PATCH 041/130] fix: stop asserting network identity from a mapping [#1] The response context echoed bapId, bapUri, bppId and bppUri straight from the request. A mapping transforms a payload; asserting who the parties are is not its job, and the two Uri fields were only ever whatever the caller happened to send -- in a deployed stack a container-internal address that means nothing outside that network, republished as though it were ours. Identity on an answer is the signature the adapter puts on it, using the key the registry publishes for it. Nothing downstream reads these four from a response, and the provider path runs no response schema validation, so the shorter context changes nothing but what is claimed. The context keeps what correlates the answer to the request: version, action, networkId, transactionId, messageId and a fresh timestamp. The test asserted the echo; it now asserts the absence. --- .../mausamgram/weather-observation.select.yaml | 16 ++++++++++++---- .../implementation/weather/mappings_test.go | 11 +++++++++-- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/config/mappings/mausamgram/weather-observation.select.yaml b/config/mappings/mausamgram/weather-observation.select.yaml index ca8fb12f..e63b014f 100644 --- a/config/mappings/mausamgram/weather-observation.select.yaml +++ b/config/mappings/mausamgram/weather-observation.select.yaml @@ -164,14 +164,22 @@ response: | $resourceId := function($day) { "res:mausamgram:forecast:" & $day.date }; { + /* Correlation only: the ids that tie this answer to the request that + asked for it, and nothing that asserts who anybody is. + + bapId, bapUri, bppId and bppUri are deliberately absent. A mapping is + a payload transformation -- it has no business asserting network + identity, and the two Uri fields it could copy are whatever the caller + happened to send, which in a deployed stack is a container-internal + address that means nothing to anyone outside it. Echoing them would + republish another party's routing details as if they were ours. + + Identity on the wire is the adapter's own: it signs what it answers + with, using the key the registry publishes for it. */ "context": { "version": beckn.context.version, "action": "on_select", "networkId": beckn.context.networkId, - "bapId": beckn.context.bapId, - "bapUri": beckn.context.bapUri, - "bppId": beckn.context.bppId, - "bppUri": beckn.context.bppUri, "transactionId": beckn.context.transactionId, "messageId": beckn.context.messageId, "timestamp": $now() diff --git a/pkg/plugin/implementation/weather/mappings_test.go b/pkg/plugin/implementation/weather/mappings_test.go index 4eacd95e..ece33d06 100644 --- a/pkg/plugin/implementation/weather/mappings_test.go +++ b/pkg/plugin/implementation/weather/mappings_test.go @@ -181,8 +181,15 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { if beckncontext["transactionId"] != "9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44" { t.Errorf("transactionId = %v, want the one from the request", beckncontext["transactionId"]) } - if beckncontext["bppId"] != "provider-network-vistaar.da.gov.in" { - t.Errorf("bppId = %v, want the one from the request", beckncontext["bppId"]) + // A mapping transforms a payload; it does not assert who anyone is. The two + // Uri fields in particular are only whatever the caller sent -- in a + // deployed stack a container-internal address -- so echoing them would + // republish another party's routing details as ours. The adapter signs what + // it answers with instead, and that signature is what carries identity. + for _, field := range []string{"bapId", "bapUri", "bppId", "bppUri"} { + if _, present := beckncontext[field]; present { + t.Errorf("response context carries %q; a mapping must not assert identity", field) + } } commitment := firstCommitment(t, answer) From ff2c61d4cfb0cabc73b4d192cb307c979b511bd7 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 12:57:46 +0530 Subject: [PATCH 042/130] test: pin the registry shape after the key and role changes [#1] The Participant schema dropped keyId and use from a published key, stopped prefixing key material with "base64:", and replaced the BAP/BPP/NETWORK role enum with consumer/provider/network. This captures a real response from a registry running the new schema, so the plugin is pinned against what a registry writes today rather than only what one wrote in August. Two things it proves that were previously incidental rather than tested: a key with no encoding label arrives at the verifier byte-for-byte, and a key with no "use" still resolves -- had an absent use been read as "unknown, so refuse", every key the registry now writes would be unusable. The older capture stays exactly as it is. The registry is append-only, so a row written before this change keeps keyId, use and the label forever, and the plugin has to go on reading both shapes. --- .../oanregistry/oanregistry_test.go | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/pkg/plugin/implementation/oanregistry/oanregistry_test.go b/pkg/plugin/implementation/oanregistry/oanregistry_test.go index 5acb47a6..bc6f438a 100644 --- a/pkg/plugin/implementation/oanregistry/oanregistry_test.go +++ b/pkg/plugin/implementation/oanregistry/oanregistry_test.go @@ -1481,6 +1481,115 @@ func TestLookupAgainstCapturedRegistryResponse(t *testing.T) { } } +// TestLookupAgainstCurrentRegistryResponse runs the plugin against a verbatim +// response captured from an OAN registry on 2 Sep 2026, after the Participant +// schema dropped three things from a published key. +// +// It pins the shape a registry writes TODAY, and every difference from the +// capture above is deliberate: +// +// role one of consumer, provider and network. The Beckn acronyms are +// gone; a role now says what a party does. +// keyId absent. Nothing could look one up: the registry assigns an +// osid on write, and that is what a sender names in the +// Authorization header, so the friendly id was decoration. +// use absent. alg carries the purpose -- ed25519 signs -- and the +// plugin already treats a missing use as "may sign". +// key bare base64, no "base64:" label. The label is still tolerated +// by the test above, because a row written before this change +// keeps it forever: the registry is append-only. +func TestLookupAgainstCurrentRegistryResponse(t *testing.T) { + t.Parallel() + + const ( + capturedParticipantID = "provider.oan.dev" + capturedKeyOSID = "1-d5b6c5ee-206c-4529-bf9d-803138ff067a" + capturedKey = "Hcmx3AEVSeHT+1J3ggqhzlbTTtTYP0tQ2eUfotR5lUI=" + capturedURL = "https://provider.oan.dev/beckn" + ) + + const captured = `{ + "totalCount": 1, + "data": [ + { + "osUpdatedAt": "2026-09-02T07:25:29.977Z", + "role": "provider", + "osUpdatedBy": "1e52cfea-50a1-4a64-814e-0d44aaa38c29", + "osid": "1-686a5071-b300-4899-94e2-7f95155ca41d", + "type": "node", + "keys": [ + { + "osUpdatedAt": "2026-09-02T07:25:29.977Z", + "osCreatedAt": "2026-09-02T07:25:29.977Z", + "osUpdatedBy": "1e52cfea-50a1-4a64-814e-0d44aaa38c29", + "osCreatedBy": "1e52cfea-50a1-4a64-814e-0d44aaa38c29", + "validUntil": "2030-01-01T00:00:00Z", + "osid": "1-d5b6c5ee-206c-4529-bf9d-803138ff067a", + "validFrom": "2026-01-01T00:00:00Z", + "alg": "ed25519", + "key": "Hcmx3AEVSeHT+1J3ggqhzlbTTtTYP0tQ2eUfotR5lUI=", + "status": "active" + } + ], + "osOwner": [ + "1e52cfea-50a1-4a64-814e-0d44aaa38c29" + ], + "participantId": "provider.oan.dev", + "baseUrl": "https://provider.oan.dev/beckn", + "osCreatedAt": "2026-09-02T07:25:29.977Z", + "name": "OAN provider layer adapter", + "osCreatedBy": "1e52cfea-50a1-4a64-814e-0d44aaa38c29", + "status": "active" + } + ] +}` + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, captured) + })) + defer srv.Close() + + client := newTestClient(t, srv.URL, nil) + results, err := client.Lookup(context.Background(), &model.Subscription{ + Subscriber: model.Subscriber{SubscriberID: capturedParticipantID}, + KeyID: capturedKeyOSID, + }) + if err != nil { + t.Fatalf("Lookup() returned an unexpected error: %v", err) + } + if len(results) != 1 { + t.Fatalf("expected exactly 1 result, got %d", len(results)) + } + + got := results[0] + // The point of the whole test: a key with no encoding label arrives intact. + // Trimming a prefix that is not there must not disturb the value, because + // what reaches the verifier goes straight to a base64 decoder. + if got.SigningPublicKey != capturedKey { + t.Errorf("signing key = %q, want the bare base64 %q", got.SigningPublicKey, capturedKey) + } + // A key with no "use" still resolves. Were that treated as "unknown, so + // refuse", every key this registry now writes would be unusable. + if !model.IsKeyStatusUsable(got.Status) { + t.Errorf("an active participant with an active key must be usable, got status %q", got.Status) + } + if got.KeyID != capturedKeyOSID { + t.Errorf("key id = %q, want the key's osid %q", got.KeyID, capturedKeyOSID) + } + if got.SubscriberID != capturedParticipantID { + t.Errorf("subscriber id = %q, want %q", got.SubscriberID, capturedParticipantID) + } + if got.URL != capturedURL { + t.Errorf("endpoint url = %q, want the captured baseUrl %q", got.URL, capturedURL) + } + if got.Type != "provider" { + t.Errorf("role = %q, want %q -- not a Beckn acronym", got.Type, "provider") + } + if got.ValidFrom.IsZero() || got.ValidUntil.IsZero() { + t.Error("expected the validity window to be parsed from the key's validFrom/validUntil") + } +} + // --- cache write and metrics edge cases ----------------------------------- // TestCacheResultSkipsWhenDisabled: cacheTTL of 0 means the cache is not From 573dc1dde4a52ffb9698c5823b65df7de72552e1 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 16:02:35 +0530 Subject: [PATCH 043/130] fix: make the on_select answer validate against the Beckn v2 spec [#1] Two things in the answer were refused by base schema validation against the pinned LTS spec, found by validating a live response rather than the request: status.descriptor.code was QUOTED, and the spec's enum is DRAFT, ACTIVE and CLOSED. QUOTED read better and validated nowhere. DRAFT is also the honest value: a quote is a draft commitment, since nothing is committed until init and confirm. each resource lacked quantity, which Commitment.resources requires while the spec defines no quantity property and carries no Quantity schema at all. That defect is upstream, but the consequence is ours: an answer without it fails validation for any consumer who validates. One resource is one day's observation, so one. The resourceAttributes were already valid -- all three per-day resources validate against openagrinet:WeatherObservation v0.1 in Direct mode. The test asserted QUOTED, so it asserted a value the spec refuses; it now asserts DRAFT and that every resource carries a quantity. --- .../mausamgram/weather-observation.select.yaml | 12 +++++++++++- pkg/plugin/implementation/weather/mappings_test.go | 13 +++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/config/mappings/mausamgram/weather-observation.select.yaml b/config/mappings/mausamgram/weather-observation.select.yaml index e63b014f..0b1fe69c 100644 --- a/config/mappings/mausamgram/weather-observation.select.yaml +++ b/config/mappings/mausamgram/weather-observation.select.yaml @@ -188,8 +188,12 @@ response: | "contract": { "commitments": [ { + /* DRAFT, not QUOTED. The Beckn v2 status enum is DRAFT, ACTIVE + and CLOSED, and a quote is still a draft: nothing is committed + until init and confirm. QUOTED read better and validated + nowhere -- base schema validation refuses it. */ "status": { - "descriptor": { "code": "QUOTED", "name": "Quoted" } + "descriptor": { "code": "DRAFT", "name": "Draft" } }, /* The offer is echoed, but its references are not: the request named the abstract point forecast, and the answer returns the @@ -214,6 +218,12 @@ response: | "resources": [$map($days, function($day) { { "id": $resourceId($day), + /* Required by Commitment.resources in the spec, which defines + no quantity property and no Quantity schema anywhere -- a + defect upstream. One resource is one day's observation, so + one. Omitting it makes every answer fail validation for a + consumer who validates. */ + "quantity": 1, "resourceAttributes": { "@context": "https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld", "@type": "openagrinet:WeatherObservation", diff --git a/pkg/plugin/implementation/weather/mappings_test.go b/pkg/plugin/implementation/weather/mappings_test.go index ece33d06..4c275765 100644 --- a/pkg/plugin/implementation/weather/mappings_test.go +++ b/pkg/plugin/implementation/weather/mappings_test.go @@ -195,8 +195,11 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { commitment := firstCommitment(t, answer) status, _ := commitment["status"].(map[string]any) descriptor, _ := status["descriptor"].(map[string]any) - if descriptor["code"] != "QUOTED" { - t.Errorf("status = %v, want QUOTED", descriptor["code"]) + // DRAFT rather than QUOTED: the Beckn v2 status enum is DRAFT, ACTIVE and + // CLOSED, so QUOTED was refused by base schema validation. A quote is a + // draft commitment -- nothing is committed until init and confirm. + if descriptor["code"] != "DRAFT" { + t.Errorf("status = %v, want DRAFT -- QUOTED is not in the spec's enum", descriptor["code"]) } if commitment["offer"] == nil { t.Error("the quoted commitment carries no offer") @@ -224,6 +227,12 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { if !strings.HasPrefix(id, "res:mausamgram:forecast:") { t.Errorf("resource id = %q, want one derived from the forecast date", id) } + // Required by Commitment.resources in the spec even though the spec + // defines no quantity property -- a consumer that validates refuses an + // answer without it. + if _, present := resource["quantity"]; !present { + t.Errorf("resource %s carries no quantity; the spec requires one on every commitment resource", id) + } returned = append(returned, id) } From 22141a50c355430110d85309f3fe1a949d54b712 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Thu, 3 Sep 2026 14:51:26 +0530 Subject: [PATCH 044/130] feat: let a provider present its credential in the query string [#1] Some upstreams take their token as a query parameter rather than a header or basic auth. The step could not do that: authScheme accepted none, basic and header only, so such a provider was unservable whatever the registry said. authScheme query adds it, named symmetrically with the header scheme -- queryName and queryValueEnv -- so the credential is still never in this config or in the registry, only the name of the variable holding it. Half a configuration is refused at startup, as the header scheme's is. It also redacts. Go quotes the whole URL in a transport error, so one unreachable host would otherwise write the token into the log at warn level: `Get "http://host/p?token=s3cr3t": dial tcp ...`. The retry path now replaces the value with REDACTED before logging or returning it. That is the reason this scheme is documented as the least safe of the four -- a query string is also logged by proxies, which nothing here can do anything about. Tests cover the parameter arriving alongside the mapped request rather than replacing it, the credential being absent from a failure's text, and a half-configured scheme being refused. --- .../internal/upstream/upstream.go | 60 +++++++++++- .../internal/upstream/upstream_test.go | 97 +++++++++++++++++++ .../implementation/weather/cmd/plugin.go | 2 + 3 files changed, 156 insertions(+), 3 deletions(-) diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index b64a1d4a..88aa4f15 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -56,6 +56,12 @@ const ( AuthSchemeNone = "none" AuthSchemeBasic = "basic" AuthSchemeHeader = "header" + // AuthSchemeQuery puts the credential in the query string, which some + // upstreams are built around whatever anyone thinks of it. It is the least + // safe of the four -- a query string is logged by proxies and appears in a + // transport error -- so the value is redacted from anything this package + // logs or returns. See redact. + AuthSchemeQuery = "query" ) // codeUpstreamUnavailable reports a provider that could not be reached or @@ -116,6 +122,13 @@ type Config struct { HeaderName string `yaml:"headerName" json:"headerName"` HeaderValueEnv string `yaml:"headerValueEnv" json:"headerValueEnv"` + // QueryName and QueryValueEnv configure authScheme query: the parameter + // name to add, and the environment variable holding its value. Named the + // same way as the header pair, for the same reason -- the credential is + // never in this config, only the name of the variable carrying it. + QueryName string `yaml:"queryName" json:"queryName"` + QueryValueEnv string `yaml:"queryValueEnv" json:"queryValueEnv"` + // MaxResponseBytes caps what is read from the provider. MaxResponseBytes int64 `yaml:"maxResponseBytes" json:"maxResponseBytes"` } @@ -224,8 +237,13 @@ func applyDefaults(cfg *Config) error { if cfg.HeaderName == "" || cfg.HeaderValueEnv == "" { return errors.New("upstream: authScheme header requires headerName and headerValueEnv") } + case AuthSchemeQuery: + if cfg.QueryName == "" || cfg.QueryValueEnv == "" { + return errors.New("upstream: authScheme query requires queryName and queryValueEnv") + } default: - return fmt.Errorf("upstream: unknown authScheme %q: must be none, basic or header", cfg.AuthScheme) + return fmt.Errorf( + "upstream: unknown authScheme %q: must be none, basic, header or query", cfg.AuthScheme) } return nil } @@ -444,8 +462,8 @@ func (s *Step) call(ctx context.Context, baseURL string, call model.ActionPlan, if err == nil { return body, nil } - lastErr = err - log.Warnf(ctx, "upstream: attempt %d/%d failed: %v", attempt, attempts, err) + lastErr = s.redact(err) + log.Warnf(ctx, "upstream: attempt %d/%d failed: %v", attempt, attempts, lastErr) } return nil, model.NewCodedErr(http.StatusBadGateway, codeUpstreamUnavailable, fmt.Errorf("upstream: provider did not answer after %d attempts: %w", attempts, lastErr)) @@ -503,10 +521,46 @@ func (s *Step) authenticate(req *http.Request) error { return fmt.Errorf("upstream: %s must be set for header auth", s.config.HeaderValueEnv) } req.Header.Set(s.config.HeaderName, value) + case AuthSchemeQuery: + value := os.Getenv(s.config.QueryValueEnv) + if value == "" { + return fmt.Errorf("upstream: %s must be set for query auth", s.config.QueryValueEnv) + } + // Set rather than Add: a second copy of the parameter is not a + // credential, it is an ambiguity, and which one an upstream reads is + // its own business. + query := req.URL.Query() + query.Set(s.config.QueryName, value) + req.URL.RawQuery = query.Encode() } return nil } +// redact removes a query-string credential from an error's text. +// +// Go's transport errors quote the whole URL -- `Get "http://host/p?token=..." +// dial tcp: ...` -- so without this, one unreachable host writes the credential +// into the log at warn level. Nothing else in this package puts a URL in a +// message, which is why this is the only place it is needed. +// +// A plain string replacement, because the value is what leaks and the value is +// what we hold. Parsing the error to find it would assume a shape net/http does +// not promise. +func (s *Step) redact(err error) error { + if err == nil || s.config.AuthScheme != AuthSchemeQuery { + return err + } + value := os.Getenv(s.config.QueryValueEnv) + if value == "" { + return err + } + text := strings.ReplaceAll(err.Error(), value, "REDACTED") + if text == err.Error() { + return err + } + return errors.New(text) +} + // buildEndpoint joins the plan's base URL and path, carrying the mapped request // as query parameters when the method takes no body. func buildEndpoint(baseURL string, call model.ActionPlan, mapped []byte) (string, error) { diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go index bf656007..1b995a57 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "net/url" "slices" "sort" "strings" @@ -167,6 +168,102 @@ func TestNewValidatesTheAuthScheme(t *testing.T) { } } +// --- query-string auth ------------------------------------------------------ + +// Some upstreams take their credential as a query parameter. It arrives on the +// request, alongside whatever the mapping produced rather than replacing it. +func TestRunSendsTheCredentialAsAQueryParameter(t *testing.T) { + // No t.Parallel: t.Setenv forbids it, and the credential has to come + // from the environment for this to be testing anything. + t.Setenv("TEST_MANDI_TOKEN", "s3cr3t") + + var got url.Values + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got = r.URL.Query() + fmt.Fprint(w, `{}`) + })) + defer upstream.Close() + + mapper := &stubMapper{requestResult: []byte(`{"statecode":"CG"}`), responseResult: []byte(`{}`)} + step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper, + func(c *Config) { + c.AuthScheme = AuthSchemeQuery + c.QueryName = "token" + c.QueryValueEnv = "TEST_MANDI_TOKEN" + }) + + if _, err := runStep(t, step, selectBody); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + if got.Get("token") != "s3cr3t" { + t.Errorf("token = %q, want the value from the environment", got.Get("token")) + } + // The mapping's own parameters must survive: the credential is added, not + // substituted for the request. + if got.Get("statecode") != "CG" { + t.Errorf("statecode = %q, want the mapped request to be intact", got.Get("statecode")) + } +} + +// The whole reason this scheme is treated as the least safe of the four: Go +// quotes the full URL in a transport error, so an unreachable host would +// otherwise write the credential into the log at warn level. +func TestRunRedactsAQueryCredentialFromAnError(t *testing.T) { + // No t.Parallel: t.Setenv forbids it, and the credential has to come + // from the environment for this to be testing anything. + t.Setenv("TEST_MANDI_TOKEN", "s3cr3t") + + plan := testPlan("http://upstream.invalid", http.MethodGet) + plan.Actions["select"] = model.ActionPlan{ + Method: http.MethodGet, Path: "/get", Mappings: testMappingRef, RetryMax: 0, + } + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + step := newStep(t, &stubRegistry{plan: plan}, mapper, func(c *Config) { + c.AuthScheme = AuthSchemeQuery + c.QueryName = "token" + c.QueryValueEnv = "TEST_MANDI_TOKEN" + }) + + _, err := runStep(t, step, selectBody) + if err == nil { + t.Fatal("expected an unreachable host to fail") + } + if strings.Contains(err.Error(), "s3cr3t") { + t.Errorf("the credential leaked into the error: %v", err) + } + if !strings.Contains(err.Error(), "REDACTED") { + t.Errorf("error %q should show the credential was removed", err) + } +} + +// Half a configuration is refused at startup, the same way the header scheme's +// is: a scheme that cannot present a credential would fail on every call. +func TestNewRefusesAHalfConfiguredQueryScheme(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + cfg *Config + }{ + {"no queryName", &Config{BindingKeys: []string{testBindingKey}, + AuthScheme: AuthSchemeQuery, QueryValueEnv: "TEST_MANDI_TOKEN"}}, + {"no queryValueEnv", &Config{BindingKeys: []string{testBindingKey}, + AuthScheme: AuthSchemeQuery, QueryName: "token"}}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + _, _, err := New(context.Background(), &stubRegistry{}, &stubMapper{}, nil, tc.cfg) + if err == nil { + t.Fatal("expected a half-configured query scheme to be refused") + } + if !strings.Contains(err.Error(), "queryName") { + t.Errorf("error %q should name what is missing", err) + } + }) + } +} + // --- dispatch --------------------------------------------------------------- // Passing through is how dispatch works: several provider steps share a diff --git a/pkg/plugin/implementation/weather/cmd/plugin.go b/pkg/plugin/implementation/weather/cmd/plugin.go index 4bb82112..b7734ea4 100644 --- a/pkg/plugin/implementation/weather/cmd/plugin.go +++ b/pkg/plugin/implementation/weather/cmd/plugin.go @@ -33,6 +33,8 @@ func (p weatherProvider) parseConfig(config map[string]string) (*weather.Config, PasswordEnv: config["passwordEnv"], HeaderName: config["headerName"], HeaderValueEnv: config["headerValueEnv"], + QueryName: config["queryName"], + QueryValueEnv: config["queryValueEnv"], } if raw, exists := config["maxResponseBytes"]; exists && raw != "" { From ed79587c824114e4c95acdd6f3a5c22304dcc96b Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Thu, 3 Sep 2026 14:53:42 +0530 Subject: [PATCH 045/130] feat: log the upstream request and its outcome at info [#1] There was no line saying what was asked of a provider or what came back, so diagnosing one meant reading the mapping and inferring the call. Now every attempt logs the URL as it went on the wire, the status and the response size: upstream: GET http://host/v1/x?statecode=CG&token=REDACTED -> 200 OK, 2 bytes At info rather than debug, because this is the first question any provider problem raises and it should not need a log level change to answer. Logging a URL is only safe because the previous commit can remove a query-string credential from text, so redact is split: an error form and a string form, the latter used here. With any other scheme there is nothing in a URL to hide and the text passes through untouched. --- .../internal/upstream/upstream.go | 33 +++++++++++++++---- .../internal/upstream/upstream_test.go | 27 +++++++++++++++ 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index 88aa4f15..b166651c 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -485,6 +485,11 @@ func (s *Step) attempt(ctx context.Context, call model.ActionPlan, endpoint stri return nil, err } + // The URL as it actually went on the wire, credential removed. At info + // rather than debug because this is the line that answers "what did we ask, + // and what came back" -- the question every provider problem starts with. + requested := s.redactString(req.URL.String()) + resp, err := s.httpClient.Do(req) if err != nil { return nil, err @@ -495,6 +500,7 @@ func (s *Step) attempt(ctx context.Context, call model.ActionPlan, endpoint stri if err != nil { return nil, fmt.Errorf("could not read the response: %w", err) } + log.Infof(ctx, "upstream: %s %s -> %s, %d bytes", call.Method, requested, resp.Status, len(body)) if int64(len(body)) > s.config.MaxResponseBytes { return nil, fmt.Errorf("response exceeds the %d byte limit", s.config.MaxResponseBytes) } @@ -547,20 +553,33 @@ func (s *Step) authenticate(req *http.Request) error { // what we hold. Parsing the error to find it would assume a shape net/http does // not promise. func (s *Step) redact(err error) error { - if err == nil || s.config.AuthScheme != AuthSchemeQuery { - return err - } - value := os.Getenv(s.config.QueryValueEnv) - if value == "" { - return err + if err == nil { + return nil } - text := strings.ReplaceAll(err.Error(), value, "REDACTED") + text := s.redactString(err.Error()) if text == err.Error() { return err } return errors.New(text) } +// redactString removes a query-string credential from any text about to be +// logged or returned -- an error, or the URL that was requested. +// +// Logging the URL is deliberate: it says what was asked of whom, which is the +// first thing anyone wants when a provider misbehaves. This is what makes that +// safe to do at info level. +func (s *Step) redactString(text string) string { + if s.config.AuthScheme != AuthSchemeQuery { + return text + } + value := os.Getenv(s.config.QueryValueEnv) + if value == "" { + return text + } + return strings.ReplaceAll(text, value, "REDACTED") +} + // buildEndpoint joins the plan's base URL and path, carrying the mapped request // as query parameters when the method takes no body. func buildEndpoint(baseURL string, call model.ActionPlan, mapped []byte) (string, error) { diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go index 1b995a57..b778c857 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -236,6 +236,33 @@ func TestRunRedactsAQueryCredentialFromAnError(t *testing.T) { } } +// The URL is logged so a provider problem can be diagnosed from what was asked +// of whom. That makes the credential's absence from it load-bearing, not +// incidental: with a query-string scheme the token is in the URL by definition. +func TestRedactStringRemovesTheCredentialFromTheURL(t *testing.T) { + // No t.Parallel: t.Setenv forbids it. + t.Setenv("TEST_MANDI_TOKEN", "s3cr3t") + + step := &Step{config: &Config{ + AuthScheme: AuthSchemeQuery, + QueryName: "token", + QueryValueEnv: "TEST_MANDI_TOKEN", + }} + got := step.redactString("http://host/v1/x?statecode=CG&token=s3cr3t") + if strings.Contains(got, "s3cr3t") { + t.Errorf("the credential survived redaction: %s", got) + } + if !strings.Contains(got, "REDACTED") || !strings.Contains(got, "statecode=CG") { + t.Errorf("redacted url = %q, want the credential replaced and the rest intact", got) + } + + // Any other scheme has nothing to hide in a URL, so the text is untouched. + plain := &Step{config: &Config{AuthScheme: AuthSchemeNone}} + if out := plain.redactString("http://host/v1/x?statecode=CG"); out != "http://host/v1/x?statecode=CG" { + t.Errorf("a url with no credential must pass through unchanged, got %q", out) + } +} + // Half a configuration is refused at startup, the same way the header scheme's // is: a scheme that cannot present a credential would fail on every call. func TestNewRefusesAHalfConfiguredQueryScheme(t *testing.T) { From d08717dbb13a51bc04bd6f10f4b531812aa12c5f Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Thu, 3 Sep 2026 16:45:00 +0530 Subject: [PATCH 046/130] fix: address the review findings on PR #2 [#1] Seven findings from the review at e64a1fe, in one commit because they are one pass over the same three packages. Two of them were things a comment claimed and the code did not do. RETRY CLASSIFICATION. The loop retried every non-nil error at full rate with no wait, so a 400 burned a retryMax of 5 inside two milliseconds and a missing credential was retried four times and then reported as 502 NET_DOWNSTREAM_UNAVAILABLE -- an operator's unset environment variable laundered into "the provider is down", which points the investigation at the wrong system. Failures no further attempt can change are now marked and not repeated: a request this step could not build, a credential it could not read, a response over the size cap, and any 4xx other than 429. 5xx and 429 still retry, because those are the provider asking to be. Between attempts there is exponential backoff from 50ms capped at 800ms, and a dead context breaks the loop before the call rather than after it. THE MAPPER STOPPED CACHING PERMANENTLY. cached() treated an expired entry as a miss but left it in the map and nothing ever deleted one, so the count only grew; once it reached MaxCacheEntries the cache refused every reference it was not already holding, for the life of the process. The comment said it "just pays to compile again next time" -- it paid every time, for every mapping the deployment had. Expired entries are now swept before the cap is measured. The test reproduces the reviewer's observation: five requests for a fourth reference produced five fetches before the fix and one after. A WITHDRAWN CAPABILITY REPORTED 500. ErrProviderRecordNotFound was wrapped in a plain fmt.Errorf, which lands in the unclassified 500 path, so "binding withdrawn" and "provider suspended" read as this adapter failing. It is a 404 now, with the sentinel still wrapped so errors.Is keeps matching. A registry that could not be consulted stays a 500, because that one is us. A RESPONSE-LEG MAPPING FAILURE BLAMED THE CALLER. Both directions returned 400 SCH_SCHEMA_ADAPTATION_FAILED, justified as the caller's payload being wrong. That holds for the request leg. On the response leg the input is the provider's answer, so a provider that changed shape sent the caller off to fix a request that was fine. The response leg is a 502. SEVERAL COMMITMENTS WERE HALVED. A payload naming the same provider and capability across N commitments resolved to one binding key without complaint, and the mapping then read commitments[0] -- so the caller received a confident, signed, spec-valid answer to part of what it asked. Refused, with the count and the advice to send them separately. The test that asserted the old behaviour is replaced by its inverse. A TTL THAT CACHED NOTHING. config/oan-provider-adapter.yaml set cacheTTL with no cache plugin, so caching was silently off and every message made three registry calls inside signature validation's budget -- while the commented block this PR added to the bpp sample warns about exactly that. The TTL is commented out with the reason, and the registry plugin now says so at startup when a TTL is set without a cache, so it cannot recur quietly. AN INVALID POINT, SIGNED. If the provider answered without echoing its location, JSONata dropped the undefined values and the mapping emitted "coordinates": [] -- an invalid Point, signed and delivered. It now emits location only when both coordinates exist. Absent is honest; empty is a lie in the shape of an answer. Also the two stale Mausamgram names after the rename to weather, and an orphaned doc comment that documented var Provider from 25 lines away. Not taken, both by decision: requiring every declared providerStep to appear in steps: -- declare-without-wiring is how every plugin behaves and the resulting 404 is truthful, though a startup line naming which are wired would have saved some debugging. And validating the generated response before signing it, which touches the handler pipeline and is its own change; the one demonstrated case it would have caught is fixed above in the mapping instead. --- .../weather-observation.select.yaml | 8 +- config/oan-provider-adapter.yaml | 16 +- .../internal/oanbinding/oanbinding.go | 16 +- .../internal/oanbinding/oanbinding_test.go | 64 +++++--- .../internal/upstream/upstream.go | 102 +++++++++++- .../internal/upstream/upstream_test.go | 154 ++++++++++++++++++ .../implementation/jsonmapper/jsonmapper.go | 45 ++++- .../jsonmapper/jsonmapper_test.go | 95 +++++++++++ .../implementation/oanregistry/oanregistry.go | 9 + .../implementation/weather/cmd/plugin.go | 4 +- 10 files changed, 474 insertions(+), 39 deletions(-) diff --git a/config/mappings/mausamgram/weather-observation.select.yaml b/config/mappings/mausamgram/weather-observation.select.yaml index 0b1fe69c..1baf2946 100644 --- a/config/mappings/mausamgram/weather-observation.select.yaml +++ b/config/mappings/mausamgram/weather-observation.select.yaml @@ -234,7 +234,13 @@ response: | "sourceId": "mausamgram", "sourceName": "IMD Mausamgram NWP" }, - "location": { + /* Emitted only when the provider echoed both coordinates. + JSONata drops undefined values inside an array, so a + provider that answered without its location echo would + otherwise produce "coordinates": [] -- an invalid Point, + signed and delivered. Absent is honest; empty is a lie + in the shape of an answer. */ + "location": $exists($lat) and $exists($lon) ? { "type": "Point", "coordinates": [$lon, $lat] }, diff --git a/config/oan-provider-adapter.yaml b/config/oan-provider-adapter.yaml index 8ac94201..891d1a7f 100644 --- a/config/oan-provider-adapter.yaml +++ b/config/oan-provider-adapter.yaml @@ -47,9 +47,19 @@ modules: url: http://registry:8081/api/v1 entity: Participant providerEntity: ProviderSchema - # Opt-in. This is exactly how long a suspended participant keeps - # verifying, and a withdrawn capability keeps being called. - cacheTTL: 60s + # cacheTTL is deliberately unset. It needs a cache plugin alongside + # it -- this client caches nothing without one -- and this sample + # runs no redis, so a TTL here would look like caching while every + # message still made its three registry calls: key lookup, binding + # search, participant search, each inside signature validation's + # budget. The plugin now says so at startup if the two are not + # configured together. + # + # Adding a cache plugin? Read the number carefully. It is exactly + # how long a suspended participant keeps verifying, and a withdrawn + # capability keeps being called. + # + # cacheTTL: 60s keyManager: id: simplekeymanager diff --git a/pkg/plugin/implementation/internal/oanbinding/oanbinding.go b/pkg/plugin/implementation/internal/oanbinding/oanbinding.go index 0f148409..9891e864 100644 --- a/pkg/plugin/implementation/internal/oanbinding/oanbinding.go +++ b/pkg/plugin/implementation/internal/oanbinding/oanbinding.go @@ -50,7 +50,21 @@ func From(paths Paths, body []byte) (Binding, error) { return Binding{}, fmt.Errorf("oanbinding: payload could not be read: %w", err) } - providers := distinct(valuesAt(payload, paths.ProviderID)) + // Before distinctness: N commitments naming the SAME provider and type + // collapse to one binding key, so they would resolve here without + // complaint -- and then the mapping reads commitments[0] and the rest are + // dropped, leaving the caller a confident, signed, spec-valid answer to + // part of what it asked. One request maps to one call, so several is a + // request this design cannot express and is refused rather than halved. + providerValues := valuesAt(payload, paths.ProviderID) + if len(providerValues) > 1 { + return Binding{}, fmt.Errorf( + "oanbinding: payload carries %d commitments; one request maps to one call, "+ + "so send them separately rather than have all but the first dropped", + len(providerValues)) + } + + providers := distinct(providerValues) types := distinct(valuesAt(payload, paths.CapabilityCode)) if len(providers) == 0 || len(types) == 0 { diff --git a/pkg/plugin/implementation/internal/oanbinding/oanbinding_test.go b/pkg/plugin/implementation/internal/oanbinding/oanbinding_test.go index 09b60469..8d231aa5 100644 --- a/pkg/plugin/implementation/internal/oanbinding/oanbinding_test.go +++ b/pkg/plugin/implementation/internal/oanbinding/oanbinding_test.go @@ -98,7 +98,11 @@ func TestFromRefusesAnAmbiguousPayload(t *testing.T) { body: `{"message":{"contract":{"commitments":[ {"offer":{"provider":{"id":"one"}},"resources":[{"resourceAttributes":{"@type":"t"}}]}, {"offer":{"provider":{"id":"two"}},"resources":[{"resourceAttributes":{"@type":"t"}}]}]}}}`, - wants: "2 providers", + // Two providers means two commitments under the Beckn v2 paths, so + // the commitment check refuses it first -- and its advice is the + // more useful of the two. The provider count still guards a + // deployment whose overridden path yields several within one. + wants: "2 commitments", }, { name: "two types within one commitment", @@ -128,24 +132,6 @@ func TestFromRefusesAnAmbiguousPayload(t *testing.T) { } } -// Repetition is not ambiguity: several commitments naming the same provider and -// type describe one call, and must resolve rather than be refused. -func TestFromAcceptsRepetitionOfTheSameBinding(t *testing.T) { - t.Parallel() - - body := `{"message":{"contract":{"commitments":[ - {"offer":{"provider":{"id":"p"}},"resources":[{"resourceAttributes":{"@type":"t"}}]}, - {"offer":{"provider":{"id":"p"}},"resources":[{"resourceAttributes":{"@type":"t"}}]}]}}}` - - got, err := From(BecknV2, []byte(body)) - if err != nil { - t.Fatalf("From() returned an unexpected error: %v", err) - } - if got.Key() != "p|t" { - t.Errorf("key = %q, want p|t", got.Key()) - } -} - func TestFromReportsAnUnreadablePayload(t *testing.T) { t.Parallel() @@ -300,3 +286,43 @@ func TestPathsValidate(t *testing.T) { t.Errorf("the default paths must validate: %v", err) } } + +// Several commitments naming the same provider and capability used to resolve +// to one binding key without complaint -- and then the mapping read +// commitments[0] and the rest were dropped, leaving the caller a confident, +// signed, spec-valid answer to part of what it asked. One request maps to one +// call, so it is refused. +func TestFromRefusesSeveralCommitments(t *testing.T) { + t.Parallel() + + body := `{"message":{"contract":{"commitments":[ + {"offer":{"provider":{"id":"mausamgram"}}, + "resources":[{"resourceAttributes":{"@type":"openagrinet:WeatherObservation"}}]}, + {"offer":{"provider":{"id":"mausamgram"}}, + "resources":[{"resourceAttributes":{"@type":"openagrinet:WeatherObservation"}}]} + ]}}}` + + _, err := From(BecknV2, []byte(body)) + if err == nil { + t.Fatal("expected two commitments to be refused rather than halved") + } + if errors.Is(err, ErrNoBinding) { + t.Error("this is not an absent binding; it is one request asking for two calls") + } + if !strings.Contains(err.Error(), "2 commitments") { + t.Errorf("error %q should say how many were sent", err) + } + + // One commitment still resolves, obviously. + single := `{"message":{"contract":{"commitments":[ + {"offer":{"provider":{"id":"mausamgram"}}, + "resources":[{"resourceAttributes":{"@type":"openagrinet:WeatherObservation"}}]} + ]}}}` + binding, err := From(BecknV2, []byte(single)) + if err != nil { + t.Fatalf("one commitment must still resolve: %v", err) + } + if binding.Key() != "mausamgram|openagrinet:WeatherObservation" { + t.Errorf("binding key = %q", binding.Key()) + } +} diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index b166651c..3050e478 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -53,6 +53,13 @@ const ( // variable to read, so a secret reaches the process through its environment and // nothing else. const ( + // RetryBackoffBase is the first wait between attempts, doubling from there + // up to RetryBackoffMax. Short, because the retry budget comes from the + // registry and an operator setting 5 retries did not ask for seconds of + // latency -- only for the provider's brief unavailability to be ridden out. + RetryBackoffBase = 50 * time.Millisecond + RetryBackoffMax = 800 * time.Millisecond + AuthSchemeNone = "none" AuthSchemeBasic = "basic" AuthSchemeHeader = "header" @@ -133,7 +140,8 @@ type Config struct { MaxResponseBytes int64 `yaml:"maxResponseBytes" json:"maxResponseBytes"` } -// Step serves the Mausamgram capability. It is safe for concurrent use. +// Step serves whatever capabilities a domain package configures it for. It is +// safe for concurrent use. type Step struct { config *Config paths oanbinding.Paths @@ -269,6 +277,17 @@ func (s *Step) Run(ctx *model.StepContext) error { plan, err := s.registry.ProviderRecord(ctx, binding.Key()) if err != nil { + // A definite "no such binding" is the caller naming something that is + // not there, so 404 -- the same reasoning the no-route path uses to + // refuse an unrecognised capability rather than ACK it. A registry that + // could not be consulted is different and stays a 500: unclassified, + // because it is this adapter that failed. + if errors.Is(err, definition.ErrProviderRecordNotFound) { + // %w, not %v: the sentinel has to stay unwrappable, or anything + // upstream testing errors.Is against it silently stops matching. + return model.NewNotFoundErr("", fmt.Errorf( + "upstream: the registry publishes no active binding for %s: %w", binding.Key(), err)) + } return fmt.Errorf("upstream: no call plan for %s: %w", binding.Key(), err) } @@ -458,17 +477,82 @@ func (s *Step) call(ctx context.Context, baseURL string, call model.ActionPlan, var lastErr error for attempt := 1; attempt <= attempts; attempt++ { + // A caller that has gone away is not worth another attempt, and neither + // is a budget already spent. Checked before the call rather than after, + // so a cancelled request costs nothing. + if err := ctx.Err(); err != nil { + if lastErr == nil { + lastErr = err + } + break + } + body, err := s.attempt(ctx, call, endpoint, mapped, timeout) if err == nil { return body, nil } lastErr = s.redact(err) log.Warnf(ctx, "upstream: attempt %d/%d failed: %v", attempt, attempts, lastErr) + + // Only some failures are worth repeating. A 4xx, a request this step + // could not build and a credential it could not read will fail + // identically however many times they are tried -- and retrying the + // credential case is the worst of them, because it reports an + // operator's missing environment variable as the provider being down. + if isPermanent(err) { + break + } + if attempt < attempts { + if err := sleep(ctx, backoff(attempt)); err != nil { + break + } + } } return nil, model.NewCodedErr(http.StatusBadGateway, codeUpstreamUnavailable, fmt.Errorf("upstream: provider did not answer after %d attempts: %w", attempts, lastErr)) } +// permanentErr marks a failure no retry can fix. Kept unexported and detected +// with errors.As, so a caller of this package sees only the underlying error. +type permanentErr struct{ error } + +func (p permanentErr) Unwrap() error { return p.error } + +// doNotRetry marks err as not worth repeating. +func doNotRetry(err error) error { return permanentErr{err} } + +// isPermanent reports whether err is one no further attempt would change. +func isPermanent(err error) bool { + var permanent permanentErr + return errors.As(err, &permanent) +} + +// backoff is how long to wait before the next attempt. +// +// Exponential from a short base and capped, because the provider being briefly +// busy is the case worth waiting out; anything longer is a timeout's job. With +// no wait at all a retryMax of 5 spends its whole budget inside a couple of +// milliseconds, which is not a retry so much as the same failure six times. +func backoff(attempt int) time.Duration { + wait := RetryBackoffBase << (attempt - 1) + if wait > RetryBackoffMax { + return RetryBackoffMax + } + return wait +} + +// sleep waits, or reports that the context ended first. +func sleep(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + // attempt makes one upstream request. func (s *Step) attempt(ctx context.Context, call model.ActionPlan, endpoint string, mapped []byte, timeout time.Duration) ([]byte, error) { attemptCtx, cancel := context.WithTimeout(ctx, timeout) @@ -476,13 +560,14 @@ func (s *Step) attempt(ctx context.Context, call model.ActionPlan, endpoint stri req, err := http.NewRequestWithContext(attemptCtx, call.Method, endpoint, requestBody(call.Method, mapped)) if err != nil { - return nil, fmt.Errorf("could not build the request: %w", err) + return nil, doNotRetry(fmt.Errorf("could not build the request: %w", err)) } if hasBody(call.Method) { req.Header.Set("Content-Type", "application/json") } if err := s.authenticate(req); err != nil { - return nil, err + // A missing or unreadable credential is configuration, not weather. + return nil, doNotRetry(err) } // The URL as it actually went on the wire, credential removed. At info @@ -502,10 +587,17 @@ func (s *Step) attempt(ctx context.Context, call model.ActionPlan, endpoint stri } log.Infof(ctx, "upstream: %s %s -> %s, %d bytes", call.Method, requested, resp.Status, len(body)) if int64(len(body)) > s.config.MaxResponseBytes { - return nil, fmt.Errorf("response exceeds the %d byte limit", s.config.MaxResponseBytes) + // Asking again will not make the answer smaller. + return nil, doNotRetry(fmt.Errorf("response exceeds the %d byte limit", s.config.MaxResponseBytes)) } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("provider returned %s", resp.Status) + err := fmt.Errorf("provider returned %s", resp.Status) + // 5xx and 429 are the provider asking to be tried again. Every other + // 4xx is a statement about the request, which will not improve. + if resp.StatusCode < http.StatusInternalServerError && resp.StatusCode != http.StatusTooManyRequests { + return nil, doNotRetry(err) + } + return nil, err } return body, nil } diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go index b778c857..e231a23d 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -168,6 +168,160 @@ func TestNewValidatesTheAuthScheme(t *testing.T) { } } +// --- what is worth retrying ------------------------------------------------- + +// A 4xx is a statement about the request. Repeating it changes nothing, and +// the whole budget was previously spent inside a couple of milliseconds. +func TestRunDoesNotRetryAClientError(t *testing.T) { + t.Parallel() + + var attempts int + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts++ + w.WriteHeader(http.StatusBadRequest) + })) + defer upstream.Close() + + plan := testPlan(upstream.URL, http.MethodGet) + plan.Actions["select"] = model.ActionPlan{ + Method: http.MethodGet, Path: "/x", Mappings: testMappingRef, RetryMax: 5, + } + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + step := newStep(t, &stubRegistry{plan: plan}, mapper) + + if _, err := runStep(t, step, selectBody); err == nil { + t.Fatal("expected a 400 from the provider to be reported") + } + if attempts != 1 { + t.Errorf("the provider was called %d times, want 1 -- a 400 is not worth retrying", attempts) + } +} + +// 5xx and 429 are the provider asking to be tried again, so those still are. +func TestRunRetriesWhatTheProviderAsksItTo(t *testing.T) { + t.Parallel() + + for _, status := range []int{http.StatusInternalServerError, http.StatusTooManyRequests} { + t.Run(http.StatusText(status), func(t *testing.T) { + t.Parallel() + + var attempts int + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts++ + w.WriteHeader(status) + })) + defer upstream.Close() + + plan := testPlan(upstream.URL, http.MethodGet) + plan.Actions["select"] = model.ActionPlan{ + Method: http.MethodGet, Path: "/x", Mappings: testMappingRef, RetryMax: 2, + } + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + step := newStep(t, &stubRegistry{plan: plan}, mapper) + + if _, err := runStep(t, step, selectBody); err == nil { + t.Fatal("expected the failure to be reported") + } + if attempts != 3 { + t.Errorf("the provider was called %d times, want 3 (1 + retryMax 2)", attempts) + } + }) + } +} + +// An operator's missing environment variable is configuration, not the provider +// being down. Retrying it reported the wrong system as broken. +func TestRunDoesNotRetryAMissingCredential(t *testing.T) { + t.Parallel() + + var called int + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + called++ + fmt.Fprint(w, `{}`) + })) + defer upstream.Close() + + plan := testPlan(upstream.URL, http.MethodGet) + plan.Actions["select"] = model.ActionPlan{ + Method: http.MethodGet, Path: "/x", Mappings: testMappingRef, RetryMax: 4, + } + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + step := newStep(t, &stubRegistry{plan: plan}, mapper, func(c *Config) { + c.AuthScheme = AuthSchemeBasic + c.UsernameEnv = "TEST_ABSENT_USER_FOR_RETRY" + c.PasswordEnv = "TEST_ABSENT_PASS_FOR_RETRY" + }) + + _, err := runStep(t, step, selectBody) + if err == nil { + t.Fatal("expected a missing credential to be reported") + } + if called != 0 { + t.Errorf("the provider was called %d times; a credential this step cannot read never reaches it", called) + } + if !strings.Contains(err.Error(), "TEST_ABSENT_USER_FOR_RETRY") { + t.Errorf("error %q should name the variable that is unset", err) + } +} + +// A caller that has gone away gets no further attempts. +func TestRunStopsWhenTheCallerHasGone(t *testing.T) { + t.Parallel() + + var attempts int + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts++ + w.WriteHeader(http.StatusInternalServerError) + })) + defer upstream.Close() + + plan := testPlan(upstream.URL, http.MethodGet) + plan.Actions["select"] = model.ActionPlan{ + Method: http.MethodGet, Path: "/x", Mappings: testMappingRef, RetryMax: 5, + } + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + step := newStep(t, &stubRegistry{plan: plan}, mapper) + + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + stepCtx := &model.StepContext{Context: cancelled, Body: []byte(selectBody)} + if err := step.Run(stepCtx); err == nil { + t.Fatal("expected a cancelled request to be reported") + } + if attempts != 0 { + t.Errorf("the provider was called %d times for an abandoned request, want 0", attempts) + } +} + +// A withdrawn binding is the caller naming something that is not there, so 404 +// -- the same reasoning the no-route path uses. A registry that could not be +// consulted is this adapter failing, and stays unclassified. +func TestRunSeparatesAWithdrawnBindingFromAnUnreachableRegistry(t *testing.T) { + t.Parallel() + + withdrawn := &stubRegistry{err: definition.ErrProviderRecordNotFound} + _, err := runStep(t, newStep(t, withdrawn, &stubMapper{}), selectBody) + if err == nil { + t.Fatal("expected a withdrawn binding to be refused") + } + var coded *model.CodedErr + if !errors.As(err, &coded) || coded.HTTPStatus() != http.StatusNotFound { + t.Errorf("a withdrawn binding gave %v, want a 404 -- a 500 hides it as this adapter's fault", err) + } + if !strings.Contains(err.Error(), testBindingKey) { + t.Errorf("error %q should name the binding with no record", err) + } + + unreachable := &stubRegistry{err: errors.New("registry unreachable")} + _, err = runStep(t, newStep(t, unreachable, &stubMapper{}), selectBody) + if err == nil { + t.Fatal("expected an unreachable registry to be reported") + } + if errors.As(err, &coded) && coded.HTTPStatus() == http.StatusNotFound { + t.Error("an unreachable registry must not report as not-found; it is this adapter failing") + } +} + // --- query-string auth ------------------------------------------------------ // Some upstreams take their credential as a query parameter. It arrives on the diff --git a/pkg/plugin/implementation/jsonmapper/jsonmapper.go b/pkg/plugin/implementation/jsonmapper/jsonmapper.go index 3088c73f..37f9c44b 100644 --- a/pkg/plugin/implementation/jsonmapper/jsonmapper.go +++ b/pkg/plugin/implementation/jsonmapper/jsonmapper.go @@ -349,11 +349,22 @@ func (m *Mapper) remember(mappingRef string, directions map[definition.Direction m.mu.Lock() defer m.mu.Unlock() + // Expired entries are dropped before the cap is measured. Without this the + // map only ever grows: cached() treats an expiry as a miss but leaves the + // entry behind, so the count reaches the cap and then refuses every ref it + // is not already holding -- permanently, for the life of the process. The + // cost is not "compile again next time" but compile every time, for every + // mapping the deployment has. + // + // A sweep rather than an eviction policy: the entries are few, this runs + // only on a store, and it drops nothing that was still usable. + m.purgeExpired() + // Bounded rather than evicting: references come from the registry, and a - // deployment serving more capabilities than the cap wants a bigger cap, not - // a cache that silently thrashes. The entry is still returned to its caller - // when it is not stored, so a request over the cap is served rather than - // refused -- it just pays to compile again next time. + // deployment serving more live capabilities than the cap wants a bigger + // cap, not a cache that silently thrashes. The entry is still returned to + // its caller when it is not stored, so a request over the cap is served + // rather than refused -- it just pays to compile again next time. if len(m.entries) >= m.config.MaxCacheEntries { if _, replacing := m.entries[mappingRef]; !replacing { return entry @@ -363,6 +374,16 @@ func (m *Mapper) remember(mappingRef string, directions map[definition.Direction return entry } +// purgeExpired drops entries past their TTL. The caller holds m.mu. +func (m *Mapper) purgeExpired() { + now := time.Now() + for ref, entry := range m.entries { + if now.After(entry.expiresAt) { + delete(m.entries, ref) + } + } +} + // cachedCount reports how many mappings are held. Used by tests to assert the // cache stays bounded. func (m *Mapper) cachedCount() int { @@ -538,11 +559,19 @@ func (m *Mapper) evaluate(ctx context.Context, mapping *compiledMapping, mapping result, err := mapping.expression.Evaluate(document, nil) mapping.evaluating.Unlock() if err != nil { - // The mapping is valid and the payload is not what it expected, so this - // is the caller's request being wrong rather than this adapter failing. log.Errorf(ctx, err, "JSON mapping %s %s half failed to evaluate: %v", mappingRef, direction, err) - return nil, model.NewBadReqErr(codeAdaptationFailed, - fmt.Errorf("mapping %q %s half could not be applied: %w", mappingRef, direction, err)) + wrapped := fmt.Errorf("mapping %q %s half could not be applied: %w", mappingRef, direction, err) + if direction == definition.DirectionResponse { + // The input here is the PROVIDER's answer, not the caller's + // request. A provider that changed shape, or a bug in the response + // half, is nothing the caller did -- reporting 400 sends them off + // to fix a request that was fine. 502: the upstream exchange is + // what failed. + return nil, model.NewCodedErr(http.StatusBadGateway, codeAdaptationFailed, wrapped) + } + // On the request leg the mapping is valid and the payload is not what it + // expected, so this is the caller's request being wrong. + return nil, model.NewBadReqErr(codeAdaptationFailed, wrapped) } return result, nil } diff --git a/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go b/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go index 04aebd82..01fa51d8 100644 --- a/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go +++ b/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync" "sync/atomic" "testing" "time" @@ -754,6 +755,100 @@ func TestTransformBoundsTheCache(t *testing.T) { } } +// Expired entries must not hold the cap. They used to: cached() treats an +// expiry as a miss but left the entry in the map, nothing ever deleted one, so +// the count only grew -- and once it reached MaxCacheEntries the cache refused +// every reference it was not already holding, permanently. The effect was not +// "compile again next time" but compile every time, for every mapping the +// deployment had. +func TestTransformKeepsCachingAfterEntriesExpire(t *testing.T) { + t.Parallel() + + var fetches int + var mu sync.Mutex + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + fetches++ + mu.Unlock() + fmt.Fprint(w, bothDirections) + })) + defer srv.Close() + + // A TTL short enough to expire between calls, and a cap small enough that + // stale entries would fill it. + mapper := newTestMapper(t, func(c *Config) { + c.MaxCacheEntries = 3 + c.CacheTTL = time.Millisecond + }) + + // Fill the cache and let everything in it go stale. + for i := 0; i < 3; i++ { + if _, err := mapper.Transform(context.Background(), + fmt.Sprintf("%s/%d.yaml", srv.URL, i), definition.DirectionRequest, requestInput()); err != nil { + t.Fatalf("Transform() returned an unexpected error: %v", err) + } + } + time.Sleep(20 * time.Millisecond) + + mu.Lock() + before := fetches + mu.Unlock() + + // A fourth reference, asked for repeatedly. It should be cached after the + // first fetch, because the three stale entries no longer occupy the cap. + fourth := srv.URL + "/fourth.yaml" + for i := 0; i < 5; i++ { + if _, err := mapper.Transform(context.Background(), fourth, + definition.DirectionRequest, requestInput()); err != nil { + t.Fatalf("Transform() returned an unexpected error: %v", err) + } + } + + mu.Lock() + after := fetches + mu.Unlock() + if got := after - before; got != 1 { + t.Errorf("a new reference was fetched %d times over 5 requests, want 1 -- "+ + "stale entries are holding the cap", got) + } +} + +// A failure on the response leg is not the caller's fault. The input there is +// the PROVIDER's answer, so a provider that changed shape, or a bug in the +// response half, used to return 400 and send the caller off to fix a request +// that was fine. +func TestTransformBlamesTheRightPartyForEachDirection(t *testing.T) { + t.Parallel() + + // A mapping whose halves both fail at evaluation: $number over a value that + // is not a number. + failing := "request: |\n $number(beckn.notANumber)\nresponse: |\n $number(response.notANumber)\n" + srv := newMappingServer(t, failing, nil) + defer srv.Close() + + mapper := newTestMapper(t) + ref := srv.URL + "/failing.yaml" + + _, err := mapper.Transform(context.Background(), ref, definition.DirectionRequest, + map[string]any{"beckn": map[string]any{"notANumber": "abc"}}) + if err == nil { + t.Fatal("expected the request half to fail") + } + var coded *model.CodedErr + if !errors.As(err, &coded) || coded.HTTPStatus() != http.StatusBadRequest { + t.Errorf("request leg gave %v, want a 400 -- the caller's payload is what the mapping could not read", err) + } + + _, err = mapper.Transform(context.Background(), ref, definition.DirectionResponse, + map[string]any{"response": map[string]any{"notANumber": "abc"}}) + if err == nil { + t.Fatal("expected the response half to fail") + } + if !errors.As(err, &coded) || coded.HTTPStatus() != http.StatusBadGateway { + t.Errorf("response leg gave %v, want a 502 -- the provider's answer is what failed, not the request", err) + } +} + // --- concurrency ------------------------------------------------------------ // Every inbound request shares one mapper, so the cache is read and written diff --git a/pkg/plugin/implementation/oanregistry/oanregistry.go b/pkg/plugin/implementation/oanregistry/oanregistry.go index 53347209..dd2eb521 100644 --- a/pkg/plugin/implementation/oanregistry/oanregistry.go +++ b/pkg/plugin/implementation/oanregistry/oanregistry.go @@ -249,6 +249,15 @@ func New(ctx context.Context, cache definition.Cache, cfg *Config) (*Client, fun return nil, nil, err } + // A TTL with no cache plugin is caching silently switched off, and the cost + // is three registry round trips inside every request -- key lookup, binding + // search, participant search -- each inside signature validation's budget. + // Said out loud at startup, because nothing downstream ever complains. + if cfg.CacheTTL > 0 && cache == nil { + log.Warnf(ctx, "OAN registry: cacheTTL is %s but no cache plugin is configured, "+ + "so nothing is cached and every message makes its registry calls again", cfg.CacheTTL) + } + entity := cfg.Entity if entity == "" { entity = DefaultEntity diff --git a/pkg/plugin/implementation/weather/cmd/plugin.go b/pkg/plugin/implementation/weather/cmd/plugin.go index b7734ea4..e78c17ef 100644 --- a/pkg/plugin/implementation/weather/cmd/plugin.go +++ b/pkg/plugin/implementation/weather/cmd/plugin.go @@ -69,11 +69,10 @@ func (p weatherProvider) New(ctx context.Context, registry definition.ProviderRe return nil, nil, err } - log.Infof(ctx, "Mausamgram step created successfully") + log.Infof(ctx, "Weather step created successfully") return step, closer, nil } -// Provider is the exported plugin instance. // splitList reads a comma-separated config value, which is how a list reaches a // plugin -- the config is map[string]string. Blanks are dropped and spaces // trimmed, so a trailing comma or a wrapped line is not a config error. @@ -93,6 +92,7 @@ func splitList(raw string) []string { return out } +// Provider is the exported plugin instance. var Provider = weatherProvider{} // Compile-time proof the provider satisfies the interface the manager asserts From f506c0a5fb99a3ea4ae0623387d41144b3a01da0 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Thu, 3 Sep 2026 17:08:21 +0530 Subject: [PATCH 047/130] fix: report an unservable payload as a bad request [#1] Refusing multi-commitment payloads was right and its status was not: the error from oanbinding.From was unclassified, so it landed in the 500 path and the caller got NET_INTERNAL_ERROR with the reason only in this process's log. That is the same fault the review raised about a withdrawn binding, in the fix for it. Found by running the live stack rather than the unit tests, which asserted the refusal without looking at the status. Everything From refuses is a statement about the payload -- unreadable JSON, or a request naming more than one call -- so 400, carrying the message that says which. --- .../internal/upstream/upstream.go | 6 +++- .../internal/upstream/upstream_test.go | 36 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index 3050e478..47623f01 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -268,7 +268,11 @@ func (s *Step) Run(ctx *model.StepContext) error { return nil } if err != nil { - return err + // Everything From refuses is a statement about the payload: unreadable + // JSON, or a request naming more than one call. Unclassified it becomes + // a 500, which says this adapter broke and leaves the reason in a log + // the caller cannot read. + return model.NewBadReqErr("", err) } if !s.serves(binding.Key()) { log.Debugf(ctx, "upstream: %s is not one of this step's capabilities, passing through", binding.Key()) diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go index e231a23d..f60564b6 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -168,6 +168,42 @@ func TestNewValidatesTheAuthScheme(t *testing.T) { } } +// A payload this step cannot derive a binding from is the caller's mistake, so +// it must not surface as a 500 with the reason only in our log. Found by +// checking the live stack after refusing multi-commitment payloads: the refusal +// was right and the status was not. +func TestRunReportsAnUnservablePayloadAsABadRequest(t *testing.T) { + t.Parallel() + + twoCommitments := `{"context":{"action":"select"},"message":{"contract":{"commitments":[ + {"offer":{"provider":{"id":"mausamgram"}}, + "resources":[{"resourceAttributes":{"@type":"openagrinet:WeatherObservation"}}]}, + {"offer":{"provider":{"id":"mausamgram"}}, + "resources":[{"resourceAttributes":{"@type":"openagrinet:WeatherObservation"}}]} + ]}}}` + + for _, tc := range []struct{ name, body, wants string }{ + {"two commitments", twoCommitments, "2 commitments"}, + {"unreadable json", `{"message":`, "could not be read"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + _, err := runStep(t, newStep(t, &stubRegistry{}, &stubMapper{}), tc.body) + if err == nil { + t.Fatal("expected the payload to be refused") + } + var coded *model.CodedErr + if !errors.As(err, &coded) || coded.HTTPStatus() != http.StatusBadRequest { + t.Errorf("got %v, want a 400 -- a 500 blames this adapter for the caller's payload", err) + } + if !strings.Contains(err.Error(), tc.wants) { + t.Errorf("error %q should say %q so the caller can act on it", err, tc.wants) + } + }) + } +} + // --- what is worth retrying ------------------------------------------------- // A 4xx is a statement about the request. Repeating it changes nothing, and From 3f54ab7dc41f93320239de85b37e3f415ce68e60 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Thu, 3 Sep 2026 23:09:01 +0530 Subject: [PATCH 048/130] fix: accept any 2xx, and say what the provider said [#1] Two things the step got wrong about a provider's answer. ONLY 200 COUNTED AS SUCCESS. A provider is entitled to answer 202 for work it accepted, 201 for something it created, or 204 for nothing to report, and all three were treated as failures -- so a perfectly good exchange was refused on the status line. Any 2xx is an answer now. 3xx never reached here: the client follows redirects. THE RESPONSE BODY WAS THROWN AWAY ON FAILURE. It was read, its length logged, and then discarded, so a failure reported "provider returned 400 Bad Request" and nothing about why. That is the first thing an operator needs, and the thing that makes a real provider's behaviour observable at all -- Agmarknet reports "no data" in the body of a 400, which until now was invisible. A failure now quotes the body: whitespace collapsed so an indented JSON or an HTML error page does not spread one failure across forty log lines, truncated at 300 characters so a page of HTML does not end up in a NACK, and "(no body)" when there is nothing to quote. It goes through the same redaction as everything else, which a test covers by having the provider echo the query string back -- credential and all -- and asserting it does not survive. One behaviour is now asserted rather than left to be discovered: a 204 passes the status check and then fails decoding, because there is no JSON to map, and the error says the body could not be read rather than blaming the status. If a real provider uses 204 for "nothing to report", that is the line that needs a decision. --- .../internal/upstream/upstream.go | 34 ++++- .../internal/upstream/upstream_test.go | 136 ++++++++++++++++++ 2 files changed, 168 insertions(+), 2 deletions(-) diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index 47623f01..d8b15fbe 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -594,8 +594,12 @@ func (s *Step) attempt(ctx context.Context, call model.ActionPlan, endpoint stri // Asking again will not make the answer smaller. return nil, doNotRetry(fmt.Errorf("response exceeds the %d byte limit", s.config.MaxResponseBytes)) } - if resp.StatusCode != http.StatusOK { - err := fmt.Errorf("provider returned %s", resp.Status) + // Any 2xx, not 200 alone. A provider is entitled to answer 202 for work it + // accepted, 204 for nothing to report, or 201 for something it created, and + // treating those as failures would refuse a perfectly good exchange. 3xx + // does not reach here: the client follows redirects. + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + err := fmt.Errorf("provider returned %s: %s", resp.Status, explain(body)) // 5xx and 429 are the provider asking to be tried again. Every other // 4xx is a statement about the request, which will not improve. if resp.StatusCode < http.StatusInternalServerError && resp.StatusCode != http.StatusTooManyRequests { @@ -606,6 +610,32 @@ func (s *Step) attempt(ctx context.Context, call model.ActionPlan, endpoint stri return body, nil } +// explainLimit is how much of a failed response is quoted. Enough for a +// provider's own message, short enough not to put a page of HTML in a log line +// or a NACK. +const explainLimit = 300 + +// explain renders a failed response body for a human. +// +// The body was already read and then thrown away, so a provider's own account +// of what was wrong -- Agmarknet says "no data" in the body of a 400 -- never +// reached anyone. The status alone says a call failed and nothing about why, +// which is the first thing an operator needs and the thing that makes a real +// provider's behaviour observable at all. +func explain(body []byte) string { + text := strings.TrimSpace(string(body)) + if text == "" { + return "(no body)" + } + // Collapse whitespace: a provider that answers with indented JSON or an + // HTML error page should not spread one failure over forty log lines. + text = strings.Join(strings.Fields(text), " ") + if len(text) > explainLimit { + return text[:explainLimit] + "... (truncated)" + } + return text +} + // authenticate presents this provider's credentials, read from the environment // at call time so a rotated secret takes effect without a restart. func (s *Step) authenticate(req *http.Request) error { diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go index f60564b6..ca22d469 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -358,6 +358,142 @@ func TestRunSeparatesAWithdrawnBindingFromAnUnreachableRegistry(t *testing.T) { } } +// --- what counts as an answer ------------------------------------------------ + +// Any 2xx is an answer. Only 200 used to be, so a provider entitled to reply +// 202 for accepted work or 201 for something created had its perfectly good +// exchange refused. +func TestRunAcceptsAnyTwoHundred(t *testing.T) { + t.Parallel() + + for _, status := range []int{http.StatusOK, http.StatusCreated, http.StatusAccepted} { + t.Run(http.StatusText(status), func(t *testing.T) { + t.Parallel() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(status) + fmt.Fprint(w, `{"answered":true}`) + })) + defer upstream.Close() + + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{"ok":1}`)} + step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper) + + ctx, err := runStep(t, step, selectBody) + if err != nil { + t.Fatalf("%d should be an answer, got %v", status, err) + } + if len(ctx.ResponseBody) == 0 { + t.Errorf("%d produced no answer", status) + } + }) + } +} + +// A 204 passes the status check and then fails decoding, because there is no +// JSON to map. Asserted rather than left to be discovered: the failure names +// the empty body instead of the status, and if a provider ever uses 204 for +// "nothing to report" this is the line that will need a decision. +func TestRunReportsAnEmptyBodyRatherThanTheStatus(t *testing.T) { + t.Parallel() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + defer upstream.Close() + + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper) + + _, err := runStep(t, step, selectBody) + if err == nil { + t.Fatal("expected a 204 with no body to be reported") + } + if !strings.Contains(err.Error(), "not JSON") { + t.Errorf("error %q should say the body could not be read, not blame the status", err) + } +} + +// A provider's own account of the failure has to survive. The body was read and +// then thrown away, so a 400 carrying {"message":"no data"} reached an operator +// as "provider returned 400 Bad Request" and nothing else -- which is the first +// thing anyone needs and the thing that makes a real provider observable. +func TestRunQuotesTheProvidersExplanation(t *testing.T) { + t.Parallel() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + fmt.Fprint(w, "{\n \"message\": \"no data available\"\n}") + })) + defer upstream.Close() + + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper) + + _, err := runStep(t, step, selectBody) + if err == nil { + t.Fatal("expected the failure to be reported") + } + if !strings.Contains(err.Error(), "no data available") { + t.Errorf("error %q should carry the provider's own message", err) + } + // Whitespace collapsed, so an indented body does not spread one failure + // over several log lines. + if strings.Contains(err.Error(), "\n") { + t.Errorf("error %q should have its whitespace collapsed", err) + } +} + +// A body is quoted, not dumped: a provider answering with a page of HTML must +// not put all of it in a log line or a NACK. +func TestExplainTruncatesAndHandlesAnEmptyBody(t *testing.T) { + t.Parallel() + + if got := explain(nil); got != "(no body)" { + t.Errorf("explain(nil) = %q, want a marker rather than an empty string", got) + } + long := explain([]byte(strings.Repeat("x", explainLimit+50))) + if len(long) > explainLimit+len("... (truncated)") { + t.Errorf("explain kept %d characters, want it truncated near %d", len(long), explainLimit) + } + if !strings.HasSuffix(long, "(truncated)") { + t.Errorf("a truncated body should say so, got %q", long[len(long)-20:]) + } +} + +// The quoted body goes through the same redaction as everything else, or a +// provider that echoes the query string back would defeat it. +func TestRunRedactsACredentialEchoedInABody(t *testing.T) { + // No t.Parallel: t.Setenv forbids it. + t.Setenv("TEST_ECHO_TOKEN", "s3cr3t") + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + // A provider quoting the request it rejected, credential and all. + fmt.Fprintf(w, `{"rejected":%q}`, r.URL.RawQuery) + })) + defer upstream.Close() + + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper, + func(c *Config) { + c.AuthScheme = AuthSchemeQuery + c.QueryName = "token" + c.QueryValueEnv = "TEST_ECHO_TOKEN" + }) + + _, err := runStep(t, step, selectBody) + if err == nil { + t.Fatal("expected the failure to be reported") + } + if strings.Contains(err.Error(), "s3cr3t") { + t.Errorf("the credential leaked through the quoted body: %v", err) + } + if !strings.Contains(err.Error(), "REDACTED") { + t.Errorf("error %q should show the credential was removed", err) + } +} + // --- query-string auth ------------------------------------------------------ // Some upstreams take their credential as a query parameter. It arrives on the From 5aa1882a596293d6e7989839c043fb18bd98b303 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Sun, 6 Sep 2026 18:29:11 +0530 Subject: [PATCH 049/130] docs(config): document every plugin option, and stop annotating upstream samples [#1] Two things, both about where an explanation belongs. REVERTS THE BECKN-ONE SAMPLES. 5ef4cbd added a commented-out oanregistry block to config/local-beckn-one-bap.yaml and -bpp.yaml, 135 lines across the two. They are upstream beckn-onix example configs and our fork has no business annotating them: the note is about an OAN plugin, so it belongs in the OAN config, which is what the rest of this commit makes it worth reading. Pure deletion -- the additions were comments, so nothing changes behaviour. DOCUMENTS THE OAN CONFIG PROPERLY. Every option the three plugins actually read, checked against the code rather than remembered: oanregistry url (required, no default), entity, providerEntity, timeout, retry_max, retry_wait_min, retry_wait_max, cacheTTL jsonmapper fetchTimeout, cacheTTL, negativeTTL, maxMappingBytes, maxCacheEntries -- all optional, defaults stated provider step bindingKeys (required), providerIdAt, capabilityCodeAt, authScheme and the env-var keys for each of basic, header and query, maxResponseBytes And, more useful than any of them, WHAT IS NOT CONFIGURED HERE. The call plan -- method, path, mappings, timeoutMs, retryMax -- is the registry's ProviderSchema row, read per request. So repointing a provider or giving a slow one longer is a registry write, not an edit here and a restart. Retry classification is not configurable at all: 4xx permanent, 5xx retried, backoff 50ms rising to 800ms. Also adds schemaValidator and validateSchema, which the deployed configs have had for a while and this reference lacked, and mounts at / rather than /beckn/ to match them. Fixes a comment that was simply wrong: it said which action a request is "comes from the payload's context.action, never from the URL". It is the other way round. Routing strips the mount path and matches what remains; the validator is the sole exception, reading context.action and ignoring the path it is handed. Nothing reconciles the two. Verified by booting this config in the published image: all seven plugins load, the module registers at /, and the server listens. The documented default binding paths are quoted from oanbinding.BecknV2 including the "[]" array markers -- without them the path matches nothing, which is exactly the kind of error a reference config should not teach. --- config/local-beckn-one-bap.yaml | 54 ------- config/local-beckn-one-bpp.yaml | 81 ----------- config/oan-provider-adapter.yaml | 243 ++++++++++++++++++++++++++----- 3 files changed, 206 insertions(+), 172 deletions(-) diff --git a/config/local-beckn-one-bap.yaml b/config/local-beckn-one-bap.yaml index 0cfcf533..7399cc5e 100644 --- a/config/local-beckn-one-bap.yaml +++ b/config/local-beckn-one-bap.yaml @@ -55,33 +55,6 @@ modules: retry_max: 3 retry_wait_min: 100ms retry_wait_max: 500ms - - # To resolve signing keys from an OAN Registry (SunbirdRC) instead, - # swap the block above for this one. Both implement RegistryLookup, so - # nothing else in the module changes. See - # pkg/plugin/implementation/oanregistry/README.md. - # - # registry: - # id: oanregistry - # config: - # # Required, and the only setting with no default. Include the API - # # version prefix -- the plugin appends /{entity}/search. Use the - # # service name, not localhost: in a container localhost is the - # # adapter itself. - # url: http://registry:8081/api/v1 - # entity: Participant - # # Tighter than dediregistry's on purpose: this runs inside - # # signature validation on every inbound message, so - # # timeout x (retry_max + 1) is time a request waits before it can - # # even be rejected. - # timeout: 2 - # retry_max: 1 - # retry_wait_min: 100ms - # retry_wait_max: 500ms - # # Omitted means caching is off. The TTL is how long a suspended - # # participant keeps verifying, so it is opt-in. Needs a cache - # # plugin configured as well. - # # cacheTTL: 60s keyManager: id: simplekeymanager config: @@ -157,33 +130,6 @@ modules: retry_max: 3 retry_wait_min: 100ms retry_wait_max: 500ms - - # To resolve signing keys from an OAN Registry (SunbirdRC) instead, - # swap the block above for this one. Both implement RegistryLookup, so - # nothing else in the module changes. See - # pkg/plugin/implementation/oanregistry/README.md. - # - # registry: - # id: oanregistry - # config: - # # Required, and the only setting with no default. Include the API - # # version prefix -- the plugin appends /{entity}/search. Use the - # # service name, not localhost: in a container localhost is the - # # adapter itself. - # url: http://registry:8081/api/v1 - # entity: Participant - # # Tighter than dediregistry's on purpose: this runs inside - # # signature validation on every inbound message, so - # # timeout x (retry_max + 1) is time a request waits before it can - # # even be rejected. - # timeout: 2 - # retry_max: 1 - # retry_wait_min: 100ms - # retry_wait_max: 500ms - # # Omitted means caching is off. The TTL is how long a suspended - # # participant keeps verifying, so it is opt-in. Needs a cache - # # plugin configured as well. - # # cacheTTL: 60s keyManager: id: simplekeymanager config: diff --git a/config/local-beckn-one-bpp.yaml b/config/local-beckn-one-bpp.yaml index f7a849d2..64e94653 100644 --- a/config/local-beckn-one-bpp.yaml +++ b/config/local-beckn-one-bpp.yaml @@ -79,33 +79,6 @@ modules: retry_max: 3 retry_wait_min: 100ms retry_wait_max: 500ms - - # To resolve signing keys from an OAN Registry (SunbirdRC) instead, - # swap the block above for this one. Both implement RegistryLookup, so - # nothing else in the module changes. See - # pkg/plugin/implementation/oanregistry/README.md. - # - # registry: - # id: oanregistry - # config: - # # Required, and the only setting with no default. Include the API - # # version prefix -- the plugin appends /{entity}/search. Use the - # # service name, not localhost: in a container localhost is the - # # adapter itself. - # url: http://registry:8081/api/v1 - # entity: Participant - # # Tighter than dediregistry's on purpose: this runs inside - # # signature validation on every inbound message, so - # # timeout x (retry_max + 1) is time a request waits before it can - # # even be rejected. - # timeout: 2 - # retry_max: 1 - # retry_wait_min: 100ms - # retry_wait_max: 500ms - # # Omitted means caching is off. The TTL is how long a suspended - # # participant keeps verifying, so it is opt-in. Needs a cache - # # plugin configured as well. - # # cacheTTL: 60s keyManager: id: simplekeymanager config: @@ -181,33 +154,6 @@ modules: retry_max: 3 retry_wait_min: 100ms retry_wait_max: 500ms - - # To resolve signing keys from an OAN Registry (SunbirdRC) instead, - # swap the block above for this one. Both implement RegistryLookup, so - # nothing else in the module changes. See - # pkg/plugin/implementation/oanregistry/README.md. - # - # registry: - # id: oanregistry - # config: - # # Required, and the only setting with no default. Include the API - # # version prefix -- the plugin appends /{entity}/search. Use the - # # service name, not localhost: in a container localhost is the - # # adapter itself. - # url: http://registry:8081/api/v1 - # entity: Participant - # # Tighter than dediregistry's on purpose: this runs inside - # # signature validation on every inbound message, so - # # timeout x (retry_max + 1) is time a request waits before it can - # # even be rejected. - # timeout: 2 - # retry_max: 1 - # retry_wait_min: 100ms - # retry_wait_max: 500ms - # # Omitted means caching is off. The TTL is how long a suspended - # # participant keeps verifying, so it is opt-in. Needs a cache - # # plugin configured as well. - # # cacheTTL: 60s keyManager: id: simplekeymanager config: @@ -292,33 +238,6 @@ modules: retry_max: 3 retry_wait_min: 100ms retry_wait_max: 500ms - - # To resolve signing keys from an OAN Registry (SunbirdRC) instead, - # swap the block above for this one. Both implement RegistryLookup, so - # nothing else in the module changes. See - # pkg/plugin/implementation/oanregistry/README.md. - # - # registry: - # id: oanregistry - # config: - # # Required, and the only setting with no default. Include the API - # # version prefix -- the plugin appends /{entity}/search. Use the - # # service name, not localhost: in a container localhost is the - # # adapter itself. - # url: http://registry:8081/api/v1 - # entity: Participant - # # Tighter than dediregistry's on purpose: this runs inside - # # signature validation on every inbound message, so - # # timeout x (retry_max + 1) is time a request waits before it can - # # even be rejected. - # timeout: 2 - # retry_max: 1 - # retry_wait_min: 100ms - # retry_wait_max: 500ms - # # Omitted means caching is off. The TTL is how long a suspended - # # participant keeps verifying, so it is opt-in. Needs a cache - # # plugin configured as well. - # # cacheTTL: 60s cache: id: cache config: diff --git a/config/oan-provider-adapter.yaml b/config/oan-provider-adapter.yaml index 891d1a7f..1c35b4d1 100644 --- a/config/oan-provider-adapter.yaml +++ b/config/oan-provider-adapter.yaml @@ -1,13 +1,19 @@ # OAN provider adapter. # -# Serves /select synchronously: verifies the sender, resolves the capability's -# call plan from the registry, calls the provider, and answers with the mapped -# result. There is no callback -- the answer is the HTTP response. +# Serves the Beckn actions synchronously: verifies the sender, resolves the +# capability's call plan from the registry, calls the provider, and answers with +# the mapped result. There is no callback -- the answer is the HTTP response. # -# Adding a provider is three things and no Go changes to this file's shape: +# Adding a provider is three things, and none of them is a Go change here: # 1. a registry row binding "|" to a call plan -# 2. two mapping files, published at the URLs that row names -# 3. one more entry under providerSteps +# 2. one mapping file per action, published at the URL that row names +# 3. one more entry under providerSteps, and its id added to steps +# +# WHAT IS CONFIGURED HERE AND WHAT IS NOT. This file holds how to reach the +# registry, how to present credentials, and which capabilities this module +# serves. It does NOT hold where a provider lives or how long to wait for it -- +# that is the registry's ProviderSchema row, read per request. Repointing a +# provider is a registry write, not an edit here and a restart. appName: "oan-provider-adapter" log: @@ -29,37 +35,72 @@ pluginManager: modules: - name: oanProvider # A subtree, not one action. The trailing slash matters: Go's ServeMux - # treats a path without one as an exact match, so /beckn/select would mount - # that action and 404 every other. Every action lands here, and which one it - # is comes from the payload's context.action, never from the URL. - path: /beckn/ + # treats a path without one as an exact match, so "/select" would mount + # that one action and 404 every other. At "/" every action lands here. + # + # WHICH ACTION IT IS COMES FROM THE URL, not the payload. The mount path is + # stripped off the request path and what remains -- "select", "discover" -- + # is what the routing config matches on. The schema validator is the one + # exception: it is handed that same stripped path, ignores it, and reads + # context.action out of the body instead. Nothing reconciles the two, though + # a mismatch usually fails validation anyway, since two actions rarely + # accept the same body. + # + # A consequence worth knowing before mounting anything on an exact path: + # stripping "/publish" off "/publish" leaves the empty string, so a module + # mounted there sees an empty endpoint and the router rejects it unless its + # routing rule keys on "" with excludeAction set. + path: / handler: type: std role: bpp subscriberId: provider-network-vistaar.da.gov.in plugins: - # Serves both halves: the sender's signing key for validateSign, and the - # capability call plans the provider steps resolve against. + # ------------------------------------------------------------------ + # oanregistry -- the OAN Registry (SunbirdRC) client. + # + # Serves both halves of the lookup: the sender's signing key for + # validateSign, and the capability call plans the provider steps + # resolve against. Two interfaces, deliberately kept apart -- they + # answer different questions, and a failure of one does not mean the + # other. + # ------------------------------------------------------------------ registry: id: oanregistry config: + # REQUIRED, and the only key with no default. Include the API + # version prefix; the plugin appends /{entity}/search. Use the + # service name rather than localhost -- inside a container + # localhost is the adapter itself. url: http://registry:8081/api/v1 - entity: Participant - providerEntity: ProviderSchema - # cacheTTL is deliberately unset. It needs a cache plugin alongside - # it -- this client caches nothing without one -- and this sample - # runs no redis, so a TTL here would look like caching while every - # message still made its three registry calls: key lookup, binding - # search, participant search, each inside signature validation's - # budget. The plugin now says so at startup if the two are not - # configured together. + + # The two entity names, both defaulted. Only worth setting if a + # deployment renamed the schemas. + entity: Participant # default: Participant + providerEntity: ProviderSchema # default: ProviderSchema + + # Tighter than a general-purpose HTTP client on purpose. This runs + # INSIDE signature validation on every inbound message, so + # timeout x (retry_max + 1) is how long a request can wait before + # it is even allowed to be rejected. + timeout: 2 # default: 2 (seconds) + retry_max: 1 # default: 1 + retry_wait_min: 100ms # default: 100ms + retry_wait_max: 500ms # default: 500ms + + # cacheTTL is deliberately unset. It needs a cache plugin + # alongside it -- this client caches nothing without one -- and + # this sample runs no redis, so a TTL here would look like + # caching while every message still made its registry calls + # inside signature validation's budget. The plugin logs a warning + # at startup if the two are not configured together. # - # Adding a cache plugin? Read the number carefully. It is exactly - # how long a suspended participant keeps verifying, and a withdrawn + # Read the number carefully before setting it. It is exactly how + # long a suspended participant keeps verifying, and a withdrawn # capability keeps being called. # - # cacheTTL: 60s + # cacheTTL: 60s # default: unset, meaning off keyManager: id: simplekeymanager @@ -71,28 +112,156 @@ modules: signValidator: id: signvalidator - # Generic. Knows nothing about any provider; fetches, compiles and - # caches whatever the registry's mapping URLs point at. + # Base Beckn v2 schema validation against the pinned LTS spec. The + # extended layer fetches each resource's own @context and validates + # against that -- a network call per payload and a second thing that + # can fail -- so it is off, and the extendedSchema_* keys below only + # take effect if it is switched on. + schemaValidator: + id: schemav2validator + config: + type: url + location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" + cacheTTL: "3600" + extendedSchema_enabled: "false" + extendedSchema_cacheTTL: "86400" + extendedSchema_maxCacheSize: "100" + extendedSchema_downloadTimeout: "30" + extendedSchema_allowedDomains: "beckn.org,raw.githubusercontent.com" + + # ------------------------------------------------------------------ + # jsonmapper -- the JSONata mapper. + # + # Generic, and named for what it is rather than for OAN: it knows + # nothing about any provider. It fetches whatever URL the registry's + # mappings field names, compiles the JSONata, caches the compiled + # form, and runs it in both directions. + # + # Every key here is optional. + # ------------------------------------------------------------------ mapper: id: jsonmapper config: - fetchTimeout: 5s - cacheTTL: 1h - negativeTTL: 1m - - # One entry per provider capability. Each recognises its own binding key - # and passes through anything else, which is the whole dispatch - # mechanism. Credentials are named, never held: authScheme says how to - # present them, and the *Env keys name environment variables. + # How long to wait for the mapping itself. This is on the request + # path on a cache miss, so it is a ceiling on a select's latency + # the first time a mapping is used. + fetchTimeout: 5s # default: 5s + + # How long a compiled mapping is reused. The practical effect is + # how long an edit to a published mapping takes to appear. + cacheTTL: 1h # default: 1h + + # How long a FAILED fetch is remembered. Without it, a mapping + # URL that 404s is re-fetched on every single request. + negativeTTL: 1m # default: 1m + + # A ceiling on the mapping document, not the payload. 256 KiB is + # far above any realistic mapping; it is here so a wrong URL + # pointing at something enormous fails fast instead of buffering. + maxMappingBytes: 262144 # default: 262144 (256 KiB) + + # How many compiled mappings to keep. One entry per + # binding-key-and-action, so the useful floor is the number of + # capabilities times the actions each serves. + maxCacheEntries: 200 # default: 200 + + # ------------------------------------------------------------------ + # providerSteps -- one entry per provider capability. + # + # DISPATCH NEEDS NO MECHANISM. Each step builds a binding key out of + # the incoming payload -- the provider id and the capability @type it + # carries -- serves the request if that key is one of its own, and + # returns nil untouched if it is not. So several sit in one pipeline + # and each recognises its own work, with no routing table to keep in + # step with the registry. + # + # If NO step claims a payload the answer is 404 NET_ENTITY_NOT_FOUND + # -- deliberately not an ACK, which would tell the caller an answer is + # coming and leave it waiting for a callback nobody will send. + # + # CREDENTIALS ARE NAMED, NEVER HELD. authScheme says how to present + # one; the *Env keys name an ENVIRONMENT VARIABLE to read it from. No + # secret is in this file, and none is in the registry either. A + # configured credential whose variable is absent fails the request + # rather than calling the provider unauthenticated. + # ------------------------------------------------------------------ providerSteps: - id: weather config: + # REQUIRED. Comma-separated, because a plugin config value is a + # string and one provider may serve several capabilities. A + # binding key contains "|" and ":", so a comma is unambiguous. bindingKeys: "mausamgram|openagrinet:WeatherObservation" + + # Where in the payload to read the two halves of the key. Both + # default to where core-v2.0.0-lts puts them, so they are only + # worth setting for a payload that differs. Note "[]": it flattens + # an array at that segment and is the ONLY operator the walk + # understands -- the same path without it matches nothing. + # + # Set both or neither; one alone is refused at startup rather + # than left to fail as every request quietly going unserved. + # + # providerIdAt: "message.contract.commitments[].offer.provider.id" + # capabilityCodeAt: "message.contract.commitments[].resources[].resourceAttributes.@type" + + # none | basic | header | query + # + # none the upstream needs no credential + # basic HTTP basic; usernameEnv + passwordEnv + # header a named header; headerName + headerValueEnv + # query a named query parameter; queryName + queryValueEnv. + # Redacted from the URL the step logs. authScheme: basic usernameEnv: MAUSAMGRAM_USER passwordEnv: MAUSAMGRAM_X_API_KEY + # The other two schemes, for reference: + # + # authScheme: header + # headerName: X-API-Key + # headerValueEnv: MAUSAMGRAM_X_API_KEY + # + # authScheme: query + # queryName: token + # queryValueEnv: MAUSAMGRAM_TOKEN + + # A ceiling on the provider's RESPONSE, so one that streams or + # misbehaves cannot exhaust memory. The body is rejected past + # this rather than truncated -- truncated JSON would fail + # mapping with a parse error that says nothing about the cause. + # + # maxResponseBytes: 4194304 # default: 4194304 (4 MiB) + + # Declaring a step above is not enough: THIS list is what runs. A step + # that appears under providerSteps but not here never executes, and the + # request falls through to the 404 above -- which looks like a registry + # problem and is not. steps: - - validateSign # the sender's key, from the registry - - weather # resolve, map out, call, map back - - signAck # signs whatever the step answered with + - validateSign # the sender's key, from the registry + - validateSchema # the pinned Beckn v2 spec + - weather # resolve, map out, call, map back + - signAck # signs whatever the step answered with + +# ---------------------------------------------------------------------------- +# NOT CONFIGURED HERE: the call plan. +# +# Everything about how to reach a provider lives in its registry +# ProviderSchema row, read per request, one entry per Beckn action: +# +# method GET or POST +# path appended to the participant's baseUrl +# mappings the URL of the mapping file for this action +# timeoutMs per-call timeout. Omitted: 15000 +# retryMax retries after the first attempt. Omitted: 0 +# +# So a provider moving host, a slow one needing longer, or a flaky one needing +# retries are all registry writes. Nothing here changes and nothing restarts. +# +# Retry classification is not configurable at all: 4xx is permanent and is not +# retried, 5xx and transport errors are, and the backoff rises from 50ms to a +# 800ms ceiling. A non-2xx reaches the caller as an error with the provider's +# own response included, and the mapping never runs on it -- which is why an +# upstream that signals "no data" with a 4xx surfaces as a failure rather than +# an empty result. +# ---------------------------------------------------------------------------- From c94581247f1fda6042763424120a9e4205e29f13 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 11:41:19 +0530 Subject: [PATCH 050/130] refactor(mappings): echo the caller's @context instead of restating it [#1] The response half hardcoded the pack URL, so the mapping had to know which one is current and could contradict what the request actually declared. It reads it off the incoming select now -- the payload always carries @context and @type on each resource, so there is nothing to invent. Backticks because @ is an operator in JSONata. Verified by running the real response mapping against a real mock upstream response and a real select: three resources out, @context matching the request in both cases tried -- the old schemas.openagrinet.global identifier and the GitHub pack URL. --- .../mausamgram/weather-observation.select.yaml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/config/mappings/mausamgram/weather-observation.select.yaml b/config/mappings/mausamgram/weather-observation.select.yaml index 1baf2946..c1d230c6 100644 --- a/config/mappings/mausamgram/weather-observation.select.yaml +++ b/config/mappings/mausamgram/weather-observation.select.yaml @@ -24,10 +24,9 @@ # here by name and version rather than by a path, because a path pins a branch # and a branch moves. # -# @context is the canonical schemas.openagrinet.global identifier. In JSON-LD -# that is a name, not a fetch target -- it does not have to resolve today, and -# substituting a raw git URL that does would put an implementation detail on the -# wire and break every consumer when the branch is renamed. +# @context is not stated here at all. The response echoes whatever the request +# declared, so this file never has to know which pack URL is current and cannot +# contradict the caller. # # Direct mode requires observationType, source, location, generatedAt and # parameters. informationMode is what selects those requirements: a catalog @@ -158,6 +157,12 @@ response: | $selected := beckn.message.contract.commitments[0]; + /* The caller's own @context, echoed back rather than restated here. A + mapping that hardcodes it has to be reissued whenever the pack URL + moves, and it can disagree with what the request actually declared. + Backticks because @ is an operator in JSONata. */ + $ctx := beckn.message.contract.commitments[0].resources[0].resourceAttributes.`@context`; + /* Bound once because it is used twice -- for a resource's own id and for the offer's reference to it. Two copies of the same expression is how a dangling reference gets reintroduced. */ @@ -225,7 +230,7 @@ response: | consumer who validates. */ "quantity": 1, "resourceAttributes": { - "@context": "https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld", + "@context": $ctx, "@type": "openagrinet:WeatherObservation", "informationMode": "Direct", "observationType": "Forecast", From 85adfa956d2b89f76c01726f723235f1dd0cea55 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 11:41:33 +0530 Subject: [PATCH 051/130] chore(config): trim the extended-schema allowlist to the host in use [#1] beckn.org came from the upstream sample and nothing here fetches from it. An allowlist earns its keep by listing what is actually used, so it is raw.githubusercontent.com alone -- where the schema packs are published. --- config/oan-provider-adapter.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/oan-provider-adapter.yaml b/config/oan-provider-adapter.yaml index 1c35b4d1..6fdc9b13 100644 --- a/config/oan-provider-adapter.yaml +++ b/config/oan-provider-adapter.yaml @@ -127,7 +127,7 @@ modules: extendedSchema_cacheTTL: "86400" extendedSchema_maxCacheSize: "100" extendedSchema_downloadTimeout: "30" - extendedSchema_allowedDomains: "beckn.org,raw.githubusercontent.com" + extendedSchema_allowedDomains: "raw.githubusercontent.com" # ------------------------------------------------------------------ # jsonmapper -- the JSONata mapper. From 23e88c2277724757b40ee32ec7e924710e2fdfb7 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 22:19:52 +0530 Subject: [PATCH 052/130] fix(upstream): redact the credential in the form that reaches the URL [#1] redactString replaced the raw value os.Getenv returned, but authenticate puts the credential into the query through url.Values.Encode, which escapes anything outside the unreserved set. So a token containing "+", "/" or "=" -- which any base64 token routinely does -- appears in a transport error as "token=a%2Bb%2Fc%3D" and the replacement walked straight past it. The URL is logged at info level on purpose, so the credential reached the log in recoverable form. Redacts both forms now. url.QueryEscape is exact rather than a guess about which characters matter: it is the same function Encode used, so the two agree by construction. The raw pass stays because a value needing no escaping is unchanged by QueryEscape, and an error built from config rather than from the request still carries the raw form. The marker is a named constant so the two replacements cannot drift. Two tests, and I checked the new one fails without the fix rather than assuming it: with the old single pass it reports the credential surviving as `token=a%2Bb%2Fc%3Dd+e`. The token in the test carries every character Encode treats specially, including the space that becomes "+". --- .../internal/upstream/upstream.go | 23 ++++++++- .../internal/upstream/upstream_test.go | 51 +++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index d8b15fbe..7c179d55 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -60,6 +60,9 @@ const ( RetryBackoffBase = 50 * time.Millisecond RetryBackoffMax = 800 * time.Millisecond + // redactedMarker stands in for a credential in anything logged or returned. + redactedMarker = "REDACTED" + AuthSchemeNone = "none" AuthSchemeBasic = "basic" AuthSchemeHeader = "header" @@ -703,7 +706,25 @@ func (s *Step) redactString(text string) string { if value == "" { return text } - return strings.ReplaceAll(text, value, "REDACTED") + text = strings.ReplaceAll(text, value, redactedMarker) + + // Also the percent-encoded form, because that is the one that actually + // reaches a URL. authenticate puts the credential in through + // url.Values.Encode, which escapes anything outside the unreserved set -- + // so a base64 token, which routinely carries "+", "/" and "=", appears in + // the error as "a%2Bb%2Fc%3D" and a replacement of the raw value alone + // walks straight past it. Escaping what we hold is exact: it is the same + // function Encode used, so the two agree by construction rather than by a + // guess about which characters matter. + // + // Both forms rather than only the encoded one: a value needing no escaping + // is unchanged by QueryEscape, and an error that quotes the credential + // without having put it through a URL -- one built from the config rather + // than from the request -- still carries the raw form. + if encoded := url.QueryEscape(value); encoded != value { + text = strings.ReplaceAll(text, encoded, redactedMarker) + } + return text } // buildEndpoint joins the plan's base URL and path, carrying the mapped request diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go index ca22d469..9ca3a5c9 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -589,6 +589,57 @@ func TestRedactStringRemovesTheCredentialFromTheURL(t *testing.T) { } } +// A credential that percent-encodes is the case the raw-value replacement +// missed, and it is not an exotic one: base64 routinely contains "+", "/" and +// "=", and a URL-safe token contains "-" and "_". authenticate builds the query +// with url.Values.Encode, so the escaped form is what reaches the wire and the +// error text -- redacting only what os.Getenv returned walked straight past it. +func TestRedactStringRemovesThePercentEncodedCredential(t *testing.T) { + // No t.Parallel: t.Setenv forbids it. + const token = "a+b/c=d e" // every character Encode treats specially + t.Setenv("TEST_MANDI_TOKEN", token) + + step := &Step{config: &Config{ + AuthScheme: AuthSchemeQuery, + QueryName: "token", + QueryValueEnv: "TEST_MANDI_TOKEN", + }} + + // Exactly how the credential appears once authenticate has run: Encode + // escapes it, so this is the string a transport error quotes. + query := url.Values{} + query.Set("token", token) + requested := "http://host/v1/x?statecode=CG&" + query.Encode() + + got := step.redactString(requested) + if strings.Contains(got, url.QueryEscape(token)) { + t.Errorf("the encoded credential survived redaction: %s", got) + } + if strings.Contains(got, token) { + t.Errorf("the raw credential survived redaction: %s", got) + } + if !strings.Contains(got, "REDACTED") || !strings.Contains(got, "statecode=CG") { + t.Errorf("redacted url = %q, want the credential replaced and the rest intact", got) + } +} + +// A value needing no escaping must still be redacted -- QueryEscape leaves it +// alone, so the encoded pass is a no-op and the raw pass has to carry it. +func TestRedactStringStillRemovesAnUnescapedCredential(t *testing.T) { + // No t.Parallel: t.Setenv forbids it. + t.Setenv("TEST_MANDI_TOKEN", "plaintoken123") + + step := &Step{config: &Config{ + AuthScheme: AuthSchemeQuery, + QueryName: "token", + QueryValueEnv: "TEST_MANDI_TOKEN", + }} + got := step.redactString("http://host/v1/x?token=plaintoken123") + if strings.Contains(got, "plaintoken123") { + t.Errorf("the credential survived redaction: %s", got) + } +} + // Half a configuration is refused at startup, the same way the header scheme's // is: a scheme that cannot present a credential would fail on every call. func TestNewRefusesAHalfConfiguredQueryScheme(t *testing.T) { From 1759712d95dda993a3f4e170b8b74545e18ac488 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 22:34:48 +0530 Subject: [PATCH 053/130] fix(upstream): keep the provider's body and the env-var names off the wire [#1] Two things reached a network peer that had no business there. Both errors are wrapped with %w into the 502 at upstream.go:518, which becomes CodedErr.BecknError().Message and is signed and sent. The provider's response body, up to 300 bytes of it. What a provider puts in a failure body is its own business -- a stack trace, an internal hostname, a database error -- and the caller can do nothing with it. The status is the caller's business and stays; the body now goes to a warn log, where the operator is. The environment variable name, on every auth scheme. Telling a peer that MANDI_TOKEN is what this deployment reads describes the inside of somebody else's stack for no benefit: the peer cannot set it, and the fix is entirely the operator's. The wire gets the scheme that failed, which is what makes it diagnosable; the name goes to the log. MOVING THE BODY TO THE LOG MOVED THE LEAK, AND THE EXISTING TESTS CAUGHT IT. A provider that rejects a request often quotes it back -- credential and all -- which is why the body was redacted when it was in the error. The first version of this change logged explain(body) raw, so the query-string token went straight into the log line. It goes through redactString now, the same as the URL alongside it already did. Three tests asserted the old behaviour and are re-pointed rather than deleted, because each still has something to prove: - the missing-credential test now asserts the variable name is ABSENT and the scheme is present - the provider-explanation test now asserts the body is absent and the status is present, and is renamed for what it checks - the echoed-credential test asserts on the same expression the code logs, so this leak cannot move again unnoticed explain still collapses whitespace, and now has its own test -- the reason survives the move, since an indented body would spread one failure over several log lines either way. --- .../internal/upstream/upstream.go | 36 ++++++++++-- .../internal/upstream/upstream_test.go | 56 +++++++++++++++---- 2 files changed, 76 insertions(+), 16 deletions(-) diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index 7c179d55..18dd1e6b 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -602,7 +602,18 @@ func (s *Step) attempt(ctx context.Context, call model.ActionPlan, endpoint stri // treating those as failures would refuse a perfectly good exchange. 3xx // does not reach here: the client follows redirects. if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { - err := fmt.Errorf("provider returned %s: %s", resp.Status, explain(body)) + // The body is logged, not returned. It goes into a 502 that is signed + // and sent to the network caller, and what a provider puts in a failure + // body is its own business -- a stack trace, an internal hostname, a + // database error. The status is the caller's business and stays; the + // body is the operator's, and the log is where the operator looks. + // Redacted on the way to the log too. A provider that rejects a + // request often quotes it back, credential and all -- so the body is + // exactly where a query-string token turns up, and moving it from the + // error to the log would only move the leak. + log.Warnf(ctx, "upstream: provider returned %s for %s %s: %s", + resp.Status, call.Method, requested, s.redactString(explain(body))) + err := fmt.Errorf("provider returned %s", resp.Status) // 5xx and 429 are the provider asking to be tried again. Every other // 4xx is a statement about the request, which will not improve. if resp.StatusCode < http.StatusInternalServerError && resp.StatusCode != http.StatusTooManyRequests { @@ -641,25 +652,40 @@ func explain(body []byte) string { // authenticate presents this provider's credentials, read from the environment // at call time so a rotated secret takes effect without a restart. +// missingCredential reports an unset credential without naming the variable on +// the wire. +// +// The variable name is deployment configuration, and this error is wrapped into +// a 502 that is signed and returned to a network peer. Telling a peer that +// MANDI_TOKEN is what this deployment reads describes the inside of somebody +// else's stack for no benefit to the caller -- the caller cannot set it, and +// the fix is entirely the operator's. So the name goes to the log, where the +// operator is, and the wire gets the scheme that failed. +func (s *Step) missingCredential(ctx context.Context, scheme, envNames string) error { + err := fmt.Errorf("upstream: this provider's %s credential is not configured", scheme) + log.Errorf(ctx, err, "upstream: %s auth is configured but %s is not set", scheme, envNames) + return err +} + func (s *Step) authenticate(req *http.Request) error { switch s.config.AuthScheme { case AuthSchemeBasic: username, password := os.Getenv(s.config.UsernameEnv), os.Getenv(s.config.PasswordEnv) if username == "" || password == "" { - return fmt.Errorf("upstream: %s and %s must both be set for basic auth", - s.config.UsernameEnv, s.config.PasswordEnv) + return s.missingCredential(req.Context(), "basic", + s.config.UsernameEnv+" and "+s.config.PasswordEnv) } req.SetBasicAuth(username, password) case AuthSchemeHeader: value := os.Getenv(s.config.HeaderValueEnv) if value == "" { - return fmt.Errorf("upstream: %s must be set for header auth", s.config.HeaderValueEnv) + return s.missingCredential(req.Context(), "header", s.config.HeaderValueEnv) } req.Header.Set(s.config.HeaderName, value) case AuthSchemeQuery: value := os.Getenv(s.config.QueryValueEnv) if value == "" { - return fmt.Errorf("upstream: %s must be set for query auth", s.config.QueryValueEnv) + return s.missingCredential(req.Context(), "query", s.config.QueryValueEnv) } // Set rather than Add: a second copy of the parameter is not a // credential, it is an ambiguity, and which one an upstream reads is diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go index 9ca3a5c9..f926a87d 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -295,8 +295,15 @@ func TestRunDoesNotRetryAMissingCredential(t *testing.T) { if called != 0 { t.Errorf("the provider was called %d times; a credential this step cannot read never reaches it", called) } - if !strings.Contains(err.Error(), "TEST_ABSENT_USER_FOR_RETRY") { - t.Errorf("error %q should name the variable that is unset", err) + // The variable name is deployment configuration and this error is signed + // and returned to a network peer, so the name belongs in the log and not + // on the wire. The scheme stays, which is what makes it diagnosable. + if strings.Contains(err.Error(), "TEST_ABSENT_USER_FOR_RETRY") || + strings.Contains(err.Error(), "TEST_ABSENT_PASS_FOR_RETRY") { + t.Errorf("error %q must not name the environment variable", err) + } + if !strings.Contains(err.Error(), "basic") { + t.Errorf("error %q should say which auth scheme could not be presented", err) } } @@ -418,7 +425,7 @@ func TestRunReportsAnEmptyBodyRatherThanTheStatus(t *testing.T) { // then thrown away, so a 400 carrying {"message":"no data"} reached an operator // as "provider returned 400 Bad Request" and nothing else -- which is the first // thing anyone needs and the thing that makes a real provider observable. -func TestRunQuotesTheProvidersExplanation(t *testing.T) { +func TestRunKeepsTheProvidersBodyOffTheWire(t *testing.T) { t.Parallel() upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -434,13 +441,30 @@ func TestRunQuotesTheProvidersExplanation(t *testing.T) { if err == nil { t.Fatal("expected the failure to be reported") } - if !strings.Contains(err.Error(), "no data available") { - t.Errorf("error %q should carry the provider's own message", err) + // The status is the caller's business and stays. The body is not: this + // error is signed and sent to a network peer, and what a provider puts in + // a failure body -- a stack trace, an internal hostname, a database + // error -- is nobody else's. It goes to the log instead. + if strings.Contains(err.Error(), "no data available") { + t.Errorf("error %q must not carry the provider's response body", err) } - // Whitespace collapsed, so an indented body does not spread one failure - // over several log lines. - if strings.Contains(err.Error(), "\n") { - t.Errorf("error %q should have its whitespace collapsed", err) + if !strings.Contains(err.Error(), "400 Bad Request") { + t.Errorf("error %q should still name the status the provider returned", err) + } +} + +// explain still collapses whitespace, because the body it prepares now goes to +// a log line rather than an error -- an indented body would spread one failure +// over several lines either way. +func TestExplainCollapsesWhitespace(t *testing.T) { + t.Parallel() + + got := explain([]byte("{\n \"message\": \"no data available\"\n}")) + if strings.Contains(got, "\n") { + t.Errorf("explain(%q) left a newline in", got) + } + if !strings.Contains(got, "no data available") { + t.Errorf("explain = %q, want the provider's message preserved", got) } } @@ -489,8 +513,18 @@ func TestRunRedactsACredentialEchoedInABody(t *testing.T) { if strings.Contains(err.Error(), "s3cr3t") { t.Errorf("the credential leaked through the quoted body: %v", err) } - if !strings.Contains(err.Error(), "REDACTED") { - t.Errorf("error %q should show the credential was removed", err) + // The body no longer reaches the error at all, so its absence from the + // wire is not what needs proving here -- the LOG is where it goes now, and + // a provider echoing the request back is exactly where a query-string + // token turns up. Assert on the same expression the code logs, so moving + // the body from the error to the log cannot quietly move the leak with it. + echoed := `{"rejected":"token=s3cr3t"}` + logged := step.redactString(explain([]byte(echoed))) + if strings.Contains(logged, "s3cr3t") { + t.Errorf("the credential survives into the log line: %s", logged) + } + if !strings.Contains(logged, "REDACTED") { + t.Errorf("logged body = %q, want the credential replaced", logged) } } From 3342a886412504099505dcb152a796745e7390eb Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 22:39:54 +0530 Subject: [PATCH 054/130] fix(upstream): bound the registry's retry budget, and stop backoff overflowing [#1] Two findings, one mechanism: both values come from a registry row, and the second is only reachable because the first is unbounded. NO CEILING. timeoutMs and retryMax were taken from the row as given. An attempt holds a goroutine and the inbound connection for its whole timeout, and http.Server's write timeout does not cancel the request context -- so a row reading retryMax 1000, timeoutMs 60000 pins both for about seventeen hours, and a handful of such requests is the adapter. The registry is trusted to say where a provider is; that is not a reason to let one row decide how long this process is busy. MaxTimeout 30s and MaxRetryMax 5 now bound it. Clamped, not refused, and logged when it bites. A row that overreaches is a configuration mistake, and failing every request for that capability is a worse answer than serving it with a sane budget and saying so. BACKOFF OVERFLOW. `RetryBackoffBase << (attempt - 1)` on an int64-backed Duration. I measured it rather than reasoning about it: at a 50ms base the shift overflows at attempt 39, and the wrapped value is NEGATIVE -- so it passes the `> RetryBackoffMax` check, is returned, and a sleep on a negative duration returns at once. The retry loop then spins as fast as the provider can refuse. From attempt 64 it yields 0, with the same effect. And it is not monotonic: attempt 40 wraps back to a sane 800ms, so the symptom appears and disappears by attempt count. The review put the negative threshold at 59; it is 38. The mechanism was right. Doubling in a loop that stops at the ceiling replaces the shift. It cannot overflow, because it never doubles a value already at or past the ceiling. The retryMax clamp puts the old bug out of reach through call() anyway, but backoff is a package function and a ceiling somewhere else is not a property of this one. budget() is extracted as a pure function so both bounds are asserted directly, rather than needing a server that sleeps for the timeout under test. Checked the backoff test fails against the old shift before keeping it. --- .../internal/upstream/upstream.go | 72 ++++++++++++++-- .../internal/upstream/upstream_test.go | 83 +++++++++++++++++++ 2 files changed, 146 insertions(+), 9 deletions(-) diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index 18dd1e6b..dfe6c2e8 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -46,6 +46,22 @@ const ( // DefaultMaxResponseBytes caps what is read from the provider. The response // is mapped in memory, so an unbounded one is an unbounded allocation. DefaultMaxResponseBytes = 4 << 20 // 4 MiB + + // MaxTimeout and MaxRetryMax bound what a registry row may ask for. + // + // Both come from DATA, not from this deployment's config, and neither is + // cheap: an attempt holds a goroutine and the inbound connection for its + // whole timeout, and http.Server's write timeout does not cancel the + // request context. So a row reading retryMax 1000, timeoutMs 60000 pins + // both for roughly seventeen hours, and a handful of such requests is the + // adapter. The registry is trusted to say where a provider is; it is not a + // reason to let one row decide how long this process is busy. + // + // Clamped rather than refused. A row that overreaches is a configuration + // mistake, and failing every request for that capability is a worse answer + // than serving it with a sane budget and saying so in the log. + MaxTimeout = 30 * time.Second + MaxRetryMax = 5 ) // Auth schemes this step can present upstream. Credentials themselves are never @@ -464,22 +480,47 @@ func decodeBody(body []byte) (any, error) { // call makes the upstream request described by the plan, retrying within its // budget. -func (s *Step) call(ctx context.Context, baseURL string, call model.ActionPlan, mapped []byte) ([]byte, error) { - endpoint, err := buildEndpoint(baseURL, call, mapped) - if err != nil { - return nil, err - } - +// budget resolves how long one attempt may take and how many retries follow +// it, applying the registry's values within this deployment's ceilings. +// +// Pure and separate from call so both bounds can be asserted without a server +// that sleeps for the timeout it is testing. +// +// retryMax counts retries, not attempts, so the call itself is always made +// once. An absent retryMax and an explicit 0 are the same instruction. +func budget(call model.ActionPlan) (time.Duration, int) { timeout := DefaultTimeout if call.TimeoutMs > 0 { timeout = time.Duration(call.TimeoutMs) * time.Millisecond } - // retryMax counts retries, not attempts, so the call itself is always made - // once. An absent retryMax and an explicit 0 are the same instruction. + if timeout > MaxTimeout { + timeout = MaxTimeout + } + retries := DefaultRetryMax if call.RetryMax > 0 { retries = call.RetryMax } + if retries > MaxRetryMax { + retries = MaxRetryMax + } + return timeout, retries +} + +func (s *Step) call(ctx context.Context, baseURL string, call model.ActionPlan, mapped []byte) ([]byte, error) { + endpoint, err := buildEndpoint(baseURL, call, mapped) + if err != nil { + return nil, err + } + + timeout, retries := budget(call) + if d := time.Duration(call.TimeoutMs) * time.Millisecond; d > timeout { + log.Warnf(ctx, "upstream: registry asks for a %v timeout; using the %v ceiling", d, timeout) + } + if call.RetryMax > retries { + log.Warnf(ctx, "upstream: registry asks for %d retries; using the %d ceiling", + call.RetryMax, retries) + } attempts := retries + 1 var lastErr error @@ -541,7 +582,20 @@ func isPermanent(err error) bool { // no wait at all a retryMax of 5 spends its whole budget inside a couple of // milliseconds, which is not a retry so much as the same failure six times. func backoff(attempt int) time.Duration { - wait := RetryBackoffBase << (attempt - 1) + if attempt <= 1 { + return RetryBackoffBase + } + // Doubled in a loop that stops at the ceiling rather than shifted and then + // clamped. `RetryBackoffBase << (attempt - 1)` overflows int64 once the + // shift reaches 38 at a 50ms base, and the wrapped value is NEGATIVE -- so + // it passes the `> RetryBackoffMax` check, is returned, and a sleep on a + // negative duration returns immediately. The retry loop then spins as fast + // as the provider can refuse. Stopping at the ceiling cannot overflow, + // because it never doubles a value already at or past it. + wait := RetryBackoffBase + for i := 1; i < attempt && wait < RetryBackoffMax; i++ { + wait *= 2 + } if wait > RetryBackoffMax { return RetryBackoffMax } diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go index f926a87d..e22bffb9 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -12,6 +12,7 @@ import ( "strings" "sync/atomic" "testing" + "time" "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" @@ -1637,3 +1638,85 @@ func TestAsQueryHandlesAnEmptyMapping(t *testing.T) { t.Errorf("asQuery({}) = (%q, %v), want an empty query and no error", got, err) } } + +// Both bounds come from a registry row, so both are data. An attempt holds a +// goroutine and the inbound connection for its whole timeout, and the server's +// write timeout does not cancel the request context -- so retryMax 1000 with +// timeoutMs 60000 is one row deciding this process is busy for seventeen hours. +func TestBudgetClampsWhatTheRegistryAsksFor(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + timeoutMs int + retryMax int + wantTimeout time.Duration + wantRetries int + }{ + {"absent uses the contract's defaults", 0, 0, DefaultTimeout, DefaultRetryMax}, + {"within the ceilings is honoured", 2000, 3, 2 * time.Second, 3}, + {"exactly at the ceilings is honoured", int(MaxTimeout / time.Millisecond), MaxRetryMax, MaxTimeout, MaxRetryMax}, + {"a timeout past the ceiling is clamped", 600000, 0, MaxTimeout, DefaultRetryMax}, + {"retries past the ceiling are clamped", 0, 1000, DefaultTimeout, MaxRetryMax}, + {"both past the ceiling are clamped", 600000, 1000, MaxTimeout, MaxRetryMax}, + {"negative values fall back to the defaults", -1, -1, DefaultTimeout, DefaultRetryMax}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + gotTimeout, gotRetries := budget(model.ActionPlan{ + TimeoutMs: tt.timeoutMs, RetryMax: tt.retryMax, + }) + if gotTimeout != tt.wantTimeout { + t.Errorf("timeout = %v, want %v", gotTimeout, tt.wantTimeout) + } + if gotRetries != tt.wantRetries { + t.Errorf("retries = %d, want %d", gotRetries, tt.wantRetries) + } + }) + } +} + +// The shift this replaced overflowed int64 once it reached 38 at a 50ms base. +// The wrapped value is NEGATIVE, so it passed the ceiling check and was +// returned, and a sleep on a negative duration returns immediately -- the +// retry loop then spun as fast as the provider could refuse. Past 64 it +// yielded 0, with the same effect. Worse, it was not monotonic: attempt 40 +// wrapped back to a sane 800ms, so the symptom came and went by attempt count. +// +// The clamp on retryMax now keeps attempts to MaxRetryMax+1, which puts the +// overflow out of reach through call(). This is asserted anyway, because +// backoff is a package function and a ceiling somewhere else is not a +// property of this one. +func TestBackoffNeverReturnsANonPositiveDuration(t *testing.T) { + t.Parallel() + + for _, attempt := range []int{0, 1, 2, 3, 4, 5, 6, 37, 38, 39, 40, 63, 64, 65, 100, 1000} { + got := backoff(attempt) + if got <= 0 { + t.Errorf("backoff(%d) = %v; a non-positive wait makes the retry loop spin", attempt, got) + } + if got > RetryBackoffMax { + t.Errorf("backoff(%d) = %v, above the %v ceiling", attempt, got, RetryBackoffMax) + } + } +} + +// The doubling itself, which the overflow fix must not have changed. +func TestBackoffDoublesToTheCeiling(t *testing.T) { + t.Parallel() + + want := []time.Duration{ + 50 * time.Millisecond, // attempt 1 + 100 * time.Millisecond, // 2 + 200 * time.Millisecond, // 3 + 400 * time.Millisecond, // 4 + 800 * time.Millisecond, // 5, at the ceiling + 800 * time.Millisecond, // 6, held there + } + for i, w := range want { + if got := backoff(i + 1); got != w { + t.Errorf("backoff(%d) = %v, want %v", i+1, got, w) + } + } +} From 2e5e1ad5f5ae3a02ea1efbe54f08608ffbd8b6ba Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 22:44:28 +0530 Subject: [PATCH 055/130] fix(oanregistry): bound the search response read [#1] io.ReadAll on resp.Body with no ceiling, in searchRecords. The other two responses this deployment reads are both capped -- upstream.go against MaxResponseBytes, jsonmapper against MaxMappingBytes -- and this one was the odd one out. It matters more here than in either of those. searchRecords is generic and serves three call sites, and I checked where they run rather than taking the review's word for it: oanregistry.go:426 reaches it from Lookup, which is the signing-key resolution inside validateSign, on EVERY inbound message. So an unbounded read is an unbounded allocation on the request path, against a URL the sample config points at plain http. Capped at 1 MiB by default, configurable as maxResponseBytes and parsed in the cmd package like every other key. One byte past the limit is read so exceeding it can be told from meeting it exactly. Refused rather than truncated. A truncated body decodes into a JSON syntax error that says nothing about the cause -- the test asserts that specifically, not just that some error occurred. Three tests: over the limit is refused naming the limit, inside the limit is untouched, and an unset limit resolves to the default rather than to unbounded. Checked the first fails without the cap: it reports "provider record not found", which is exactly the misleading symptom the fix removes. --- .../implementation/oanregistry/cmd/plugin.go | 12 ++++ .../implementation/oanregistry/oanregistry.go | 18 ++++++ .../oanregistry/providerrecord.go | 15 ++++- .../oanregistry/providerrecord_test.go | 57 +++++++++++++++++++ 4 files changed, 101 insertions(+), 1 deletion(-) diff --git a/pkg/plugin/implementation/oanregistry/cmd/plugin.go b/pkg/plugin/implementation/oanregistry/cmd/plugin.go index efb35ca3..461432a6 100644 --- a/pkg/plugin/implementation/oanregistry/cmd/plugin.go +++ b/pkg/plugin/implementation/oanregistry/cmd/plugin.go @@ -92,6 +92,18 @@ func (o oanRegistryProvider) parseConfig(config map[string]string) (*oanregistry cfg.RetryMax = retryMax } + // Parse maxResponseBytes + if maxBytesStr, exists := config["maxResponseBytes"]; exists && maxBytesStr != "" { + maxBytes, err := strconv.ParseInt(maxBytesStr, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid maxResponseBytes value '%s': %w", maxBytesStr, err) + } + if maxBytes <= 0 { + return nil, fmt.Errorf("maxResponseBytes must be positive, got %d", maxBytes) + } + cfg.MaxResponseBytes = maxBytes + } + // Parse retry_wait_min if retryWaitMinStr, exists := config["retry_wait_min"]; exists && retryWaitMinStr != "" { retryWaitMin, err := time.ParseDuration(retryWaitMinStr) diff --git a/pkg/plugin/implementation/oanregistry/oanregistry.go b/pkg/plugin/implementation/oanregistry/oanregistry.go index dd2eb521..950829cc 100644 --- a/pkg/plugin/implementation/oanregistry/oanregistry.go +++ b/pkg/plugin/implementation/oanregistry/oanregistry.go @@ -39,6 +39,12 @@ const ( DefaultRetryMax = 1 DefaultRetryWaitMin = 100 * time.Millisecond DefaultRetryWaitMax = 500 * time.Millisecond + + // DefaultMaxResponseBytes caps a registry search response. Generous for a + // handful of records, and a ceiling rather than an expectation: the read + // happens inside signature validation on every inbound message, so an + // unbounded one is an unbounded allocation on the request path. + DefaultMaxResponseBytes = 1 << 20 // 1 MiB ) // Registry field names. They live here rather than in config because they @@ -143,6 +149,11 @@ type Config struct { RetryMax int `yaml:"retry_max" json:"retry_max"` RetryWaitMin time.Duration `yaml:"retry_wait_min" json:"retry_wait_min"` RetryWaitMax time.Duration `yaml:"retry_wait_max" json:"retry_wait_max"` + // MaxResponseBytes caps a search response. Zero means + // DefaultMaxResponseBytes; a response past it is refused rather than + // truncated, because half a JSON document fails to decode with an error + // that says nothing about the cause. + MaxResponseBytes int64 `yaml:"maxResponseBytes" json:"maxResponseBytes"` } // Client resolves participants from the OAN registry. It is safe for concurrent @@ -153,6 +164,7 @@ type Client struct { client *retryablehttp.Client cache definition.Cache cacheTTL time.Duration + maxResponseBytes int64 } // participant is the subset of a registry record this plugin reads. The @@ -309,12 +321,18 @@ func New(ctx context.Context, cache definition.Cache, cfg *Config) (*Client, fun return max } + maxResponseBytes := cfg.MaxResponseBytes + if maxResponseBytes <= 0 { + maxResponseBytes = DefaultMaxResponseBytes + } + client := &Client{ searchURL: searchURLFor(cfg.URL, entity), providerSearchURL: searchURLFor(cfg.URL, providerEntity), client: rc, cache: cache, cacheTTL: cfg.CacheTTL, + maxResponseBytes: maxResponseBytes, } closer := func() error { diff --git a/pkg/plugin/implementation/oanregistry/providerrecord.go b/pkg/plugin/implementation/oanregistry/providerrecord.go index 08895ff2..43b6946c 100644 --- a/pkg/plugin/implementation/oanregistry/providerrecord.go +++ b/pkg/plugin/implementation/oanregistry/providerrecord.go @@ -325,10 +325,23 @@ func searchRecords[T any](ctx context.Context, c *Client, tracer trace.Tracer, u } defer resp.Body.Close() - respBody, err := io.ReadAll(resp.Body) + // Bounded, like every other response this deployment reads. searchRecords + // serves the signing-key lookup as well as the two provider lookups, and + // the signing-key one runs inside validateSign on EVERY inbound message -- + // so an unbounded read here is an unbounded allocation on the request path, + // against a URL that a sample config points at plain http. + // + // One byte past the limit is read so exceeding it can be told from meeting + // it exactly, and the response is then refused rather than truncated: + // half a JSON document fails to decode with an error about syntax, which + // says nothing about the cause. + respBody, err := io.ReadAll(io.LimitReader(resp.Body, c.maxResponseBytes+1)) if err != nil { return nil, fmt.Errorf("failed to read search response: %w", err) } + if int64(len(respBody)) > c.maxResponseBytes { + return nil, fmt.Errorf("search response exceeds the %d byte limit", c.maxResponseBytes) + } if resp.StatusCode != http.StatusOK { // The body can carry registry internals, so it is logged but never // returned in the error. diff --git a/pkg/plugin/implementation/oanregistry/providerrecord_test.go b/pkg/plugin/implementation/oanregistry/providerrecord_test.go index 97cec03b..2f83315f 100644 --- a/pkg/plugin/implementation/oanregistry/providerrecord_test.go +++ b/pkg/plugin/implementation/oanregistry/providerrecord_test.go @@ -595,3 +595,60 @@ func TestProviderRecordCacheKeyIsDistinctFromTheKeyLookupCacheKey(t *testing.T) t.Errorf("provider plan cache key %q shares the signing-key namespace", cache.setKey) } } + +// searchRecords serves the signing-key lookup as well as the two provider +// lookups, and the signing-key one runs inside validateSign on every inbound +// message -- so an unbounded read here is an unbounded allocation on the +// request path. The other two reads this deployment makes, in upstream and in +// jsonmapper, have always been bounded; this one was not. +func TestSearchRefusesAResponsePastTheLimit(t *testing.T) { + const limit = 512 + + // Valid JSON, and far too much of it: the size is what is refused, not the + // shape, which is what makes the error worth reading. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `[{"osid":"1","bindingKey":"k","padding":%q}]`, strings.Repeat("x", limit*4)) + })) + defer srv.Close() + + client := newTestClient(t, srv.URL, nil, func(c *Config) { c.MaxResponseBytes = limit }) + _, err := client.ProviderRecord(context.Background(), "imd-mock|openagrinet:WeatherObservation") + if err == nil { + t.Fatal("expected an oversized search response to be refused") + } + if !strings.Contains(err.Error(), "exceeds") { + t.Errorf("error = %v, want it to name the limit rather than a decode failure", err) + } + // Refused, not truncated. A truncated body fails to decode with an error + // about JSON syntax, which says nothing about the cause. + if strings.Contains(err.Error(), "unexpected end of JSON") { + t.Errorf("error = %v; the body was truncated and then decoded, not refused", err) + } +} + +// A response inside the limit is unaffected -- the cap must not cost a byte of +// headroom to the ordinary case. +func TestSearchAcceptsAResponseInsideTheLimit(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `[]`) + })) + defer srv.Close() + + client := newTestClient(t, srv.URL, nil, func(c *Config) { c.MaxResponseBytes = 512 }) + _, err := client.ProviderRecord(context.Background(), "imd-mock|openagrinet:WeatherObservation") + // An empty result is its own not-found error, not a size complaint. + if err != nil && strings.Contains(err.Error(), "exceeds") { + t.Errorf("a small response was refused for size: %v", err) + } +} + +// An unset limit must not mean an unbounded one. +func TestNewAppliesTheDefaultResponseLimit(t *testing.T) { + client := newTestClient(t, "http://127.0.0.1:1/api/v1", nil) + if client.maxResponseBytes != DefaultMaxResponseBytes { + t.Errorf("maxResponseBytes = %d, want the %d default rather than unbounded", + client.maxResponseBytes, DefaultMaxResponseBytes) + } +} From b6283d032e89ebb3416daa2f91b6371ee5b33f04 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 22:47:07 +0530 Subject: [PATCH 056/130] fix(upstream): keep the error chain matchable through redaction [#1] redact returned errors.New(text) whenever it changed anything, which reported the right thing and threw the chain away. The redacted value is what gets %w-wrapped into the final 502, so under a query-string scheme -- and only then, since nothing else redacts -- errors.Is(err, context.DeadlineExceeded) silently stopped matching. The review was precise about why nothing failed visibly: isPermanent tests the error BEFORE redaction, so retry classification was never affected. I checked that in the loop rather than taking it on trust. fmt.Errorf("%s: %w", text, err) was the obvious repair and it undoes the redaction: %w formats the original, credential included. So redactedErr reports the redacted text from Error() and the original from Unwrap(), which keeps both properties. Verified CodedErr.Error() makes a single call rather than walking the chain, so the redacted text is still what reaches the wire. The original's text stays reachable through errors.Unwrap. That is a deliberate act by a caller who wants the cause; %v, %s and %w on the value itself all go through Error() and stay redacted. Two tests. The first asserts all four properties together -- credential gone, marker present, errors.Is matches, and both still hold after the 502 wraps it. Checked it fails on errors.New: it reports the cause lost both before and after wrapping. The second pins the identity case, that an error needing no change is returned as itself rather than as a copy. --- .../internal/upstream/upstream.go | 27 +++++++- .../internal/upstream/upstream_test.go | 61 +++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index dfe6c2e8..4ea64e2a 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -769,9 +769,34 @@ func (s *Step) redact(err error) error { if text == err.Error() { return err } - return errors.New(text) + return redactedErr{text: text, err: err} } +// redactedErr reports a redacted message while keeping the original reachable +// for errors.Is and errors.As. +// +// errors.New(text) was the obvious thing and it broke the chain: the redacted +// value is what gets %w-wrapped into the final 502, so under a query-string +// scheme -- and only then, since nothing else redacts -- errors.Is(err, +// context.DeadlineExceeded) silently stopped matching. Retry classification +// was never affected, because isPermanent tests the error before redaction, +// which is why nothing failed visibly. +// +// fmt.Errorf("%s: %w", text, err) would have restored the chain and undone the +// redaction with it: %w formats the original, credential included. Reporting +// the redacted text from Error() and the original from Unwrap() keeps both. +// +// The original's text is reachable through errors.Unwrap, which is a +// deliberate act by a caller who wants the cause -- and %v, %s and %w on the +// value itself all go through Error() and stay redacted. +type redactedErr struct { + text string + err error +} + +func (e redactedErr) Error() string { return e.text } +func (e redactedErr) Unwrap() error { return e.err } + // redactString removes a query-string credential from any text about to be // logged or returned -- an error, or the URL that was requested. // diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go index e22bffb9..63651262 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -1720,3 +1720,64 @@ func TestBackoffDoublesToTheCeiling(t *testing.T) { } } } + +// redact used to return errors.New(text), which reported the right thing and +// broke errors.Is. The redacted value is what gets %w-wrapped into the final +// 502, so under a query-string scheme -- and only then -- a caller testing for +// a timeout stopped matching. Nothing failed visibly because retry +// classification tests the error before redaction. +func TestRedactKeepsTheErrorChainMatchable(t *testing.T) { + // No t.Parallel: t.Setenv forbids it. + t.Setenv("TEST_CHAIN_TOKEN", "s3cr3t") + + step := &Step{config: &Config{ + AuthScheme: AuthSchemeQuery, + QueryName: "token", + QueryValueEnv: "TEST_CHAIN_TOKEN", + }} + + // The shape net/http produces: the cause wrapped behind text that quotes + // the whole URL, credential and all. + original := fmt.Errorf(`Get "http://host/x?token=s3cr3t": %w`, context.DeadlineExceeded) + got := step.redact(original) + + if strings.Contains(got.Error(), "s3cr3t") { + t.Errorf("the credential survived redaction: %v", got) + } + if !strings.Contains(got.Error(), "REDACTED") { + t.Errorf("redacted error = %q, want the credential replaced", got) + } + if !errors.Is(got, context.DeadlineExceeded) { + t.Errorf("errors.Is lost the cause through redaction: %v", got) + } + // And wrapping it again, which is what the 502 does, must not undo either + // property. + wrapped := fmt.Errorf("upstream: provider did not answer: %w", got) + if strings.Contains(wrapped.Error(), "s3cr3t") { + t.Errorf("the credential reappeared once wrapped: %v", wrapped) + } + if !errors.Is(wrapped, context.DeadlineExceeded) { + t.Errorf("errors.Is lost the cause once wrapped: %v", wrapped) + } +} + +// Nothing to redact must return the error itself, not a copy: an error that +// needed no change should keep its identity so == and errors.Is on the value +// both still work. +func TestRedactLeavesAnUnchangedErrorAlone(t *testing.T) { + // No t.Parallel: t.Setenv forbids it. + t.Setenv("TEST_CHAIN_TOKEN_2", "s3cr3t") + + step := &Step{config: &Config{ + AuthScheme: AuthSchemeQuery, + QueryName: "token", + QueryValueEnv: "TEST_CHAIN_TOKEN_2", + }} + original := errors.New("nothing sensitive here") + if got := step.redact(original); got != original { + t.Errorf("redact returned a different error for text it did not change: %v", got) + } + if step.redact(nil) != nil { + t.Error("redact(nil) must stay nil") + } +} From 7428a050c71dbb9a6f02298f48232e8bd4f5d1c6 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 22:49:57 +0530 Subject: [PATCH 057/130] fix(upstream): send the method in the case the RFC gives it [#1] hasBody upper-cased the method privately, which made it look case-insensitive when it is not. NewRequestWithContext transmits it verbatim, so a registry row reading `method: "post"` sent `post /path HTTP/1.1`. The body was attached correctly -- hasBody had normalised -- but nginx and most gateways answer 405 to a lowercase method, which classifies permanent and surfaces as a 502 "provider did not answer". A row that is right in every respect but its capitalisation is a bad way to spend an afternoon. Confirmed against net/http rather than assumed: "post" and "PoSt" both go out exactly as written, and only an empty method is substituted, to GET. canonicalMethod is now the one place that normalises, and both hasBody and the request go through it, so they cannot disagree again. Only known methods are rewritten -- upper-casing everything would be a new restriction on an upstream entitled to a method this list has not heard of, and net/http already refuses one that is not a valid token. Empty stays empty, since net/http documents "" as GET and hasBody already agrees it carries no body. The two log lines in attempt now report the method actually sent rather than the row's spelling, which is the point of logging it. Two tests: the table for canonicalMethod including the pass-through and empty cases, and an end-to-end one asserting what the provider sees on the request line. Checked the second fails without the fix -- it reports the provider seeing "post". --- .../internal/upstream/upstream.go | 40 ++++++++++++-- .../internal/upstream/upstream_test.go | 55 +++++++++++++++++++ 2 files changed, 90 insertions(+), 5 deletions(-) diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index 4ea64e2a..dc0e7f8e 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -619,11 +619,12 @@ func (s *Step) attempt(ctx context.Context, call model.ActionPlan, endpoint stri attemptCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - req, err := http.NewRequestWithContext(attemptCtx, call.Method, endpoint, requestBody(call.Method, mapped)) + method := canonicalMethod(call.Method) + req, err := http.NewRequestWithContext(attemptCtx, method, endpoint, requestBody(method, mapped)) if err != nil { return nil, doNotRetry(fmt.Errorf("could not build the request: %w", err)) } - if hasBody(call.Method) { + if hasBody(method) { req.Header.Set("Content-Type", "application/json") } if err := s.authenticate(req); err != nil { @@ -646,7 +647,7 @@ func (s *Step) attempt(ctx context.Context, call model.ActionPlan, endpoint stri if err != nil { return nil, fmt.Errorf("could not read the response: %w", err) } - log.Infof(ctx, "upstream: %s %s -> %s, %d bytes", call.Method, requested, resp.Status, len(body)) + log.Infof(ctx, "upstream: %s %s -> %s, %d bytes", method, requested, resp.Status, len(body)) if int64(len(body)) > s.config.MaxResponseBytes { // Asking again will not make the answer smaller. return nil, doNotRetry(fmt.Errorf("response exceeds the %d byte limit", s.config.MaxResponseBytes)) @@ -666,7 +667,7 @@ func (s *Step) attempt(ctx context.Context, call model.ActionPlan, endpoint stri // exactly where a query-string token turns up, and moving it from the // error to the log would only move the leak. log.Warnf(ctx, "upstream: provider returned %s for %s %s: %s", - resp.Status, call.Method, requested, s.redactString(explain(body))) + resp.Status, method, requested, s.redactString(explain(body))) err := fmt.Errorf("provider returned %s", resp.Status) // 5xx and 429 are the provider asking to be tried again. Every other // 4xx is a statement about the request, which will not improve. @@ -939,10 +940,39 @@ func requestBody(method string, mapped []byte) io.Reader { // hasBody reports whether a method carries a request body. func hasBody(method string) bool { - switch strings.ToUpper(method) { + switch canonicalMethod(method) { case http.MethodGet, http.MethodHead, http.MethodDelete, "": return false default: return true } } + +// canonicalMethod returns a known HTTP method in the spelling the RFC gives +// it, and anything else unchanged. +// +// hasBody used to upper-case privately, which made the method look +// case-insensitive when it is not: NewRequestWithContext transmits it verbatim, +// so a registry row reading `method: "post"` sent `post /path HTTP/1.1`. The +// body was attached correctly -- hasBody had normalised -- but nginx and most +// gateways answer 405 to a lowercase method, which classifies permanent and +// surfaces as a 502 "provider did not answer". A row that is right in every +// respect but its capitalisation is a bad way to spend an afternoon. +// +// Only known methods are rewritten. Upper-casing everything would be a new +// restriction on an upstream entitled to a method this list has not heard of, +// and net/http already refuses one that is not a valid token. +// +// An empty method is left empty: net/http documents "" as GET and substitutes +// it, and hasBody agrees that it carries no body, so the two are already +// consistent and inventing a value here would only hide where it comes from. +func canonicalMethod(method string) string { + upper := strings.ToUpper(method) + switch upper { + case http.MethodGet, http.MethodHead, http.MethodPost, http.MethodPut, + http.MethodPatch, http.MethodDelete, http.MethodConnect, + http.MethodOptions, http.MethodTrace: + return upper + } + return method +} diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go index 63651262..ad99b0f1 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -1781,3 +1781,58 @@ func TestRedactLeavesAnUnchangedErrorAlone(t *testing.T) { t.Error("redact(nil) must stay nil") } } + +// hasBody upper-cased privately, which made the method look case-insensitive +// when it is not: NewRequestWithContext transmits it verbatim, so a registry +// row reading `method: "post"` sent `post /path HTTP/1.1`. The body was +// attached correctly, and nginx answered 405 -- classified permanent, and +// surfacing as a 502 "provider did not answer". +func TestCanonicalMethodFixesTheCaseTheRowWasWrittenIn(t *testing.T) { + t.Parallel() + + tests := []struct{ in, want string }{ + {"post", http.MethodPost}, + {"PoSt", http.MethodPost}, + {"POST", http.MethodPost}, + {"get", http.MethodGet}, + {"delete", http.MethodDelete}, + {"patch", http.MethodPatch}, + // Left alone: upper-casing everything would restrict an upstream + // entitled to a method this list has not heard of. + {"FrobNicate", "FrobNicate"}, + // Empty stays empty; net/http documents "" as GET and substitutes it. + {"", ""}, + } + for _, tt := range tests { + if got := canonicalMethod(tt.in); got != tt.want { + t.Errorf("canonicalMethod(%q) = %q, want %q", tt.in, got, tt.want) + } + } +} + +// The end of it: what the provider actually receives on the request line. +func TestRunSendsTheMethodInCanonicalCase(t *testing.T) { + t.Parallel() + + var seen string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen = r.Method + fmt.Fprint(w, `{}`) + })) + defer upstream.Close() + + plan := testPlan(upstream.URL, http.MethodGet) + // A row written in the case an operator happened to type. + plan.Actions["select"] = model.ActionPlan{ + Method: "post", Path: "/x", Mappings: testMappingRef, + } + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + step := newStep(t, &stubRegistry{plan: plan}, mapper) + + if _, err := runStep(t, step, selectBody); err != nil { + t.Fatalf("runStep returned an unexpected error: %v", err) + } + if seen != http.MethodPost { + t.Errorf("the provider saw method %q, want %q", seen, http.MethodPost) + } +} From 32e84c5d096529557702b369893f9807d1299a5d Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 22:53:22 +0530 Subject: [PATCH 058/130] fix(upstream): validate the base url, and refuse dot segments and fragments [#1] The registry publishes two urls per action and only one of them was checked. verifyPath was careful about the path; nothing looked at baseUrl at all, while jsonmapper has always validated its own reference for scheme and host. WHAT THAT COST. A row reading `baseUrl: "registry:8081"` -- a scheme left off, which is the easy mistake when every other address in this stack is a compose service name -- failed inside NewRequestWithContext and arrived as 502 upstream: provider did not answer after 2 attempts: Get "registry:8081/get-daily": unsupported protocol scheme "registry" That names the provider for an error in the row describing it, and retries on the way there. It is a bad request now, on the first attempt, saying which url and why. verifyPath also refuses "." and ".." segments and a fragment. A dot segment would be resolved by net/url, so the request that left would not be the request the row described -- and the registry says which path answers an action, so a row climbing out of it is either a mistake or an attempt to reach something it does not name. A fragment is never sent, so a row carrying one describes a request that cannot be made; better refused than silently dropped by the transport, which makes the row look honoured. A dot inside a segment stays legal -- /v1/data.json and /a..b both pass -- so this refuses the traversal, not the character. Three tests: the base-url table including the review's exact case, the path cases in both directions, and an end-to-end one asserting the CLASSIFICATION, which is the actual point. Checked the last fails without the fix, and it reproduces the misleading 502 above verbatim. --- .../internal/upstream/upstream.go | 50 ++++++++++++ .../internal/upstream/upstream_test.go | 81 +++++++++++++++++++ 2 files changed, 131 insertions(+) diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index dc0e7f8e..57d2d0d5 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -836,6 +836,9 @@ func (s *Step) redactString(text string) string { // buildEndpoint joins the plan's base URL and path, carrying the mapped request // as query parameters when the method takes no body. func buildEndpoint(baseURL string, call model.ActionPlan, mapped []byte) (string, error) { + if err := verifyBaseURL(baseURL); err != nil { + return "", err + } if err := verifyPath(call.Path); err != nil { return "", err } @@ -885,6 +888,53 @@ func verifyPath(path string) error { return model.NewBadReqErr("", fmt.Errorf( "upstream: path %q has an empty segment; write it with single slashes", path)) } + // A dot segment is refused rather than resolved. The registry says which + // path answers an action, and a row that climbs out of it is either a + // mistake or an attempt to reach something the row does not name -- and + // net/url would quietly resolve it either way, so the request that left + // would not be the request the row described. + for _, segment := range strings.Split(path, "/") { + if segment == ".." || segment == "." { + return model.NewBadReqErr("", fmt.Errorf( + "upstream: path %q contains the %q segment; publish the path it resolves to instead", + path, segment)) + } + } + // A fragment is never sent, so a row carrying one describes a request that + // cannot be made. Refused here rather than silently dropped by the + // transport, which would make the row look honoured. + if strings.Contains(path, "#") { + return model.NewBadReqErr("", fmt.Errorf( + "upstream: path %q contains a fragment, which is never sent to a server", path)) + } + return nil +} + +// verifyBaseURL checks the participant's base url before it is joined to a +// path, so a row that cannot produce a request says so as a bad request rather +// than as the provider being unreachable. +// +// Without this, `baseUrl: "registry:8081"` -- a scheme left off -- failed +// inside http.NewRequestWithContext and arrived as a 502 "provider did not +// answer after 1 attempts: could not build the request". That names the +// provider for an error in the row describing it, and it is retried on the way +// there. jsonmapper has always checked its own reference this way; this is the +// same check on the other url the registry publishes. +func verifyBaseURL(baseURL string) error { + if baseURL == "" { + return model.NewBadReqErr("", errors.New("upstream: the registry publishes no base url for this provider")) + } + parsed, err := url.Parse(baseURL) + if err != nil { + return model.NewBadReqErr("", fmt.Errorf("upstream: invalid base url %q: %w", baseURL, err)) + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return model.NewBadReqErr("", fmt.Errorf( + "upstream: base url %q must be http or https", baseURL)) + } + if parsed.Host == "" { + return model.NewBadReqErr("", fmt.Errorf("upstream: base url %q names no host", baseURL)) + } return nil } diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go index ad99b0f1..1cc073c3 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -1836,3 +1836,84 @@ func TestRunSendsTheMethodInCanonicalCase(t *testing.T) { t.Errorf("the provider saw method %q, want %q", seen, http.MethodPost) } } + +// The registry publishes two urls per action and only one of them was checked. +// jsonmapper has always validated its mapping reference this way; this is the +// same check on the base url beside it. +func TestVerifyBaseURL(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + baseURL string + wantErr bool + }{ + {"http is fine", "http://provider:9100", false}, + {"https is fine", "https://provider.example.com/api", false}, + {"empty is refused", "", true}, + // The case from the review: a scheme left off. Without this check it + // failed inside NewRequestWithContext and arrived as a 502. + {"a host and port with no scheme is refused", "registry:8081", true}, + {"a bare host is refused", "provider", true}, + {"a scheme that is not http is refused", "file:///etc/passwd", true}, + {"a scheme with no host is refused", "http://", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := verifyBaseURL(tt.baseURL) + if tt.wantErr && err == nil { + t.Errorf("verifyBaseURL(%q) = nil, want an error", tt.baseURL) + } + if !tt.wantErr && err != nil { + t.Errorf("verifyBaseURL(%q) = %v, want nil", tt.baseURL, err) + } + }) + } +} + +// A dot segment would be resolved by net/url, so the request that left would +// not be the request the row described. A fragment is never sent at all. +func TestVerifyPathRefusesDotSegmentsAndFragments(t *testing.T) { + t.Parallel() + + for _, path := range []string{ + "/../admin", + "/v1/../../etc", + "/v1/./get-daily", + "/get-daily#section", + } { + if err := verifyPath(path); err == nil { + t.Errorf("verifyPath(%q) = nil, want it refused", path) + } + } + // A dot inside a segment is an ordinary character and must still pass. + for _, path := range []string{"/v1/get-daily", "/v1/data.json", "/a..b"} { + if err := verifyPath(path); err != nil { + t.Errorf("verifyPath(%q) = %v, want nil", path, err) + } + } +} + +// The classification is the point. A row that cannot produce a request is a +// bad request, not a provider that failed to answer -- and it must not be +// retried on the way to being reported. +func TestRunReportsAnUnusableBaseURLAsABadRequest(t *testing.T) { + t.Parallel() + + plan := testPlan("registry:8081", http.MethodGet) // scheme left off + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} + step := newStep(t, &stubRegistry{plan: plan}, mapper) + + _, err := runStep(t, step, selectBody) + if err == nil { + t.Fatal("expected an unusable base url to be reported") + } + var coded *model.CodedErr + if !errors.As(err, &coded) || coded.HTTPStatus() != http.StatusBadRequest { + t.Errorf("error = %v, want a bad request rather than a bad gateway", err) + } + if strings.Contains(err.Error(), "did not answer") { + t.Errorf("error = %v; a row that cannot build a request is not the provider failing", err) + } +} From 936d037c9e943e0427173d520792a7489646146f Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 22:56:37 +0530 Subject: [PATCH 059/130] fix(oanbinding): count commitments, not the values they happen to yield [#1] The arity guard read len(valuesAt(payload, paths.ProviderID)), which counts RESOLVED STRINGS. walk drops a leaf that is absent or is not a string, so a payload whose second commitment carries no provider id -- or a numeric one, or no offer at all -- yielded one value. One value read as one commitment, the guard passed, and the mapping then answered commitments[0] and dropped the other. That is precisely the outcome the comment above the guard says is refused: a confident, signed, spec-valid answer to part of what was asked. The guard was written for the right reason and measured the wrong thing. countAt walks to the array segment and reports its length, whatever the leaves beyond it do. Kept separate from valuesAt rather than folded into it, because the two answer different questions and conflating them is what caused this. One commitment that does not resolve still returns ErrNoBinding, so it passes through to the next step rather than becoming a refusal -- asserted, since that is the boundary the fix could easily have moved. Three sub-cases, one per way a commitment can fail to resolve, plus a table for countAt covering an absent path, a non-array at the marker and an empty array. Checked all three fail against the old count. Two related threads are replies rather than changes, and I checked both before concluding that: - "arity is refused before ownership is determined". Arity is capability-independent, so every step would refuse the same payload and no step is deprived of work it could have done. The 400 also says more than the 404 that passing through would produce. - "oanbinding has no logging on its refusal paths". stdHandler.go:210 already logs every pipeline error with its full text, and these errors reach it through NewBadReqErr. Logging here would duplicate that and require threading a context into a pure function for it. --- .../internal/oanbinding/oanbinding.go | 13 ++- .../internal/oanbinding/oanbinding_test.go | 104 ++++++++++++++++++ .../internal/oanbinding/paths.go | 30 +++++ 3 files changed, 143 insertions(+), 4 deletions(-) diff --git a/pkg/plugin/implementation/internal/oanbinding/oanbinding.go b/pkg/plugin/implementation/internal/oanbinding/oanbinding.go index 9891e864..b53290ef 100644 --- a/pkg/plugin/implementation/internal/oanbinding/oanbinding.go +++ b/pkg/plugin/implementation/internal/oanbinding/oanbinding.go @@ -56,15 +56,20 @@ func From(paths Paths, body []byte) (Binding, error) { // dropped, leaving the caller a confident, signed, spec-valid answer to // part of what it asked. One request maps to one call, so several is a // request this design cannot express and is refused rather than halved. - providerValues := valuesAt(payload, paths.ProviderID) - if len(providerValues) > 1 { + // Counted at the array, not over the values it yields. valuesAt returns + // resolved strings and walk drops a leaf that is absent or is not a + // string, so two commitments where one carries no provider id produced one + // value -- which read as one commitment, passed this guard, and left the + // mapping to answer commitments[0] and drop the other. Exactly the outcome + // the paragraph above says is refused. + if commitments := countAt(payload, paths.ProviderID); commitments > 1 { return Binding{}, fmt.Errorf( "oanbinding: payload carries %d commitments; one request maps to one call, "+ "so send them separately rather than have all but the first dropped", - len(providerValues)) + commitments) } - providers := distinct(providerValues) + providers := distinct(valuesAt(payload, paths.ProviderID)) types := distinct(valuesAt(payload, paths.CapabilityCode)) if len(providers) == 0 || len(types) == 0 { diff --git a/pkg/plugin/implementation/internal/oanbinding/oanbinding_test.go b/pkg/plugin/implementation/internal/oanbinding/oanbinding_test.go index 8d231aa5..1a9d0836 100644 --- a/pkg/plugin/implementation/internal/oanbinding/oanbinding_test.go +++ b/pkg/plugin/implementation/internal/oanbinding/oanbinding_test.go @@ -1,6 +1,7 @@ package oanbinding import ( + "encoding/json" "errors" "strings" "testing" @@ -326,3 +327,106 @@ func TestFromRefusesSeveralCommitments(t *testing.T) { t.Errorf("binding key = %q", binding.Key()) } } + +// The guard counted resolved provider-id VALUES, and walk drops a leaf that is +// absent or is not a string -- so a payload whose second commitment carries no +// provider id yielded one value, read as one commitment, and passed. The +// mapping then answered commitments[0] and dropped the other, which is the +// confident, signed, partial answer the guard exists to prevent. +func TestFromRefusesSeveralCommitmentsEvenWhenOneDoesNotResolve(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + payload string + }{ + { + name: "the second commitment has no provider id at all", + payload: `{"message":{"contract":{"commitments":[ + {"offer":{"provider":{"id":"mausamgram"}}, + "resources":[{"resourceAttributes":{"@type":"openagrinet:WeatherObservation"}}]}, + {"offer":{"provider":{}}, + "resources":[{"resourceAttributes":{"@type":"openagrinet:WeatherObservation"}}]} + ]}}}`, + }, + { + name: "the second commitment's provider id is not a string", + payload: `{"message":{"contract":{"commitments":[ + {"offer":{"provider":{"id":"mausamgram"}}, + "resources":[{"resourceAttributes":{"@type":"openagrinet:WeatherObservation"}}]}, + {"offer":{"provider":{"id":42}}, + "resources":[{"resourceAttributes":{"@type":"openagrinet:WeatherObservation"}}]} + ]}}}`, + }, + { + name: "the second commitment has no offer", + payload: `{"message":{"contract":{"commitments":[ + {"offer":{"provider":{"id":"mausamgram"}}, + "resources":[{"resourceAttributes":{"@type":"openagrinet:WeatherObservation"}}]}, + {"resources":[{"resourceAttributes":{"@type":"openagrinet:WeatherObservation"}}]} + ]}}}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + _, err := From(BecknV2, []byte(tt.payload)) + if err == nil { + t.Fatal("two commitments must be refused, not half answered") + } + if errors.Is(err, ErrNoBinding) { + t.Errorf("err = %v; this is a refusal, not a payload for another step", err) + } + if !strings.Contains(err.Error(), "2 commitments") { + t.Errorf("err = %v, want it to report both commitments", err) + } + }) + } +} + +// One commitment whose provider id does not resolve is not this step's work -- +// it must stay ErrNoBinding rather than becoming a refusal, so the next step +// in the pipeline still sees it. +func TestFromStillPassesThroughASingleUnresolvableCommitment(t *testing.T) { + t.Parallel() + + payload := `{"message":{"contract":{"commitments":[ + {"offer":{"provider":{}}, + "resources":[{"resourceAttributes":{"@type":"openagrinet:WeatherObservation"}}]} + ]}}}` + _, err := From(BecknV2, []byte(payload)) + if !errors.Is(err, ErrNoBinding) { + t.Errorf("err = %v, want ErrNoBinding so the payload passes through", err) + } +} + +func TestCountAt(t *testing.T) { + t.Parallel() + + const two = `{"message":{"contract":{"commitments":[{"a":1},{"b":2}]}}}` + tests := []struct { + name string + payload string + path string + want int + }{ + {"counts the array regardless of the leaf", two, BecknV2.ProviderID, 2}, + {"an absent path counts nothing", `{"message":{}}`, BecknV2.ProviderID, 0}, + {"a non-array at the marker counts nothing", + `{"message":{"contract":{"commitments":{"a":1}}}}`, BecknV2.ProviderID, 0}, + {"an empty array counts nothing", + `{"message":{"contract":{"commitments":[]}}}`, BecknV2.ProviderID, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + var payload any + if err := json.Unmarshal([]byte(tt.payload), &payload); err != nil { + t.Fatalf("bad test payload: %v", err) + } + if got := countAt(payload, tt.path); got != tt.want { + t.Errorf("countAt = %d, want %d", got, tt.want) + } + }) + } +} diff --git a/pkg/plugin/implementation/internal/oanbinding/paths.go b/pkg/plugin/implementation/internal/oanbinding/paths.go index 9120f7cb..5c47deb2 100644 --- a/pkg/plugin/implementation/internal/oanbinding/paths.go +++ b/pkg/plugin/implementation/internal/oanbinding/paths.go @@ -60,6 +60,36 @@ func valuesAt(node any, path string) []string { return walk(node, strings.Split(path, ".")) } +// countAt reports how many elements the first array segment of path holds, +// whether or not the leaf beyond it resolves to anything. +// +// valuesAt cannot answer this. It returns resolved STRINGS, and walk drops a +// leaf that is missing or is not a string -- so two commitments where one +// carries no provider id yield one value, and a count of those values reads as +// one commitment. That is the difference between refusing a request this +// design cannot express and silently answering half of it. +func countAt(node any, path string) int { + for _, segment := range strings.Split(path, ".") { + fields, ok := node.(map[string]any) + if !ok { + return 0 + } + child, present := fields[strings.TrimSuffix(segment, arrayMarker)] + if !present { + return 0 + } + if strings.HasSuffix(segment, arrayMarker) { + elements, ok := child.([]any) + if !ok { + return 0 + } + return len(elements) + } + node = child + } + return 0 +} + func walk(node any, segments []string) []string { if len(segments) == 0 { // The leaf. Only strings are binding-key material; a number or an From 8ee889d4d1591f9b6f212b6cf492b715a3248cbf Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 23:07:55 +0530 Subject: [PATCH 060/130] refactor(model): hoist servedActions onto ProviderRecord [#1] The same seven lines existed byte-identically in oanregistry and in internal/upstream. Its whole purpose is that one record reads the same way wherever it is reported, so two copies had to agree on sort order across two packages -- and one of them would eventually not, leaving the same record reading differently depending on which log line you were looking at. A method on ProviderRecord rather than a third package holding one function. It is entirely about that type, the type is new in this PR so there is no compatibility question, and both callers already had it in hand. Sorted for the reason the copies were: it goes into log lines and error messages, and map iteration would make one record read differently on each request. The test asserts that across twenty passes, because a single pass cannot tell a sorted result from a lucky iteration order. Both copies and their now-unused sort imports are gone; both callers use the method. --- pkg/model/model.go | 17 ++++++++++ pkg/model/model_test.go | 33 +++++++++++++++++++ .../internal/upstream/upstream.go | 11 +------ .../oanregistry/providerrecord.go | 11 +------ 4 files changed, 52 insertions(+), 20 deletions(-) diff --git a/pkg/model/model.go b/pkg/model/model.go index 5efb9ff5..85744272 100644 --- a/pkg/model/model.go +++ b/pkg/model/model.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "net/url" + "sort" "strconv" "strings" "time" @@ -93,6 +94,22 @@ type ProviderRecord struct { Actions map[string]ActionPlan } +// ServedActions lists the actions this record serves, sorted. +// +// A method on the type rather than a helper beside each caller: its whole +// purpose is that the same record reads the same way wherever it is reported, +// and two copies of it would have had to agree on sort order across two +// packages. Sorted because it goes into log lines and error messages, and map +// iteration would make the same record read differently on each request. +func (r *ProviderRecord) ServedActions() []string { + names := make([]string, 0, len(r.Actions)) + for action := range r.Actions { + names = append(names, action) + } + sort.Strings(names) + return names +} + // ActionPlan is how to make one action's upstream call. type ActionPlan struct { Method string diff --git a/pkg/model/model_test.go b/pkg/model/model_test.go index 3aa92080..ff19e96e 100644 --- a/pkg/model/model_test.go +++ b/pkg/model/model_test.go @@ -3,6 +3,7 @@ package model import ( "encoding/json" "errors" + "slices" "strings" "testing" ) @@ -305,3 +306,35 @@ func TestWrapExtractContextErr(t *testing.T) { } }) } + +// The reason this is one method and not a helper beside each caller: the same +// record has to read the same way wherever it is reported, and map iteration +// would make it read differently on each request. +func TestProviderRecordServedActionsIsSorted(t *testing.T) { + t.Parallel() + + record := &ProviderRecord{Actions: map[string]ActionPlan{ + "select": {}, + "catalog/publish": {}, + "confirm": {}, + "discover": {}, + }} + want := []string{"catalog/publish", "confirm", "discover", "select"} + + // Repeated, because one pass cannot tell a sorted result from a lucky map + // iteration order. + for i := 0; i < 20; i++ { + got := record.ServedActions() + if !slices.Equal(got, want) { + t.Fatalf("ServedActions() = %v, want %v", got, want) + } + } +} + +func TestProviderRecordServedActionsOnAnEmptyRecord(t *testing.T) { + t.Parallel() + + if got := (&ProviderRecord{}).ServedActions(); len(got) != 0 { + t.Errorf("ServedActions() = %v, want empty", got) + } +} diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index 57d2d0d5..05221161 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -24,7 +24,6 @@ import ( "net/url" "os" "slices" - "sort" "strconv" "strings" "time" @@ -357,7 +356,7 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { // registry mistake into a one-line fix. return model.NewBadReqErr("", fmt.Errorf( "upstream: %s does not serve action %q; it serves %s", - plan.BindingKey, action, strings.Join(servedActions(plan), ", "))) + plan.BindingKey, action, strings.Join(plan.ServedActions(), ", "))) } beckn, err := decodeBody(ctx.Body) @@ -423,14 +422,6 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { // servedActions lists the actions a capability covers, sorted so the same // record reads the same way twice. -func servedActions(plan *model.ProviderRecord) []string { - names := make([]string, 0, len(plan.Actions)) - for action := range plan.Actions { - names = append(names, action) - } - sort.Strings(names) - return names -} // buildRequest produces what the provider is sent. // diff --git a/pkg/plugin/implementation/oanregistry/providerrecord.go b/pkg/plugin/implementation/oanregistry/providerrecord.go index 43b6946c..0e859a52 100644 --- a/pkg/plugin/implementation/oanregistry/providerrecord.go +++ b/pkg/plugin/implementation/oanregistry/providerrecord.go @@ -17,7 +17,6 @@ import ( "fmt" "io" "net/http" - "sort" "strings" "time" @@ -131,7 +130,7 @@ func (c *Client) ProviderRecord(ctx context.Context, bindingKey string) (*model. } plan := toProviderRecord(binding, owner) - log.Debugf(ctx, "OAN registry resolved bindingKey=%s to %s serving %s", bindingKey, plan.BaseURL, strings.Join(servedActions(plan), ", ")) + log.Debugf(ctx, "OAN registry resolved bindingKey=%s to %s serving %s", bindingKey, plan.BaseURL, strings.Join(plan.ServedActions(), ", ")) c.cacheProviderRecord(ctx, cacheKey, plan) span.SetAttributes(telemetry.AttrErrorType.String(outcomeFound)) @@ -141,14 +140,6 @@ func (c *Client) ProviderRecord(ctx context.Context, bindingKey string) (*model. // servedActions lists the actions a plan covers, sorted so the same record logs // the same way twice. -func servedActions(plan *model.ProviderRecord) []string { - names := make([]string, 0, len(plan.Actions)) - for action := range plan.Actions { - names = append(names, action) - } - sort.Strings(names) - return names -} // refuse records a deliberate denial and returns the caller's sentinel. The // registry answered; the answer was no. From 26ff4efacfc6019d9d2e241f713ae9af257ba685 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 23:12:55 +0530 Subject: [PATCH 061/130] perf(jsonmapper): collapse concurrent misses into one fetch [#1] compiled() checked the cache, missed, and fetched -- with nothing coordinating callers. On a cold start every concurrent request for the same capability fetched the mapping over HTTP and compiled it. Measured: 25 concurrent misses made 25 fetches. The existing comment acknowledged the duplicate compile. The round trip is the larger cost and was not mentioned: it is bounded by fetchTimeout, so N requests arriving together waited N times for one document, and the publisher saw N identical reads every time a capability came up. singleflight.Group, which is already a direct dependency. The reference is re-checked inside the group, because a store can land between the miss and the turn to run, and reusing it is cheaper and more consistent than fetching a second copy. The trade is written down rather than left to be discovered: the shared call inherits the FIRST caller's context, so if that caller goes away the work is cancelled for everyone waiting. Bounded by fetchTimeout, and the losers get a cancellation they can retry rather than a wrong answer. THE REVIEW'S SHARPER POINT IS ALREADY FIXED, and I checked before writing anything. It said that at the cap remember() refuses to store, so a mapping past it would fetch and compile forever. purgeExpired() now runs BEFORE the cap is measured, and the comment above it describes exactly that failure -- introduced by 4885635, "address the review findings on PR #2". So that half of the thread is answered; this commit is only the single-flight half. Two tests. The first asserts one fetch for 25 overlapping callers, with a slow handler so they genuinely overlap, and it reports 25 without the fix. The second guards the two things single-flight could plausibly break: a later request must still be served from cache rather than refetching, and a different reference must not wait behind or inherit an unrelated one. Clean under -race. --- .../implementation/jsonmapper/jsonmapper.go | 33 ++++++++- .../jsonmapper/jsonmapper_test.go | 73 +++++++++++++++++++ 2 files changed, 104 insertions(+), 2 deletions(-) diff --git a/pkg/plugin/implementation/jsonmapper/jsonmapper.go b/pkg/plugin/implementation/jsonmapper/jsonmapper.go index 37f9c44b..d363fd3c 100644 --- a/pkg/plugin/implementation/jsonmapper/jsonmapper.go +++ b/pkg/plugin/implementation/jsonmapper/jsonmapper.go @@ -21,6 +21,8 @@ import ( "sync" "time" + "golang.org/x/sync/singleflight" + "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" @@ -167,6 +169,11 @@ type Mapper struct { mu sync.RWMutex entries map[string]cacheEntry + + // inflight collapses concurrent misses for the same reference into one + // fetch. Not part of the cache: it holds work in progress, and an entry + // that has been stored is served by entries above without reaching it. + inflight singleflight.Group } // New creates a Mapper, applying defaults for anything left unset. @@ -314,8 +321,30 @@ func (m *Mapper) compiled(ctx context.Context, mappingRef string) (cacheEntry, e return entry, entry.err } - directions, checks, err := m.fetchAndCompile(ctx, mappingRef) - return m.remember(mappingRef, directions, checks, err), err + // One fetch per reference, however many requests miss at once. + // + // Without this, a cold start sends every concurrent request for the same + // capability to fetch the mapping over HTTP and then compile it. The + // compile is the cheaper half: the round trip is bounded by fetchTimeout, + // so fifty requests arriving together waited fifty times for the same + // document instead of once, and the publisher saw fifty identical reads. + // + // The shared call inherits the FIRST caller's context, which is + // singleflight's known trade: if that caller goes away the work is + // cancelled for everyone waiting on it. Bounded here by fetchTimeout, and + // the losers see a cancellation they can retry rather than a wrong answer. + shared, err, _ := m.inflight.Do(mappingRef, func() (any, error) { + // Re-checked inside the group: a concurrent store may have landed + // between the miss above and the turn to run, and reusing it is both + // cheaper and more consistent than fetching a second copy. + if entry, found := m.cached(mappingRef); found { + return entry, entry.err + } + directions, checks, err := m.fetchAndCompile(ctx, mappingRef) + return m.remember(mappingRef, directions, checks, err), err + }) + entry, _ := shared.(cacheEntry) + return entry, err } // cached returns a live cache entry, if there is one. diff --git a/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go b/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go index 01fa51d8..09e88fea 100644 --- a/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go +++ b/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go @@ -880,3 +880,76 @@ func TestTransformIsSafeUnderConcurrentUse(t *testing.T) { } } } + +// A cold cache used to send every concurrent miss for the same reference to +// fetch the mapping and compile it. The compile is the cheaper half -- the +// round trip is bounded by fetchTimeout, so N requests arriving together +// waited N times for the same document, and the publisher saw N identical +// reads for one capability coming up. +func TestCompiledFetchesOnceForConcurrentMisses(t *testing.T) { + t.Parallel() + + var fetches atomic.Int32 + // Slow enough that the callers genuinely overlap; without single-flight + // they all get past the miss check before the first store lands. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + fetches.Add(1) + time.Sleep(50 * time.Millisecond) + fmt.Fprint(w, bothDirections) + })) + defer srv.Close() + + mapper := newTestMapper(t) + + const callers = 25 + var wg sync.WaitGroup + errs := make([]error, callers) + for i := range callers { + wg.Add(1) + go func() { + defer wg.Done() + _, errs[i] = mapper.compiled(context.Background(), srv.URL) + }() + } + wg.Wait() + + for i, err := range errs { + if err != nil { + t.Fatalf("caller %d got an error: %v", i, err) + } + } + if got := fetches.Load(); got != 1 { + t.Errorf("the mapping was fetched %d times for %d concurrent misses, want 1", got, callers) + } +} + +// Single-flight must not turn a second, later request into a second fetch -- +// that is what the cache is for -- nor make a different reference wait behind +// an unrelated one. +func TestCompiledStillCachesAndKeepsReferencesIndependent(t *testing.T) { + t.Parallel() + + var fetchesA, fetchesB atomic.Int32 + srvA := newMappingServer(t, bothDirections, &fetchesA) + defer srvA.Close() + srvB := newMappingServer(t, bothDirections, &fetchesB) + defer srvB.Close() + + mapper := newTestMapper(t) + + for range 3 { + if _, err := mapper.compiled(context.Background(), srvA.URL); err != nil { + t.Fatalf("unexpected error for A: %v", err) + } + } + if _, err := mapper.compiled(context.Background(), srvB.URL); err != nil { + t.Fatalf("unexpected error for B: %v", err) + } + + if got := fetchesA.Load(); got != 1 { + t.Errorf("reference A was fetched %d times across three sequential calls, want 1", got) + } + if got := fetchesB.Load(); got != 1 { + t.Errorf("reference B was fetched %d times, want 1 -- it must not share A's result", got) + } +} From b21fdf490b0b886e88670a098210dc658a316f12 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 23:17:10 +0530 Subject: [PATCH 062/130] fix(handler): refuse to sign a response that cannot carry a message [#1] sendResponse checked only that the body was non-empty and then wrote 200 with Content-Type application/json over whatever arrived. So a mapping whose response half is written as `$.response.temperature` rather than as an object produced `28.5`, and the adapter answered 200 application/json 28.5 and then SIGNED it. Content-Type was not even a lie -- a JSON number is a valid document -- but it is not a Beckn envelope, and a consumer looking for message.contract finds nothing and cannot tell that from a protocol change. Refused rather than passed on. A signed confident non-answer is worse than a NACK: the caller cannot retry what it does not know failed, and the signature says this adapter meant it. A mapping bug should fail where the mapping is. verifyEnvelope checks the SHAPE only -- that the body is a non-empty JSON object. Which members belong in a response is the spec's business and the schema validator's, and this runs on every answer. What it catches is the class of mapping mistake that yields a scalar, an array or null: valid JSON that cannot carry a message however it is read. The NACK carries a plain error on purpose, so nackBecknError's default branch turns it into a generic 500. The caller learns the answer failed without being handed the internals of a mapping it does not own; the detail goes to the log line above it, where the operator is. Twelve cases: seven refused, three accepted -- including a member the spec does not define, since the shape is all this checks -- plus two behavioural tests. The scalar one fails without the fix on both counts, reporting a 200 and the scalar reaching the wire. The other asserts a real envelope is written byte-for-byte with its content type, so the check cannot cost the ordinary path. --- core/module/handler/responsestep.go | 36 ++++++++++ core/module/handler/responsestep_test.go | 90 ++++++++++++++++++++++++ 2 files changed, 126 insertions(+) diff --git a/core/module/handler/responsestep.go b/core/module/handler/responsestep.go index e5cce064..a2f32686 100644 --- a/core/module/handler/responsestep.go +++ b/core/module/handler/responsestep.go @@ -45,9 +45,45 @@ func sendResponse(ctx *model.StepContext, w http.ResponseWriter) []byte { if len(ctx.ResponseBody) == 0 { return sendAck(ctx, w) } + // A step's answer has to be an envelope, and only the length was checked. + // A mapping whose response half is written as `$.response.temperature` + // rather than as an object produces `28.5`, which is valid JSON -- so + // Content-Type was not a lie -- and the adapter answered 200 with it and + // then SIGNED it. A consumer looking for message.contract finds nothing + // and cannot tell that from a protocol change. + // + // Refused rather than passed on, because a signed confident non-answer is + // worse than a NACK: the caller cannot retry what it does not know failed, + // and the signature says this adapter meant it. A mapping bug should fail + // where the mapping is, and the NACK names the step so it is findable. + if err := verifyEnvelope(ctx.ResponseBody); err != nil { + log.Errorf(ctx, err, "a step produced a response that is not a Beckn envelope; refusing to sign it") + // A plain error on purpose: nackBecknError's default branch turns it + // into a generic 500, so the caller learns the answer failed without + // being handed the internals of a mapping it does not own. The detail + // is in the log line above, where the operator is. + return sendNack(ctx, w, err) + } return writeJSONResponse(ctx, w, ctx.ResponseBody) } +// verifyEnvelope checks that a step's answer is a JSON object. +// +// Only the shape, not the contents: which members belong in a response is the +// spec's business and the schema validator's, and this runs on every answer. +// What it catches is the class of mapping mistake that yields a scalar or an +// array -- valid JSON that cannot carry a Beckn message however it is read. +func verifyEnvelope(body []byte) error { + var envelope map[string]json.RawMessage + if err := json.Unmarshal(body, &envelope); err != nil { + return fmt.Errorf("response body is not a JSON object: %w", err) + } + if len(envelope) == 0 { + return errors.New("response body is an empty JSON object, so it carries no message") + } + return nil +} + // writeJSONResponse writes body as a 200 JSON response, reporting what it wrote. func writeJSONResponse(ctx context.Context, w http.ResponseWriter, body []byte) []byte { w.Header().Set("Content-Type", "application/json") diff --git a/core/module/handler/responsestep_test.go b/core/module/handler/responsestep_test.go index 18b86f3d..82a496f6 100644 --- a/core/module/handler/responsestep_test.go +++ b/core/module/handler/responsestep_test.go @@ -1079,3 +1079,93 @@ func TestInitSteps_ValidateAckSignAppendsToResponseSteps(t *testing.T) { t.Errorf("expected 1 response step, got %d", len(h.responseSteps)) } } + +// sendResponse checked only that the body was non-empty, so a mapping whose +// response half is written as `$.response.temperature` rather than as an +// object produced `28.5` -- valid JSON, so Content-Type was not a lie -- and +// the adapter answered 200 with it and then signed it. A consumer looking for +// message.contract finds nothing and cannot tell that from a protocol change. +func TestVerifyEnvelopeRefusesWhatCannotCarryAMessage(t *testing.T) { + t.Parallel() + + refused := map[string]string{ + "a bare number": `28.5`, + "a bare string": `"no data"`, + "a bare boolean": `true`, + "null": `null`, + "an array": `[{"message":{}}]`, + "an empty object": `{}`, + "not json at all": `28.5 and then some`, + } + for name, body := range refused { + t.Run(name, func(t *testing.T) { + t.Parallel() + if err := verifyEnvelope([]byte(body)); err == nil { + t.Errorf("verifyEnvelope(%s) = nil, want it refused", body) + } + }) + } + + accepted := map[string]string{ + "a full envelope": `{"context":{"action":"on_select"},"message":{"contract":{}}}`, + "one member is enough": `{"message":{}}`, + // The shape is all this checks. Which members belong in a response is + // the spec's business and the schema validator's. + "an unexpected member": `{"whatever":1}`, + } + for name, body := range accepted { + t.Run(name, func(t *testing.T) { + t.Parallel() + if err := verifyEnvelope([]byte(body)); err != nil { + t.Errorf("verifyEnvelope(%s) = %v, want nil", body, err) + } + }) + } +} + +// The behaviour, not just the check: a step that produced something unusable +// must NACK rather than be signed and sent. A signed confident non-answer is +// worse than a NACK, because the caller cannot retry what it does not know +// failed and the signature says this adapter meant it. +func TestSendResponseNacksAScalarInsteadOfSigningIt(t *testing.T) { + t.Parallel() + + ctx := makeStepCtx("2.0.0", "msg-1", "sub-1", "") + ctx.ResponseBody = []byte(`28.5`) + + w := httptest.NewRecorder() + written := sendResponse(ctx, w) + + if w.Code == http.StatusOK { + t.Errorf("status = %d; a body that cannot carry a message must not be a 200", w.Code) + } + if string(written) == "28.5" { + t.Error("the scalar was written to the wire unchanged") + } + // And what is sent instead is a NACK the caller can act on. + if !strings.Contains(w.Body.String(), string(model.StatusNACK)) { + t.Errorf("body = %s, want a NACK", w.Body.String()) + } +} + +// A real envelope is untouched -- the check must not cost the ordinary path. +func TestSendResponseWritesAnEnvelopeUnchanged(t *testing.T) { + t.Parallel() + + const body = `{"context":{"action":"on_select"},"message":{"contract":{}}}` + ctx := makeStepCtx("2.0.0", "msg-1", "sub-1", "") + ctx.ResponseBody = []byte(body) + + w := httptest.NewRecorder() + written := sendResponse(ctx, w) + + if w.Code != http.StatusOK { + t.Errorf("status = %d, want 200", w.Code) + } + if string(written) != body { + t.Errorf("written = %s, want the body unchanged", written) + } + if got := w.Header().Get("Content-Type"); got != "application/json" { + t.Errorf("Content-Type = %q, want application/json", got) + } +} From 8a17ba62cfdf2e3167f26b1a39fdb94d0522500f Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 23:20:09 +0530 Subject: [PATCH 063/130] docs(mappings): correct two claims about the pack that were wrong [#1] Comments only. No behaviour changes, because the fields they describe are the pack's to settle and that is somebody else's work -- but a comment that states something false about the pack is ours, and it will mislead the next person to read it. WHAT WAS WRONG. The aggregation note said "the pack sets no additionalProperties, so it validates; it is simply not governed." WeatherObservation v0.1 closes parameters.items with `additionalProperties: false`, so the field is refused outright. The comment had the consequence backwards, which is the worst kind of stale comment: it tells a reader the thing is fine. It also now says why the field is kept anyway. The pack's `parameter` enum has no minimum or maximum variants, so a day's tmin and tmax arrive as two identical Temperature readings with nothing to tell them apart. Dropping the field would make the answer conformant and useless. WHAT WAS MISSING. The Point-location check carried no note at all, and it requires a field the pack's OnDemand branch EXCLUDES -- so a request that conforms is refused here, and one that passes here does not conform. Nothing breaks today only because the exclusion sits under if/then, which the validator parses and never evaluates; that is an accident to rely on rather than a design, and it is now written down as such. Both notes name the resolution as a pack change and say what this mapping does when it lands, so the next reader knows this is a known disagreement rather than an oversight. coverageAreas is named as the likely home for the point: it is inherited from AgricultureResource, is not excluded in OnDemand, and accepts a GeoJSON geometry. --- .../weather-observation.select.yaml | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/config/mappings/mausamgram/weather-observation.select.yaml b/config/mappings/mausamgram/weather-observation.select.yaml index c1d230c6..b5aea57f 100644 --- a/config/mappings/mausamgram/weather-observation.select.yaml +++ b/config/mappings/mausamgram/weather-observation.select.yaml @@ -44,12 +44,24 @@ # the id that was asked for, and leaving it would point the offer at something # that appears nowhere in the answer. # -# ONE FIELD HERE IS NOT IN THE PACK, deliberately. The pack sets no -# additionalProperties, so it validates; it is simply not governed. +# ONE FIELD HERE IS NOT IN THE PACK, and this does NOT validate against it. +# An earlier version of this comment said the pack set no additionalProperties. +# It does: WeatherObservation v0.1 closes parameters.items with +# `additionalProperties: false`, so the field below is refused, not merely +# ungoverned. # -# aggregation The pack's parameter entry is parameter/value/unit only. This +# aggregation The pack's parameter entry is parameter/value/unit only, and +# its `parameter` enum has no minimum or maximum variants. This # provider reports a minimum AND a maximum for temperature and -# humidity, which are indistinguishable without it. +# humidity every day, so without this field the two arrive as +# two identical Temperature readings and a consumer cannot tell +# which is which. +# +# It is kept because dropping it loses information the pack cannot express any +# other way -- removing it would make the answer conformant and useless. The +# resolution is a pack change, an aggregation field or enum variants, and this +# mapping follows once that lands. Nothing validates a response today, so it +# does not bite until a consumer checks. # # Fields that are the same for every day -- the point, the source, the # observation type -- sit once at the top. Only what varies per day repeats. @@ -84,6 +96,23 @@ required: ) message: "this capability needs a Point location; the provider forecasts one point at a time" + # NOTE, and it is not a small one: the pack and this check disagree today. + # WeatherObservation v0.1's OnDemand branch does not leave `location` + # optional -- it EXCLUDES it, alongside observationType, source, generatedAt, + # observedAt, modelRunAt, validity and parameters. So a request conforming to + # the pack carries no location and this check refuses it, while a request + # satisfying this check does not conform. + # + # Nothing breaks in practice: the exclusion sits under if/then, which the + # validator parses and never evaluates. That is an accident to rely on rather + # than a design. + # + # The pack has no third mode for a question -- OnDemand describes what a + # provider can obtain, Direct describes an obtained reading, and a request is + # neither. Resolving it is a pack change. `coverageAreas` is inherited from + # AgricultureResource, is NOT excluded in OnDemand, and accepts a GeoJSON + # geometry, so it is the likely home for the point once that is settled. + # The request half decides what the provider is asked for. Whatever it produces # IS the request: query parameters for a method with no body, a body for one that # takes it. From 3806f168eb4b9c8cc99250b05ef64dec3b681429 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 23:23:18 +0530 Subject: [PATCH 064/130] docs(config): placeholder the registry url [#1] http://registry:8081/api/v1 was a working value for exactly one deployment shape. "registry" resolves only inside a Docker network that happens to name the service that way -- not in Kubernetes, not in a local run, not against a hosted registry -- so a reader who copied it into any of those got a DNS failure on every inbound message, inside signature validation. Same treatment as the subscriber id, and for the same reason: a reference config should show what a value looks like without presenting one deployment's as everyone's. Three worked examples sit beside it -- compose, kubernetes, hosted -- so the placeholder still teaches the shape. Kept the warning that it must be a hostname the adapter itself can resolve rather than localhost. That is the mistake the original comment existed to prevent, and it survives the change. --- config/oan-provider-adapter.yaml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/config/oan-provider-adapter.yaml b/config/oan-provider-adapter.yaml index 6fdc9b13..025956bc 100644 --- a/config/oan-provider-adapter.yaml +++ b/config/oan-provider-adapter.yaml @@ -70,10 +70,24 @@ modules: id: oanregistry config: # REQUIRED, and the only key with no default. Include the API - # version prefix; the plugin appends /{entity}/search. Use the - # service name rather than localhost -- inside a container - # localhost is the adapter itself. - url: http://registry:8081/api/v1 + # version prefix; the plugin appends /{entity}/search. + # + # A placeholder rather than a working value, because the + # working value is per-deployment. The one that used to be + # here resolved only inside a Docker network that happened to + # name the service "registry" -- not in Kubernetes, not in a + # local run, not against a hosted registry -- so a reader who + # copied it into any of those got a DNS failure on every + # inbound message, inside signature validation. + # + # Whatever goes here, use a hostname the ADAPTER can resolve, + # not localhost: inside a container localhost is the adapter + # itself. + # + # compose http://registry:8081/api/v1 + # kubernetes http://registry.oan.svc.cluster.local:8081/api/v1 + # hosted https://registry.example.org/api/v1 + url: <> # The two entity names, both defaulted. Only worth setting if a # deployment renamed the schemas. From 2be8ce6e648a6ab858065f042c3e8dde3fe6f519 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Tue, 8 Sep 2026 13:20:51 +0530 Subject: [PATCH 065/130] fix(jsonmapper): serialise mapping evaluation across the package, not per mapping [#1] The per-mapping lock was too narrow, and its comment said so out loud: "the lock is per mapping, so different mappings still run in parallel". That is the race. Evaluate mutates more than the expression it is called on -- the library keeps its built-in functions in a package-level frame, and applying one writes token and position onto that shared *Function for error reporting. Every mapping uses built-ins, so two different mappings evaluating at once write the same fields. A provider adapter serving both its capabilities concurrently does exactly that. Measured rather than reasoned about: eight goroutines, each with its own jsonata.OpenLatest() instance AND its own compiled expression, racing on an expression shaped like the shipped mappings. Separate instances are not separate state, so nothing narrower than package scope is sufficient. TestTransformIsSafeUnderConcurrentUse already ran 20 goroutines over 3 mappings and stayed green, because its fixtures are field lookups only and the shared writes happen when a function is APPLIED. The new test binds functions to variables and passes one to $filter by name, as the shipped mappings do, and gives each goroutine a distinct reference so nothing but package-level state is shared. It reports 12 races with the old lock and none with this one. Cost is bounded: evaluation is ~22us and no longer overlaps, while the upstream HTTP call each mapped request goes on to make is outside the lock and dominates. The real fix is upstream -- the shared writes are error-reporting metadata that belongs on the call, not on the function. reqmapper and schemaversionmediator evaluate JSONata too and have the same exposure. Not touched here: separate plugins, separate owners, and a lock in this package cannot reach across a .so boundary. --- .../implementation/jsonmapper/jsonmapper.go | 59 ++++++++++++------ .../jsonmapper/jsonmapper_test.go | 62 +++++++++++++++++++ 2 files changed, 101 insertions(+), 20 deletions(-) diff --git a/pkg/plugin/implementation/jsonmapper/jsonmapper.go b/pkg/plugin/implementation/jsonmapper/jsonmapper.go index d363fd3c..cefbdb16 100644 --- a/pkg/plugin/implementation/jsonmapper/jsonmapper.go +++ b/pkg/plugin/implementation/jsonmapper/jsonmapper.go @@ -103,20 +103,42 @@ type Config struct { MaxCacheEntries int `yaml:"maxCacheEntries" json:"maxCacheEntries"` } +// evaluating serialises every Evaluate in this package, across all mappings. +// +// It has to be this wide. Evaluate mutates more than the expression it is +// called on: the library keeps its built-in functions in a package-level frame +// (v206's staticFrame), and applying one writes token and position onto that +// shared *Function for error reporting. Every mapping uses built-ins, so any +// two concurrent evaluations race -- including two DIFFERENT mappings, which a +// per-mapping lock explicitly allowed to run in parallel. That was this code's +// previous shape, and it was wrong. +// +// Measured, not reasoned about: eight goroutines, each with its own +// jsonata.OpenLatest() instance and its own compiled expression, evaluating an +// expression shaped like the shipped mappings, produce race reports under +// -race. Separate instances are not separate state, so nothing narrower than +// package scope is sufficient. +// +// The cost is real and bounded: mapping evaluation no longer overlaps, at ~22us +// a call, while the upstream HTTP request each mapped call goes on to make is +// outside this lock and dominates. If it ever does matter, the fix is upstream +// -- the shared writes are error-reporting metadata that could be carried on +// the call rather than the function -- not a narrower lock here. +// +// reqmapper and schemaversionmediator evaluate JSONata too and have the same +// exposure. Not addressed here: they are separate plugins with their own +// owners, and this lock cannot reach across a .so boundary anyway. +var evaluating sync.Mutex + // cacheEntry is one compiled mapping, or the failure that stopped it compiling. // Failures are cached too, which is the whole point of the negative TTL. // -// The mutex guards evaluation, not the entry: jsonata.Expression.Evaluate -// mutates the expression it is called on -- it binds into the expression's own -// frame -- so one compiled mapping cannot serve two requests at once. Confirmed -// with the race detector, not assumed. +// Evaluation is serialised process-wide by evaluating, above -- see there for +// why it cannot be per mapping. // // Evaluating under a lock rather than compiling per request is the cheaper // trade by a wide margin: evaluation is ~22us against ~184us to compile, and -// both are dwarfed by the upstream call the mapped request goes on to make. The -// lock is per mapping, so different mappings still run in parallel. A pool of -// compiled expressions would remove even that, and is the upgrade if one -// mapping ever becomes hot enough to matter. +// both are dwarfed by the upstream call the mapped request goes on to make. type cacheEntry struct { // directions holds the compiled halves the file carries. A file is fetched // and compiled as a whole, so both are ready after the first request for @@ -136,7 +158,6 @@ type cacheEntry struct { // taking the halves down with it. type compiledRequirement struct { expression jsonata.Expression - evaluating *sync.Mutex message string err error } @@ -151,7 +172,6 @@ type compiledRequirement struct { // file could not be read" stay different answers. type compiledMapping struct { expression jsonata.Expression - evaluating *sync.Mutex err error } @@ -275,10 +295,10 @@ func (m *Mapper) Verify(ctx context.Context, mappingRef string, input any) error return precondition.err } - // See compiledMapping: Evaluate mutates the expression it is called on. - precondition.evaluating.Lock() + // See evaluating: serialised across the package, not per expression. + evaluating.Lock() result, evalErr := precondition.expression.Evaluate(document, nil) - precondition.evaluating.Unlock() + evaluating.Unlock() if evalErr != nil { log.Errorf(ctx, evalErr, "JSON mapping %s precondition failed to evaluate: %v", mappingRef, evalErr) return model.NewBadReqErr(codeAdaptationFailed, fmt.Errorf( @@ -468,7 +488,6 @@ func (m *Mapper) compileRequirement(ctx context.Context, mappingRef string, decl } return &compiledRequirement{ expression: expression, - evaluating: &sync.Mutex{}, message: declared.Message, } } @@ -485,7 +504,7 @@ func (m *Mapper) compileMapping(ctx context.Context, mappingRef string, directio log.Errorf(ctx, err, "JSON mapper could not compile the %s half of %s: %v", direction, mappingRef, err) return &compiledMapping{err: fmt.Errorf("jsonmapper: mapping %q %s half failed to compile: %w", mappingRef, direction, err)} } - return &compiledMapping{expression: expression, evaluating: &sync.Mutex{}} + return &compiledMapping{expression: expression} } // fetch retrieves a mapping's bytes, bounded in both time and size. @@ -581,12 +600,12 @@ func (m *Mapper) evaluate(ctx context.Context, mapping *compiledMapping, mapping return nil, fmt.Errorf("jsonmapper: mapping %q: %w", mappingRef, err) } - // See compiledMapping: Evaluate mutates the expression, so one half serves - // one request at a time. The other half is unaffected, and marshalling above - // is deliberately outside the lock. - mapping.evaluating.Lock() + // See evaluating: serialised across the package, because the library's + // shared built-ins make even two different mappings unsafe to overlap. + // Marshalling above is deliberately outside the lock. + evaluating.Lock() result, err := mapping.expression.Evaluate(document, nil) - mapping.evaluating.Unlock() + evaluating.Unlock() if err != nil { log.Errorf(ctx, err, "JSON mapping %s %s half failed to evaluate: %v", mappingRef, direction, err) wrapped := fmt.Errorf("mapping %q %s half could not be applied: %w", mappingRef, direction, err) diff --git a/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go b/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go index 09e88fea..667bff17 100644 --- a/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go +++ b/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go @@ -953,3 +953,65 @@ func TestCompiledStillCachesAndKeepsReferencesIndependent(t *testing.T) { t.Errorf("reference B was fetched %d times, want 1 -- it must not share A's result", got) } } + +// namedFunctionMapping is shaped like the shipped mappings: it binds functions +// to variables and passes one to $filter by name. That matters, because the +// library's shared writes happen when a function is APPLIED -- a mapping of +// only field lookups never reaches them, which is why +// TestTransformIsSafeUnderConcurrentUse ran green over a real race for as long +// as its fixtures stayed simple. +const namedFunctionMapping = `request: | + ( + $ok := function($v) { $exists($v) and $v != "" }; + { "txn": $ok(beckn.context.transactionId) ? beckn.context.transactionId : "none" } + ) + +response: | + ( + $ok := function($r) { $exists($r) }; + $tag := function($v) { $exists($v) ? $lowercase($v) }; + { + "txn": beckn.context.transactionId, + "kept": $count($filter([response.fcstday1.rain, 1, 2], $ok)), + "tag": $tag("MM") + } + ) +` + +// Two DIFFERENT mappings, evaluated at the same time. This is the case the +// per-mapping lock deliberately allowed to run in parallel, and the library's +// built-ins are package-level state, so it raced: applying $exists or $count +// writes error-reporting fields onto one shared *Function. A provider adapter +// serving both its capabilities at once does exactly this. +func TestConcurrentDifferentMappingsDoNotRace(t *testing.T) { + t.Parallel() + + srv := newMappingServer(t, namedFunctionMapping, nil) + defer srv.Close() + + mapper := newTestMapper(t) + errs := make(chan error, 24) + var wg sync.WaitGroup + for i := 0; i < 24; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + direction, input := definition.DirectionRequest, requestInput() + if i%2 == 1 { + direction, input = definition.DirectionResponse, responseInput() + } + // A distinct reference per goroutine: distinct cache entries, so + // nothing but package-level state is shared between them. + _, err := mapper.Transform(context.Background(), + fmt.Sprintf("%s/distinct-%d.yaml", srv.URL, i), direction, input) + errs <- err + }(i) + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Errorf("concurrent Transform() over distinct mappings failed: %v", err) + } + } +} From 72039b19e021bf965c9d57e9a565368da8e3d63f Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Tue, 8 Sep 2026 15:05:13 +0530 Subject: [PATCH 066/130] fix(handler): refuse a non-envelope answer before the signer runs, not after [#1] The envelope check landed one step too late, on the wrong side of signing. It sat in sendResponse, which ServeHTTP calls AFTER the response-step loop -- and ackSigner is one of those steps. So a step answering `28.5` was signed, then refused, and the NACK went out under a Signature header computed over the scalar. A peer verifying that sees a digest mismatch, so a mapping bug reached it as suspected tampering rather than as the 500 this code intends. The 404 guard immediately above already had this right, and says why in its own comment: "Checked before the response steps rather than after, because ackSigner signs the body it expects to be written; NACKing later would ship a signature over the ACK with a NACK body." The refusal now sits beside it, on the same side of signing, and signs the NACK through the same h.signNackResponse path. Reproduced before fixing: signed "28.5" while sending {"message":{"status":"NACK",...}}. The test moves up with it. TestSendResponseNacksAScalarInsteadOfSigningIt called sendResponse directly, where no signer exists, so it could not have seen this -- it checked the refusal and not what the refusal shipped. The replacement drives ServeHTTP with a real ack signer and asserts on the bytes signed, because mockSigner returns a fixed signature string whatever it is given, so the header text alone cannot tell two bodies apart. --- core/module/handler/responsestep.go | 22 +++------------------ core/module/handler/responsestep_test.go | 25 ------------------------ core/module/handler/stdHandler.go | 22 +++++++++++++++++++++ 3 files changed, 25 insertions(+), 44 deletions(-) diff --git a/core/module/handler/responsestep.go b/core/module/handler/responsestep.go index a2f32686..a8835b18 100644 --- a/core/module/handler/responsestep.go +++ b/core/module/handler/responsestep.go @@ -45,25 +45,9 @@ func sendResponse(ctx *model.StepContext, w http.ResponseWriter) []byte { if len(ctx.ResponseBody) == 0 { return sendAck(ctx, w) } - // A step's answer has to be an envelope, and only the length was checked. - // A mapping whose response half is written as `$.response.temperature` - // rather than as an object produces `28.5`, which is valid JSON -- so - // Content-Type was not a lie -- and the adapter answered 200 with it and - // then SIGNED it. A consumer looking for message.contract finds nothing - // and cannot tell that from a protocol change. - // - // Refused rather than passed on, because a signed confident non-answer is - // worse than a NACK: the caller cannot retry what it does not know failed, - // and the signature says this adapter meant it. A mapping bug should fail - // where the mapping is, and the NACK names the step so it is findable. - if err := verifyEnvelope(ctx.ResponseBody); err != nil { - log.Errorf(ctx, err, "a step produced a response that is not a Beckn envelope; refusing to sign it") - // A plain error on purpose: nackBecknError's default branch turns it - // into a generic 500, so the caller learns the answer failed without - // being handed the internals of a mapping it does not own. The detail - // is in the log line above, where the operator is. - return sendNack(ctx, w, err) - } + // The envelope check is NOT here. It is in ServeHTTP, ahead of the response + // steps, because ackSigner is one of those steps: refusing at this point + // means the Signature header is already set, over the body being refused. return writeJSONResponse(ctx, w, ctx.ResponseBody) } diff --git a/core/module/handler/responsestep_test.go b/core/module/handler/responsestep_test.go index 82a496f6..a64df2da 100644 --- a/core/module/handler/responsestep_test.go +++ b/core/module/handler/responsestep_test.go @@ -1123,31 +1123,6 @@ func TestVerifyEnvelopeRefusesWhatCannotCarryAMessage(t *testing.T) { } } -// The behaviour, not just the check: a step that produced something unusable -// must NACK rather than be signed and sent. A signed confident non-answer is -// worse than a NACK, because the caller cannot retry what it does not know -// failed and the signature says this adapter meant it. -func TestSendResponseNacksAScalarInsteadOfSigningIt(t *testing.T) { - t.Parallel() - - ctx := makeStepCtx("2.0.0", "msg-1", "sub-1", "") - ctx.ResponseBody = []byte(`28.5`) - - w := httptest.NewRecorder() - written := sendResponse(ctx, w) - - if w.Code == http.StatusOK { - t.Errorf("status = %d; a body that cannot carry a message must not be a 200", w.Code) - } - if string(written) == "28.5" { - t.Error("the scalar was written to the wire unchanged") - } - // And what is sent instead is a NACK the caller can act on. - if !strings.Contains(w.Body.String(), string(model.StatusNACK)) { - t.Errorf("body = %s, want a NACK", w.Body.String()) - } -} - // A real envelope is untouched -- the check must not cost the ordinary path. func TestSendResponseWritesAnEnvelopeUnchanged(t *testing.T) { t.Parallel() diff --git a/core/module/handler/stdHandler.go b/core/module/handler/stdHandler.go index 39c02063..62ecd53b 100644 --- a/core/module/handler/stdHandler.go +++ b/core/module/handler/stdHandler.go @@ -247,6 +247,28 @@ func (h *stdHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + // Checked here for the same reason as the 404 above, and it is the + // same failure: ackSigner signs the body it expects to be written, + // so refusing after the response steps ships a Signature computed + // over the answer we just rejected, with a NACK body under it. A + // peer verifying that signature sees a digest mismatch, and reads a + // mapping bug as suspected tampering. + // + // It lived in sendResponse, which runs after the loop -- one step + // too late, on the wrong side of signing. + if len(stepCtx.ResponseBody) > 0 { + if err = verifyEnvelope(stepCtx.ResponseBody); err != nil { + log.Errorf(stepCtx, err, "a step produced a response that is not a Beckn envelope; refusing to sign it") + // A plain error on purpose: nackBecknError's default branch + // turns it into a generic 500, so the caller learns the + // answer failed without being handed the internals of a + // mapping it does not own. The detail is in the log above. + h.signNackResponse(stepCtx, err) + responseBody = sendNack(stepCtx, wrapped, err) + return + } + } + // No routing — ONIX writes the ACK directly. Run response steps here // with resp=nil (publisher path semantics). for _, step := range h.responseSteps { From e402f962edcd1621f4a2883a4084626963708fdd Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Tue, 8 Sep 2026 15:05:13 +0530 Subject: [PATCH 067/130] fix(upstream): redact the credential for every auth scheme, not just query [#1] redactString returned early unless the scheme was query. The reasoning held for the URL -- only a query credential reaches one -- and was wrong about the body, which the same function is applied to on the non-2xx path. So a basic or header credential echoed by a provider went into the log in the clear at warn level. That is not a corner case. An API gateway quoting the Authorization header it rejected is the ordinary shape of a 401 or 403 body, a wrong-credential 4xx is not retried, and basic is the scheme the reference config ships -- so it repeated once per request for as long as the credential was wrong. Redacting the value we hold is not sufficient on its own, and the schemes differ in how: basic SetBasicAuth sends base64(user:pass), so the password alone never appears on the wire and replacing it misses the echoed header query the value goes through url.Values.Encode, so "a+b/c=" appears as "a%2Bb%2Fc%3D" (this half was already handled) header sent as-is Both the wrapped and raw forms are replaced in each case, longest first so a value containing another cannot be turned into a partial redaction, since an error built from the config rather than the request still quotes the credential unwrapped. The basic username is deliberately NOT redacted, and there is a test pinning that: it identifies rather than authenticates, and is routinely a short common word -- redacting "user" or "admin" would eat unrelated text and cost the operator the line they came for. --- .../internal/upstream/upstream.go | 114 +++++++++++---- .../internal/upstream/upstream_test.go | 137 ++++++++++++++++++ 2 files changed, 219 insertions(+), 32 deletions(-) diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index 05221161..b835bf15 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -16,6 +16,7 @@ package upstream import ( "bytes" "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -24,6 +25,7 @@ import ( "net/url" "os" "slices" + "sort" "strconv" "strings" "time" @@ -420,9 +422,6 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { return nil } -// servedActions lists the actions a capability covers, sorted so the same -// record reads the same way twice. - // buildRequest produces what the provider is sent. // // Whatever the mapping produces IS the request: a body for a method that takes @@ -789,41 +788,92 @@ type redactedErr struct { func (e redactedErr) Error() string { return e.text } func (e redactedErr) Unwrap() error { return e.err } -// redactString removes a query-string credential from any text about to be -// logged or returned -- an error, or the URL that was requested. +// redactString removes the configured credential from any text about to be +// logged or returned -- an error, a provider's response body, or the URL that +// was requested. +// +// Logging those is deliberate: they say what was asked of whom and what came +// back, which is the first thing anyone wants when a provider misbehaves. This +// is what makes that safe to do at info and warn level. // -// Logging the URL is deliberate: it says what was asked of whom, which is the -// first thing anyone wants when a provider misbehaves. This is what makes that -// safe to do at info level. +// EVERY scheme, not just query. This used to return early unless the scheme was +// query, on the reasoning that only a query credential reaches a URL -- true of +// the URL, and wrong about the body. A provider quoting the request it rejected +// is the ordinary shape of a 401 or 403 body, an API gateway echoing the +// Authorization header is routine, and a wrong-credential 4xx is not retried, +// so it lands in the log once per request for as long as the credential is +// wrong. basic is the scheme the reference config ships. func (s *Step) redactString(text string) string { - if s.config.AuthScheme != AuthSchemeQuery { - return text - } - value := os.Getenv(s.config.QueryValueEnv) - if value == "" { - return text - } - text = strings.ReplaceAll(text, value, redactedMarker) - - // Also the percent-encoded form, because that is the one that actually - // reaches a URL. authenticate puts the credential in through - // url.Values.Encode, which escapes anything outside the unreserved set -- - // so a base64 token, which routinely carries "+", "/" and "=", appears in - // the error as "a%2Bb%2Fc%3D" and a replacement of the raw value alone - // walks straight past it. Escaping what we hold is exact: it is the same - // function Encode used, so the two agree by construction rather than by a - // guess about which characters matter. - // - // Both forms rather than only the encoded one: a value needing no escaping - // is unchanged by QueryEscape, and an error that quotes the credential - // without having put it through a URL -- one built from the config rather - // than from the request -- still carries the raw form. - if encoded := url.QueryEscape(value); encoded != value { - text = strings.ReplaceAll(text, encoded, redactedMarker) + for _, secret := range s.secretForms() { + text = strings.ReplaceAll(text, secret, redactedMarker) } return text } +// secretForms returns every form the configured credential can appear in, +// longest first so a value that contains another is replaced before its +// substring turns the longer one into a partial redaction. +// +// Per scheme, because the schemes leak differently and redacting the value we +// hold is not enough on its own: +// +// - basic wraps the pair: SetBasicAuth sends base64(user:pass), so the +// password alone does not appear on the wire and replacing it misses the +// echoed header entirely. +// - query escapes: authenticate goes through url.Values.Encode, so a base64 +// token carrying "+", "/" or "=" appears as "a%2Bb%2Fc%3D". Escaping what +// we hold is exact -- same function Encode used, so the two agree by +// construction rather than by a guess about which characters matter. +// - header sends the value as-is. +// +// The raw form is kept alongside the wrapped one in both cases: an error built +// from the config rather than from the request still quotes the credential +// unwrapped. +func (s *Step) secretForms() []string { + switch s.config.AuthScheme { + case AuthSchemeBasic: + username, password := os.Getenv(s.config.UsernameEnv), os.Getenv(s.config.PasswordEnv) + if password == "" { + return nil + } + forms := []string{password} + if username != "" { + // The wire form, which is what a gateway echoes back. + forms = append(forms, + base64.StdEncoding.EncodeToString([]byte(username+":"+password))) + } + // The username is deliberately NOT redacted. It identifies rather than + // authenticates, and it is routinely a short common word -- redacting + // "user" or "admin" would eat unrelated text and cost the operator the + // log line they came for. The pair and the password are the secrets. + return longestFirst(forms) + case AuthSchemeHeader: + value := os.Getenv(s.config.HeaderValueEnv) + if value == "" { + return nil + } + return []string{value} + case AuthSchemeQuery: + value := os.Getenv(s.config.QueryValueEnv) + if value == "" { + return nil + } + forms := []string{value} + if encoded := url.QueryEscape(value); encoded != value { + forms = append(forms, encoded) + } + return longestFirst(forms) + } + return nil +} + +// longestFirst orders replacement candidates so a longer form is substituted +// before any shorter one it contains. +func longestFirst(forms []string) []string { + sort.Slice(forms, func(i, j int) bool { return len(forms[i]) > len(forms[j]) }) + return forms +} + // buildEndpoint joins the plan's base URL and path, carrying the mapped request // as query parameters when the method takes no body. func buildEndpoint(baseURL string, call model.ActionPlan, mapped []byte) (string, error) { diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go index 1cc073c3..cf96370c 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -2,6 +2,7 @@ package upstream import ( "context" + "encoding/base64" "errors" "fmt" "net/http" @@ -1917,3 +1918,139 @@ func TestRunReportsAnUnusableBaseURLAsABadRequest(t *testing.T) { t.Errorf("error = %v; a row that cannot build a request is not the provider failing", err) } } + +// Redaction has to cover the scheme a deployment actually configured, and the +// reference config ships basic. It used to cover only query, so a basic or +// header credential echoed by a provider went to the log in the clear at warn +// level -- and a wrong-credential 4xx is not retried, so that repeats once per +// request for as long as the credential is wrong. +// +// The three bodies here are the ordinary shapes: an API gateway quoting the +// Authorization header it rejected, a provider naming the password, and one +// naming the custom header's value. +func TestRedactStringCoversEveryScheme(t *testing.T) { + // No t.Parallel: t.Setenv forbids it. + t.Setenv("TEST_USER", "mausam") + t.Setenv("TEST_PASS", "s3cr3t") + t.Setenv("TEST_HDR", "hdr-k3y") + t.Setenv("TEST_QRY", "a+b/c=") + + // What SetBasicAuth actually puts on the wire. + wire := base64.StdEncoding.EncodeToString([]byte("mausam:s3cr3t")) + + for _, tc := range []struct { + name string + tweak func(*Config) + body string + secret string + wantOut string + }{ + { + name: "basic, the wire form a gateway echoes", + tweak: func(c *Config) { + c.AuthScheme = AuthSchemeBasic + c.UsernameEnv, c.PasswordEnv = "TEST_USER", "TEST_PASS" + }, + body: `{"error":"invalid Authorization: Basic ` + wire + `"}`, + secret: wire, + }, + { + name: "basic, the password quoted raw", + tweak: func(c *Config) { + c.AuthScheme = AuthSchemeBasic + c.UsernameEnv, c.PasswordEnv = "TEST_USER", "TEST_PASS" + }, + body: `{"error":"bad password s3cr3t"}`, + secret: "s3cr3t", + }, + { + name: "header, the value as sent", + tweak: func(c *Config) { + c.AuthScheme = AuthSchemeHeader + c.HeaderName, c.HeaderValueEnv = "X-API-Key", "TEST_HDR" + }, + body: `{"error":"bad X-API-Key: hdr-k3y"}`, + secret: "hdr-k3y", + }, + { + name: "query, still covered, raw form", + tweak: func(c *Config) { + c.AuthScheme = AuthSchemeQuery + c.QueryName, c.QueryValueEnv = "token", "TEST_QRY" + }, + body: `{"rejected":"token=a+b/c="}`, + secret: "a+b/c=", + }, + { + name: "query, still covered, percent-encoded form", + tweak: func(c *Config) { + c.AuthScheme = AuthSchemeQuery + c.QueryName, c.QueryValueEnv = "token", "TEST_QRY" + }, + body: `{"rejected":"token=` + url.QueryEscape("a+b/c=") + `"}`, + secret: url.QueryEscape("a+b/c="), + }, + } { + t.Run(tc.name, func(t *testing.T) { + step := newStep(t, &stubRegistry{plan: testPlan("http://provider.invalid", http.MethodGet)}, + &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, tc.tweak) + + // The same expression the non-2xx path logs. + logged := step.redactString(explain([]byte(tc.body))) + + if strings.Contains(logged, tc.secret) { + t.Errorf("the credential survives into the log line: %s", logged) + } + if !strings.Contains(logged, redactedMarker) { + t.Errorf("logged = %q, want the credential replaced", logged) + } + }) + } +} + +// A username identifies rather than authenticates, and is routinely a short +// common word -- redacting it would eat unrelated text and cost the operator +// the line they came for. Pinned so the choice is deliberate rather than an +// oversight someone "fixes" without noticing what it costs. +func TestRedactStringLeavesTheBasicUsername(t *testing.T) { + t.Setenv("TEST_USER", "mausam") + t.Setenv("TEST_PASS", "s3cr3t") + + step := newStep(t, &stubRegistry{plan: testPlan("http://provider.invalid", http.MethodGet)}, + &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, + func(c *Config) { + c.AuthScheme = AuthSchemeBasic + c.UsernameEnv, c.PasswordEnv = "TEST_USER", "TEST_PASS" + }) + + got := step.redactString(`user mausam failed to authenticate with s3cr3t`) + if strings.Contains(got, "s3cr3t") { + t.Errorf("the password survived: %s", got) + } + if !strings.Contains(got, "mausam") { + t.Errorf("got %q, want the username kept -- it is what makes the line useful", got) + } +} + +// Nothing configured, nothing to hide: an unset credential must not turn every +// empty string in the text into a redaction marker. +func TestRedactStringWithNoCredentialConfigured(t *testing.T) { + for _, scheme := range []string{AuthSchemeNone, AuthSchemeBasic, AuthSchemeHeader, AuthSchemeQuery} { + t.Run(scheme, func(t *testing.T) { + const text = `{"error":"provider said no"}` + step := newStep(t, &stubRegistry{plan: testPlan("http://provider.invalid", http.MethodGet)}, + &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, + func(c *Config) { + c.AuthScheme = scheme + // Env vars named but deliberately unset. + c.UsernameEnv, c.PasswordEnv = "TEST_UNSET_U", "TEST_UNSET_P" + c.HeaderName, c.HeaderValueEnv = "X-K", "TEST_UNSET_H" + c.QueryName, c.QueryValueEnv = "t", "TEST_UNSET_Q" + }) + + if got := step.redactString(text); got != text { + t.Errorf("redactString() = %q, want the text unchanged", got) + } + }) + } +} From 0d62b52e1a3ebae43764f4823c6b9c98b2e560f8 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Tue, 8 Sep 2026 15:05:13 +0530 Subject: [PATCH 068/130] test(handler): make the duplicate-provider-step test reach the guard [#1] TestInitStepsRefusesTwoProviderStepsWithTheSameID passed with the guard deleted, which is worse than having no test: it reads as covered. The handler was built as &stdHandler{moduleName: "test-module"} with no mapper and no registry, both of which loadProviderStep checks for up front. So startup failed on the FIRST config entry with "Mapper plugin not configured", the loop never reached the second entry, and the duplicate guard never ran. The assertion then matched "mausamgram" -- which is in that error too, because every loadProviderStep failure names the id. Now supplies both dependencies so the loop gets to the second entry, and asserts on the duplicate wording rather than on the id alone. Verified by deleting the guard: the test fails. Also adds a single-entry case, so the duplicate test cannot start passing on any startup failure whatsoever rather than specifically on the duplicate. The guard itself is untouched -- it was always correct. Only its test was hollow, and the bug it prevents is silent: two entries sharing an id leave one overwritten in the step map, so a capability stops being served with no error anywhere. --- core/module/handler/responsebody_test.go | 178 ++++++++++++++++++++++- 1 file changed, 177 insertions(+), 1 deletion(-) diff --git a/core/module/handler/responsebody_test.go b/core/module/handler/responsebody_test.go index 55ec8f0d..829ab973 100644 --- a/core/module/handler/responsebody_test.go +++ b/core/module/handler/responsebody_test.go @@ -38,6 +38,27 @@ func (s *routeSettingStep) Run(ctx *model.StepContext) error { return nil } +// stubProviderMapper and stubProviderRegistry satisfy the two dependencies +// loadProviderStep requires before it will build a provider step. The registry +// implements ProviderRecordLookup as well, which loadProviderStep narrows to. +type stubProviderMapper struct{} + +func (stubProviderMapper) Verify(context.Context, string, any) error { return nil } + +func (stubProviderMapper) Transform(context.Context, string, definition.Direction, any) ([]byte, error) { + return nil, nil +} + +type stubProviderRegistry struct{} + +func (stubProviderRegistry) Lookup(context.Context, *model.Subscription) ([]model.Subscription, error) { + return nil, nil +} + +func (stubProviderRegistry) ProviderRecord(context.Context, string) (*model.ProviderRecord, error) { + return nil, nil +} + var errStepFailed = errors.New("step failed") const v2SelectBody = `{"context":{"version":"2.0.0","action":"select","messageId":"msg-1"}}` @@ -175,7 +196,17 @@ func TestServeHTTPTreatsAnEmptyAnswerAsNoAnswer(t *testing.T) { // With binding keys a list, one entry serves several capabilities, so a repeated // id is now a mistake rather than the way to configure a second one. func TestInitStepsRefusesTwoProviderStepsWithTheSameID(t *testing.T) { - h := &stdHandler{moduleName: "test-module"} + // The mapper and registry are not decoration. loadProviderStep checks for + // both up front, so without them startup fails on the FIRST entry with + // "Mapper plugin not configured" and the loop never reaches the second -- + // meaning the duplicate guard never runs. This test passed with the guard + // deleted outright, because that error also contains the id, which was all + // the assertion checked. + h := &stdHandler{ + moduleName: "test-module", + mapper: &stubProviderMapper{}, + registry: &stubProviderRegistry{}, + } cfg := &Config{ Plugins: PluginCfg{ ProviderSteps: []plugin.Config{ @@ -189,11 +220,39 @@ func TestInitStepsRefusesTwoProviderStepsWithTheSameID(t *testing.T) { if err == nil { t.Fatal("expected two provider steps with the same id to be refused") } + // Asserted on the duplicate wording, not just the id: the id appears in + // every loadProviderStep failure too, so matching it alone cannot tell the + // duplicate refusal from a missing dependency. + if !strings.Contains(err.Error(), "configured more than once") { + t.Errorf("error %q should be the duplicate-id refusal", err) + } if !strings.Contains(err.Error(), "mausamgram") { t.Errorf("error %q should name the id that repeats", err) } } +// A single entry must still load, or the test above would pass on any failure +// at all rather than specifically on the duplicate. +func TestInitStepsAcceptsOneProviderStep(t *testing.T) { + h := &stdHandler{ + moduleName: "test-module", + mapper: &stubProviderMapper{}, + registry: &stubProviderRegistry{}, + } + cfg := &Config{ + Plugins: PluginCfg{ + ProviderSteps: []plugin.Config{{ID: "mausamgram", Config: map[string]string{}}}, + }, + } + + if err := h.initSteps(context.Background(), noopPluginManager{}, cfg); err != nil { + t.Fatalf("one provider step should load: %v", err) + } + if !h.hasProviderSteps { + t.Error("hasProviderSteps should be set once a provider step is configured") + } +} + // --- an unanswered request in a provider module ------------------------------ // silentStep is the dispatch no-op: a provider step recognising the request as @@ -379,3 +438,120 @@ func TestAckSignerStillSignsTheGeneratedAckWhenNoStepAnswered(t *testing.T) { t.Errorf("signed %q, want the generated ack %q", signer.signedBody, wantAck) } } + +// A step's answer has to be an envelope. A mapping whose response half is +// written as `$.response.temperature` rather than as an object produces `28.5`, +// which is valid JSON -- so Content-Type was not a lie -- and the adapter +// answered 200 with it and then SIGNED it. A consumer looking for +// message.contract finds nothing and cannot tell that from a protocol change. +// +// Refused rather than passed on, because a signed confident non-answer is worse +// than a NACK: the caller cannot retry what it does not know failed, and the +// signature says this adapter meant it. +// +// Driven through ServeHTTP with a real ack signer installed, which is the whole +// point. The check used to sit in sendResponse, after the response steps -- so +// it refused an answer the signer had already covered, and shipped a NACK body +// under a Signature over the scalar. Testing sendResponse directly could not +// see that, because the signer is not in that call. +func TestServeHTTPDoesNotSignAnAnswerItRefuses(t *testing.T) { + signer := &mockSigner{returnSig: "sig"} + as, err := newAckSignerStep(signer, &mockKM{ + keyset: &model.Keyset{UniqueKeyID: "k1", SigningPrivate: "priv"}}) + if err != nil { + t.Fatalf("failed to build the ack signer: %v", err) + } + // Asserted to the concrete type exactly as initSteps does, so h.ackSigner + // is populated and the NACK path can sign -- which is what is under test. + ackSigner, ok := as.(*ackSignerStep) + if !ok { + t.Fatalf("newAckSignerStep returned %T, want *ackSignerStep", as) + } + + h := &stdHandler{ + SubscriberID: "test-sub", + role: model.RoleBPP, + moduleName: "test-module", + steps: []definition.Step{&answeringStep{answer: []byte(`28.5`)}}, + responseSteps: []definition.ResponseStep{as}, + ackSigner: ackSigner, + hasProviderSteps: true, + } + + recorder := serve(t, h, v2SelectBody) + + // The scalar must not reach the wire, and the caller must be told. + if recorder.Code == http.StatusOK { + t.Errorf("status = %d; a body that cannot carry a message must not be a 200", recorder.Code) + } + if strings.Contains(recorder.Body.String(), "28.5") { + t.Error("the scalar was written to the wire") + } + var got model.Response + if err := json.Unmarshal(recorder.Body.Bytes(), &got); err != nil { + t.Fatalf("failed to decode the response: %v", err) + } + if got.Message.Status != model.StatusNACK { + t.Errorf("status = %q, want a NACK", got.Message.Status) + } + + // And the part that was actually broken: the signature that ships must + // cover the body that ships. Before this fix it covered the refused + // scalar while a NACK went out beneath it, and a peer verifying that sees + // a digest mismatch -- reading a mapping bug as suspected tampering. + // + // Asserted on the bytes signed, not on the header text: mockSigner returns + // a fixed signature string whatever it is given, so the header alone + // cannot tell the two bodies apart. + if string(signer.signedBody) == "28.5" { + t.Error("the refused scalar was signed; the refusal has to happen before the signer runs") + } + if signer.signAckCalled && string(signer.signedBody) != recorder.Body.String() { + t.Errorf("signed %q but sent %q -- the signature must cover what ships", + signer.signedBody, recorder.Body.String()) + } +} + +// The ordinary path must be untouched: a real envelope is answered, signed, and +// sent, so the guard above cannot be satisfied by refusing everything. +func TestServeHTTPStillSignsAndSendsARealEnvelope(t *testing.T) { + const answer = `{"context":{"action":"on_select"},"message":{"contract":{}}}` + + signer := &mockSigner{returnSig: "sig-over-the-answer"} + as, err := newAckSignerStep(signer, &mockKM{ + keyset: &model.Keyset{UniqueKeyID: "k1", SigningPrivate: "priv"}}) + if err != nil { + t.Fatalf("failed to build the ack signer: %v", err) + } + // Asserted to the concrete type exactly as initSteps does, so h.ackSigner + // is populated and the NACK path can sign -- which is what is under test. + ackSigner, ok := as.(*ackSignerStep) + if !ok { + t.Fatalf("newAckSignerStep returned %T, want *ackSignerStep", as) + } + + h := &stdHandler{ + SubscriberID: "test-sub", + role: model.RoleBPP, + moduleName: "test-module", + steps: []definition.Step{&answeringStep{answer: []byte(answer)}}, + responseSteps: []definition.ResponseStep{as}, + ackSigner: ackSigner, + hasProviderSteps: true, + } + + recorder := serve(t, h, v2SelectBody) + + if recorder.Code != http.StatusOK { + t.Errorf("status = %d, want 200", recorder.Code) + } + if recorder.Body.String() != answer { + t.Errorf("body = %s, want the answer unchanged", recorder.Body.String()) + } + if string(signer.signedBody) != answer { + t.Errorf("signed %q, want the answer that was sent", signer.signedBody) + } + if sig := recorder.Header().Get("Signature"); !strings.Contains(sig, "sig-over-the-answer") { + t.Errorf("Signature header = %q, want the answer's signature", sig) + } +} From 2978c93c4944b011efd4d7c72624fe7c2337f5d8 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Tue, 8 Sep 2026 15:05:13 +0530 Subject: [PATCH 069/130] docs: correct four claims that contradict the code, and drop two dead comments [#1] None of these is a runtime defect. In a change where the rationale comments are part of the deliverable, each one told the next reader the opposite of what the code does. 1. The reference config said 4xx is never retried. 429 is, explicitly -- it is the one 4xx that says try again. 2. Same block: "a non-2xx reaches the caller as an error with the provider's own response included" is backwards, and in the direction that matters. The error carries the status only; the body goes to the log, deliberately, because what a provider puts in a failure body is its own business. An operator reading the config would go looking for the upstream body in the NACK, which the code specifically prevents. 3. The mapper README made SCH_SCHEMA_ADAPTATION_FAILED a blanket bad request. Only the request leg is a 400; the response leg is a 502, because the input there is the provider's answer rather than anything the caller sent. An integrator classifying the code as 4xx would misfile upstream-shape failures as client errors. 4. The registry README's provider_record outcome list read as exhaustive and omitted binding_no_actions, which is live and reachable. A dashboard built from the documented set would silently merge "active, owned, and serves nothing" into one of the other refusals. Also removes both orphaned servedActions doc comments, left behind in providerrecord.go and upstream.go when the function was hoisted onto model.ProviderRecord. Each now sits above an unrelated function and documents nothing; the method is documented at its new home. Placeholders the subscriberId in the reference config too, which an earlier commit message claimed to have done and had not -- only url was changed. Both sites now carry <> with the previous value in a comment, and a note that the handler's and the keyManager's must match. --- config/oan-provider-adapter.yaml | 46 ++++++++++++++++--- .../implementation/jsonmapper/README.md | 14 +++++- .../implementation/oanregistry/README.md | 12 +++-- .../oanregistry/providerrecord.go | 3 -- 4 files changed, 60 insertions(+), 15 deletions(-) diff --git a/config/oan-provider-adapter.yaml b/config/oan-provider-adapter.yaml index 025956bc..2bba72e1 100644 --- a/config/oan-provider-adapter.yaml +++ b/config/oan-provider-adapter.yaml @@ -54,7 +54,24 @@ modules: handler: type: std role: bpp - subscriberId: provider-network-vistaar.da.gov.in + + # This adapter's own network identity -- the id a peer verifies its + # signatures against. A placeholder rather than a working value, + # because one deployment's identity is not everyone's, and a reader + # who copies this signs as somebody else. + # + # The registry requires it to be hostname-shaped, and it is never + # resolved by DNS -- routing between adapters is the router plugin's + # config. Pick it once and deliberately: the registry is append-only, + # delete is soft, and a soft-deleted id keeps the unique index, so an + # id can never be reused. + # + # was provider-network-vistaar.da.gov.in + # dev provider.oan.dev + # + # Must match the keyManager's subscriberId below -- they are the same + # identity, named twice because the plugins are configured separately. + subscriberId: <> plugins: # ------------------------------------------------------------------ @@ -119,7 +136,13 @@ modules: keyManager: id: simplekeymanager config: - subscriberId: provider-network-vistaar.da.gov.in + # The same identity as the handler's subscriberId above, and it + # has to be: this is the id the keys are looked up under. Set one + # and not the other and the adapter signs as an id the registry + # has no keys for, which fails at the peer rather than here. + # + # was provider-network-vistaar.da.gov.in + subscriberId: <> signer: id: signer @@ -273,9 +296,18 @@ modules: # retries are all registry writes. Nothing here changes and nothing restarts. # # Retry classification is not configurable at all: 4xx is permanent and is not -# retried, 5xx and transport errors are, and the backoff rises from 50ms to a -# 800ms ceiling. A non-2xx reaches the caller as an error with the provider's -# own response included, and the mapping never runs on it -- which is why an -# upstream that signals "no data" with a 4xx surfaces as a failure rather than -# an empty result. +# retried -- EXCEPT 429, which is, because "you are going too fast" is the one +# 4xx that says try again. 5xx and transport errors are retried too, and the +# backoff rises from 50ms to a 800ms ceiling. +# +# A non-2xx reaches the caller as an error carrying THE STATUS ONLY -- not the +# provider's response body. That is deliberate, and worth knowing before you go +# looking for the body in a NACK: what a provider puts in a failure body is its +# own business, and it has been known to be a stack trace, an internal hostname +# or a database error. The body goes to the adapter's log instead, at warn +# level, with the configured credential redacted. +# +# The mapping never runs on a non-2xx either -- which is why an upstream that +# signals "no data" with a 4xx surfaces as a failure rather than an empty +# result. # ---------------------------------------------------------------------------- diff --git a/pkg/plugin/implementation/jsonmapper/README.md b/pkg/plugin/implementation/jsonmapper/README.md index 0b2ac9c7..e126e701 100644 --- a/pkg/plugin/implementation/jsonmapper/README.md +++ b/pkg/plugin/implementation/jsonmapper/README.md @@ -188,5 +188,15 @@ the upgrade if one mapping ever becomes hot enough to matter. A mapping that cannot be fetched, parsed or compiled is an operator or registry fault and surfaces as a plain error. A mapping that ran but could not be applied -is the payload's shape being wrong, and surfaces as a `SCH_SCHEMA_ADAPTATION_FAILED` -bad request. +surfaces as `SCH_SCHEMA_ADAPTATION_FAILED` — but **the HTTP status depends on +which half failed**, and the difference matters if you classify on that code: + +- **request half → 400.** The input is the caller's own payload, so its shape + being wrong is the caller's to fix. +- **response half → 502.** The input there is the PROVIDER's answer, not + anything the caller sent. A provider that changed shape, or a bug in the + response mapping, is nothing the caller did — telling them to fix a request + that was fine sends them after the wrong thing. + +So do not treat the code as uniformly 4xx for retry or alerting: an +upstream-shape failure carrying it is a 502. diff --git a/pkg/plugin/implementation/oanregistry/README.md b/pkg/plugin/implementation/oanregistry/README.md index a35bd96a..cbce9a81 100644 --- a/pkg/plugin/implementation/oanregistry/README.md +++ b/pkg/plugin/implementation/oanregistry/README.md @@ -222,9 +222,15 @@ plugin error counter. The `error_type` dimension is one of: Provider-record lookups report under `operation=provider_record`, with their own outcomes: `binding_not_found` · `binding_inactive` · `binding_unowned` · -`participant_not_found` · `participant_inactive` · `no_upstream_url` · -`no_binding_key`. Each refusal is kept distinct: they all deny the call, but a -withdrawn capability and a suspended provider are different operational events. +`binding_no_actions` · `participant_not_found` · `participant_inactive` · +`no_upstream_url` · `no_binding_key`. Each refusal is kept distinct: they all +deny the call, but a withdrawn capability and a suspended provider are different +operational events. + +`binding_no_actions` is the easiest of these to leave out of a dashboard and the +least obvious to reproduce: the binding is active and owned by this provider, +and it still serves nothing, because its record carries no actions. Counting it +under one of the others would merge "misconfigured" into "withdrawn". Signing-key lookups report under `operation=lookup`: diff --git a/pkg/plugin/implementation/oanregistry/providerrecord.go b/pkg/plugin/implementation/oanregistry/providerrecord.go index 0e859a52..cf3dd5a6 100644 --- a/pkg/plugin/implementation/oanregistry/providerrecord.go +++ b/pkg/plugin/implementation/oanregistry/providerrecord.go @@ -138,9 +138,6 @@ func (c *Client) ProviderRecord(ctx context.Context, bindingKey string) (*model. return plan, nil } -// servedActions lists the actions a plan covers, sorted so the same record logs -// the same way twice. - // refuse records a deliberate denial and returns the caller's sentinel. The // registry answered; the answer was no. func (c *Client) refuse(ctx context.Context, span trace.Span, start time.Time, outcome string) error { From 82a5f1e1378008c004db5c286ed652b9935c1c07 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Tue, 8 Sep 2026 16:03:15 +0530 Subject: [PATCH 070/130] fix(jsonmapper): release the evaluation lock even if the library panics [#1] The unlock was written after the call rather than deferred, so a panic inside Evaluate left the mutex held with nothing to release it. Nothing in the request path recovers, so the lock stays taken for the life of the process. This is a defect the previous commit introduced, and it is worse than it looks because of what that commit did: the lock is now package-wide. A wedged mutex is no longer one broken mapping, it is EVERY mapping for EVERY provider step, cleared only by a restart. Widening the lock widened the blast radius, and the unlock should have been deferred in the same change. A helper rather than a defer at each of the two call sites, because the precondition loop evaluates once per check: a defer there releases only when the whole loop returns, which would hold the lock across every check in the file -- a smaller version of the same mistake. The panic is converted to an error rather than re-raised. net/http recovers per connection, so re-raising costs the caller its connection with no NACK and nothing in our log naming the mapping, for what is indistinguishable from a mapping that could not be applied. Reported as one, so it lands where the fault is. Testable because jsonata.Expression is an interface: the test injects an expression that panics, asserts the error, then takes the lock from another goroutine with a timeout. Un-defer the unlock and it fails on that timeout rather than hanging the suite. --- .../implementation/jsonmapper/jsonmapper.go | 48 ++++++++++--- .../jsonmapper/jsonmapper_test.go | 68 +++++++++++++++++++ 2 files changed, 108 insertions(+), 8 deletions(-) diff --git a/pkg/plugin/implementation/jsonmapper/jsonmapper.go b/pkg/plugin/implementation/jsonmapper/jsonmapper.go index cefbdb16..4ade3194 100644 --- a/pkg/plugin/implementation/jsonmapper/jsonmapper.go +++ b/pkg/plugin/implementation/jsonmapper/jsonmapper.go @@ -103,6 +103,41 @@ type Config struct { MaxCacheEntries int `yaml:"maxCacheEntries" json:"maxCacheEntries"` } +// evaluateLocked runs one expression under the package lock. +// +// The unlock is DEFERRED, not written after the call. A panic inside the +// library -- and this is a library we have already found a data race in -- +// would otherwise leave the mutex held with nothing to release it. Because the +// lock is package-wide, that is not one wedged mapping but every mapping in +// the process, for every provider step, until a restart. Widening the lock +// widened that blast radius, so the defer matters more here than it did when +// the lock was per mapping. +// +// A helper rather than a defer at each call site, because the precondition +// loop evaluates once per check: a defer there would release only when the +// whole loop returned, holding the lock across every check in the file. +// +// The panic is converted to an error rather than re-raised. net/http recovers +// a panic per connection, so re-raising costs the caller its connection with +// no NACK and nothing in our log naming the mapping -- for what is, from the +// caller's side, indistinguishable from a mapping that could not be applied. +// Reported as one, so it lands where the fault is. +func evaluateLocked(expr jsonata.Expression, document []byte) (result []byte, err error) { + evaluating.Lock() + defer evaluating.Unlock() + + // Registered after the unlock so it runs BEFORE it: recover, name the + // failure, then release. + defer func() { + if recovered := recover(); recovered != nil { + result = nil + err = fmt.Errorf("jsonata evaluation panicked: %v", recovered) + } + }() + + return expr.Evaluate(document, nil) +} + // evaluating serialises every Evaluate in this package, across all mappings. // // It has to be this wide. Evaluate mutates more than the expression it is @@ -295,10 +330,9 @@ func (m *Mapper) Verify(ctx context.Context, mappingRef string, input any) error return precondition.err } - // See evaluating: serialised across the package, not per expression. - evaluating.Lock() - result, evalErr := precondition.expression.Evaluate(document, nil) - evaluating.Unlock() + // See evaluateLocked: serialised across the package, and released even + // if the library panics. + result, evalErr := evaluateLocked(precondition.expression, document) if evalErr != nil { log.Errorf(ctx, evalErr, "JSON mapping %s precondition failed to evaluate: %v", mappingRef, evalErr) return model.NewBadReqErr(codeAdaptationFailed, fmt.Errorf( @@ -600,12 +634,10 @@ func (m *Mapper) evaluate(ctx context.Context, mapping *compiledMapping, mapping return nil, fmt.Errorf("jsonmapper: mapping %q: %w", mappingRef, err) } - // See evaluating: serialised across the package, because the library's + // See evaluateLocked: serialised across the package, because the library's // shared built-ins make even two different mappings unsafe to overlap. // Marshalling above is deliberately outside the lock. - evaluating.Lock() - result, err := mapping.expression.Evaluate(document, nil) - evaluating.Unlock() + result, err := evaluateLocked(mapping.expression, document) if err != nil { log.Errorf(ctx, err, "JSON mapping %s %s half failed to evaluate: %v", mappingRef, direction, err) wrapped := fmt.Errorf("mapping %q %s half could not be applied: %w", mappingRef, direction, err) diff --git a/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go b/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go index 667bff17..04313d6d 100644 --- a/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go +++ b/pkg/plugin/implementation/jsonmapper/jsonmapper_test.go @@ -1015,3 +1015,71 @@ func TestConcurrentDifferentMappingsDoNotRace(t *testing.T) { } } } + +// panickingExpression stands in for the library misbehaving. jsonata.Expression +// is an interface, so this needs no cooperation from the library -- which is +// the point: the failure being guarded against is one we cannot ask it for. +type panickingExpression struct{} + +func (panickingExpression) Evaluate([]byte, map[string]interface{}) ([]byte, error) { + panic("library exploded mid-evaluation") +} +func (panickingExpression) SetMaxDepth(int) {} +func (panickingExpression) SetMaxTime(int) {} +func (panickingExpression) SetMaxRange(int) {} +func (panickingExpression) Assign(string, interface{}) {} +func (panickingExpression) RegisterFunction(string, interface{}, string) error { return nil } +func (panickingExpression) AST() interface{} { return nil } +func (panickingExpression) Errors() []error { return nil } + +// A panic inside Evaluate must not leave the lock held. It is package-wide, so +// a wedged mutex is not one broken mapping -- it is every mapping in the +// process, for every provider step, until someone restarts it. Writing the +// unlock after the call rather than deferring it is what would cause that, and +// widening the lock is what turned it from a bounded fault into an outage. +func TestEvaluateLockedReleasesTheLockOnPanic(t *testing.T) { + // Not parallel: it asserts on the state of a package-level lock. + result, err := evaluateLocked(panickingExpression{}, []byte(`{}`)) + + if err == nil { + t.Fatal("a panic must surface as an error, not be swallowed") + } + if !strings.Contains(err.Error(), "panicked") { + t.Errorf("error = %v, want it to say the evaluation panicked", err) + } + if result != nil { + t.Errorf("result = %q, want nothing on a failed evaluation", result) + } + + // The part that matters. If the unlock were not deferred, this would block + // forever rather than fail. + done := make(chan struct{}) + go func() { + evaluating.Lock() + evaluating.Unlock() + close(done) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("the lock was never released: every mapping in the process is now wedged") + } +} + +// And the ordinary path still works through the same helper, so the guard above +// cannot be satisfied by a helper that never evaluates anything. +func TestEvaluateLockedRunsAnOrdinaryExpression(t *testing.T) { + mapper := newTestMapper(t) + expr, err := mapper.instance.Compile(`{ "kept": $count([1,2,3]) }`, false) + if err != nil { + t.Fatalf("failed to compile: %v", err) + } + + got, err := evaluateLocked(expr, []byte(`{}`)) + if err != nil { + t.Fatalf("evaluateLocked() returned an unexpected error: %v", err) + } + if !strings.Contains(string(got), `"kept"`) { + t.Errorf("got %q, want the evaluated object", got) + } +} From 214a7934c7a29a88f9ad69440c9b7df61f64113a Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Tue, 8 Sep 2026 16:03:16 +0530 Subject: [PATCH 071/130] docs: say that a mapping reads _local, because it does [#1] Four places described the mapping's inputs and three of them were wrong. The code passes _local on both legs, and Prerequisites documents it as "handed to the mapping as _local" -- while Mapper.Transform said input "deliberately does not carry values the caller resolved for itself", the README's "what a mapping can read" table omitted it, upstream.go's own comment said "what each party sent and nothing else", and the mausamgram mapping said "nothing else is in scope". Harmless today only because every shipped plugin declares an empty Prerequisites map. The first provider to add a real one would have been told by three sources that its resolved values are unavailable to the mapping, when they are already being passed. The code is right, so the three claims are corrected rather than the behaviour. The distinction they were reaching for is kept, because it is worth keeping: _local is for what a payload cannot carry and a mapping cannot obtain -- a code looked up from a name, a point resolved to a market -- and NOT for values the plugin already holds and merely used to make the call. Reading those back is a second name for the same data. Also adds the tests that were missing, which is why this drifted: nothing asserted _local at all, so the whole path was dead code. One test resolves two values through a real Prerequisites entry and checks they arrive on both legs; another checks that no prerequisites yields an empty _local rather than an absent one, so a mapping referring to it reads nothing rather than failing. --- .../weather-observation.select.yaml | 14 +++- pkg/plugin/definition/mapper.go | 14 +++- .../internal/upstream/upstream.go | 3 +- .../internal/upstream/upstream_test.go | 84 +++++++++++++++++++ .../implementation/jsonmapper/README.md | 18 ++-- 5 files changed, 118 insertions(+), 15 deletions(-) diff --git a/config/mappings/mausamgram/weather-observation.select.yaml b/config/mappings/mausamgram/weather-observation.select.yaml index b5aea57f..f8cdb410 100644 --- a/config/mappings/mausamgram/weather-observation.select.yaml +++ b/config/mappings/mausamgram/weather-observation.select.yaml @@ -14,10 +14,16 @@ # and the response half additionally reads: # response the provider's answer, in its own shape # -# Nothing else is in scope. Values the provider step resolved before the call are -# not passed in: the step holds them and used them to make the call, so a mapping -# reading them back would be a second name for the same data. Where the answer -# needs them, it takes them from what the provider echoed. +# and both halves also read: +# _local whatever the step's prerequisites resolved before the call -- a +# code looked up from a name, a point resolved to a market. EMPTY +# here: this plugin declares no prerequisites, so nothing in this +# file reads it. +# +# Nothing else is in scope. In particular, values the step already holds and +# merely used to make the call are not in _local: reading those back would be a +# second name for the same data. Where the answer needs one, it takes it from +# what the provider echoed. # The response half follows the openagrinet:WeatherObservation v0.1 schema pack, # Direct mode. The pack lives in OpenAgriNet/network-specs; it is referred to diff --git a/pkg/plugin/definition/mapper.go b/pkg/plugin/definition/mapper.go index 2ff68f83..e5c7be95 100644 --- a/pkg/plugin/definition/mapper.go +++ b/pkg/plugin/definition/mapper.go @@ -32,10 +32,16 @@ type Mapper interface { // Which action the mapping serves is settled by the registry entry that // named it, so only the direction is passed here. // - // input carries what a party sent: the inbound payload, and on the way back - // the provider's answer. It deliberately does not carry values the caller - // resolved for itself -- the caller holds those already, so routing them - // through a mapping would be a detour and a second name for the same data. + // input carries what a party sent -- the inbound payload, and on the way + // back the provider's answer -- plus, under _local, any values the caller + // resolved before making the call. + // + // _local is for what a payload cannot carry and a mapping cannot obtain: a + // code looked up from a name, a point resolved to a market. A caller with + // nothing to add passes an empty map, so a mapping referring to _local + // reads nothing rather than failing. Values the caller already holds and + // merely used to make the call do NOT belong here -- routing those back + // through a mapping is a second name for the same data. // // A direction the file has no transform for produces nothing, with no error. // What nothing means belongs to the caller: on the request leg it means there diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index b835bf15..40d28fda 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -399,7 +399,8 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { // The same mapping reference as the request, other half: one file carries // both directions for this action. // - // The mapping is handed what each party sent and nothing else. + // The mapping is handed what each party sent, plus whatever prerequisites + // resolved, under _local. Empty when there are none. becknResponse, err := s.mapper.Transform(ctx, call.Mappings, definition.DirectionResponse, map[string]any{ "beckn": beckn, "_local": local, diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go index cf96370c..33530e33 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -2054,3 +2054,87 @@ func TestRedactStringWithNoCredentialConfigured(t *testing.T) { }) } } + +// _local is part of the mapping interface, and until now nothing asserted it. +// Every shipped plugin declares an empty Prerequisites map, so the whole path +// was dead code: the first provider to add a real prerequisite would have found +// out at runtime whether its resolved values reach the mapping at all. +// +// Both legs, because a resolved value is usually needed on the way back too -- +// a code looked up to make the call is what names the thing in the answer. +func TestRunHandsResolvedPrerequisitesToTheMappingAsLocal(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"ok":true}`) + })) + defer upstream.Close() + + prerequisites := Prerequisites{ + testBindingKey: func(context.Context, any) (map[string]any, error) { + return map[string]any{"stationId": "42", "marketCode": "2056"}, nil + }, + } + + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{"answered":true}`)} + step, closer, err := New(context.Background(), + &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper, prerequisites, + &Config{BindingKeys: []string{testBindingKey}}) + if err != nil { + t.Fatalf("New() returned an unexpected error: %v", err) + } + t.Cleanup(func() { _ = closer() }) + + if _, err := runStep(t, step, selectBody); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + + for _, leg := range []struct { + name string + input any + }{ + {name: "request", input: mapper.requestInput}, + {name: "response", input: mapper.responseInput}, + } { + t.Run(leg.name+" leg", func(t *testing.T) { + asMap, ok := leg.input.(map[string]any) + if !ok { + t.Fatalf("%s leg input = %T, want a map", leg.name, leg.input) + } + local, present := asMap["_local"].(map[string]any) + if !present { + t.Fatalf("%s leg carries no _local; a resolved prerequisite never reaches the mapping", leg.name) + } + if local["stationId"] != "42" || local["marketCode"] != "2056" { + t.Errorf("_local = %v, want both resolved values", local) + } + }) + } +} + +// With no prerequisites -- every plugin shipped today -- _local is present and +// empty rather than absent, so a mapping referring to it reads nothing instead +// of failing on an unknown name. +func TestRunPassesAnEmptyLocalWhenThereAreNoPrerequisites(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"ok":true}`) + })) + defer upstream.Close() + + mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{"answered":true}`)} + step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper) + + if _, err := runStep(t, step, selectBody); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + + asMap, ok := mapper.requestInput.(map[string]any) + if !ok { + t.Fatalf("request input = %T, want a map", mapper.requestInput) + } + local, present := asMap["_local"].(map[string]any) + if !present { + t.Fatal("_local is absent; a mapping referring to it would fail rather than read nothing") + } + if len(local) != 0 { + t.Errorf("_local = %v, want it empty", local) + } +} diff --git a/pkg/plugin/implementation/jsonmapper/README.md b/pkg/plugin/implementation/jsonmapper/README.md index e126e701..1f1d5913 100644 --- a/pkg/plugin/implementation/jsonmapper/README.md +++ b/pkg/plugin/implementation/jsonmapper/README.md @@ -130,12 +130,18 @@ stays in the record for now.) |---|---|---| | `beckn` | the inbound Beckn payload | the inbound Beckn payload | | `response` | — | the provider's raw answer | - -**What a party sent, and nothing else.** Values a provider plugin resolved before -the call are deliberately not passed in: the plugin holds them and used them to -make the call, so a mapping reading them back would be a detour and a second name -for the same data. Where the answer needs such a value, it takes it from what the -provider echoed. +| `_local` | what prerequisites resolved | what prerequisites resolved | + +**What a party sent, plus what a payload could not carry.** `_local` holds values +the provider plugin resolved before the call — a code looked up from a name, a +point resolved to a market — for the case where a mapping needs them and neither +party sent them. It is an empty map when the plugin has no prerequisites, which is +every plugin shipped today, so a mapping referring to `_local` reads nothing rather +than failing. + +What does **not** go in `_local`: values the plugin already holds and merely used to +make the call. Reading those back through a mapping is a detour and a second name for +the same data. Where the answer needs one, it takes it from what the provider echoed. ## Why the direction is a parameter, not a convention From 6ae1ba607f5b6711b65e0cf8d024ee8892c18b8f Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Thu, 3 Sep 2026 15:12:16 +0530 Subject: [PATCH 072/130] feat: add the Mandi provider plugin [#8] Serves openagrinet:MandiPrice against Agmarknet's Vistaar select, as a sibling to weather: a domain package of 58 lines wrapping internal/upstream, which needed no change for it. That was the test of whether the machinery and domain split from 2b3cab1 actually held, and it did. The package has NO prerequisites, and the reason is the pack rather than luck. A MandiPrice select names the market it wants -- market.marketCode, market.district, market.state -- and carries a commodity code and a validity window, which is every parameter the upstream takes. There is no top-level location in the pack, so nothing has to turn a point into a market, which is the one thing the provider backend needs a spatial SQL query for and the one thing this adapter may not do. The mapping carries the whole contract. Three things in it are not obvious: the upstream's records use Title Case keys WITH SPACES -- `Modal Price` -- so they need backticks, and its prices are STRINGS, so they need $number before they satisfy the pack's numeric types. Both are pinned by a verbatim capture from the provider backend's own documentation. dates convert twice. The pack speaks ISO, the upstream speaks dd-MM-yyyy, so the request half converts out and the response half converts back. the pack requires none of the fields the upstream needs -- an OnDemand select requires only supportedCommodities and supportedPriceFields, leaving market and validity optional. So a spec-valid select can be unanswerable, and the mapping's required: block refuses those with its own message rather than earning a 400 or, worse, an empty result that reads as "no prices". Resource ids are built from codes rather than the names the upstream reports: "Kasdol APMC" and "Paddy(Common)" carry spaces and brackets, and an id a consumer may put in a URL should not. Verified: the shipped mapping run through the real mapper and the real step answers two records as two Direct resources with their prices converted, the offer's references rewritten to match, and absent min/max left absent rather than zeroed. Both directions validate -- the select against beckn.yaml and MandiPrice v0.1 in OnDemand mode, the on_select against beckn.yaml and the pack in Direct mode, with no errors. --- .../agmarknet/mandi-price.select.yaml | 200 ++++++++ install/build-plugins.sh | 1 + pkg/plugin/implementation/mandi/cmd/plugin.go | 104 ++++ pkg/plugin/implementation/mandi/mandi.go | 38 ++ .../implementation/mandi/mappings_test.go | 458 ++++++++++++++++++ .../implementation/mandi/prerequisites.go | 20 + 6 files changed, 821 insertions(+) create mode 100644 config/mappings/agmarknet/mandi-price.select.yaml create mode 100644 pkg/plugin/implementation/mandi/cmd/plugin.go create mode 100644 pkg/plugin/implementation/mandi/mandi.go create mode 100644 pkg/plugin/implementation/mandi/mappings_test.go create mode 100644 pkg/plugin/implementation/mandi/prerequisites.go diff --git a/config/mappings/agmarknet/mandi-price.select.yaml b/config/mappings/agmarknet/mandi-price.select.yaml new file mode 100644 index 00000000..2f0e70b3 --- /dev/null +++ b/config/mappings/agmarknet/mandi-price.select.yaml @@ -0,0 +1,200 @@ +# Agmarknet Vistaar, openagrinet:MandiPrice, select. Both directions, one file. +# +# One file per binding-action rather than one per direction, because both legs of +# an exchange are one contract: the response has to answer the request that was +# sent, and splitting them lets one change without the other. +# +# The upstream is Agmarknet's Vistaar select. It takes governed codes -- state, +# district, market, commodity -- plus a date range, and every one of them is in +# the payload, so nothing here needs resolving before the call. That is why the +# mandi plugin has no prerequisites: a MandiPrice select names the market it +# wants rather than a point to search from. +# +# NOTHING HERE IS OUTSIDE THE PACK. openagrinet:MandiPrice v0.1 carries every +# field this answer sets. Where the upstream reports something the pack has no +# home for, it is dropped rather than invented. + +# What this capability cannot serve, refused before the provider is called. +# +# The pack requires none of these: a MandiPrice select is OnDemand, and that +# branch requires only supportedCommodities and supportedPriceFields. It leaves +# market and validity optional, and defines market.district and market.state as +# "name or governed code". So a payload can be perfectly valid and still be +# unanswerable by this upstream, which wants codes and a date range. +# +# Refusing here names what is missing. Sending it anyway earns a 400 from +# Agmarknet, or worse an empty result that reads as "no prices". +required: + - check: | + ( + $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; + $exists($ra.supportedCommodities[0].code) + ) + message: "this capability needs a commodity code in supportedCommodities[0].code" + - check: | + ( + $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; + $exists($ra.market.state) and $exists($ra.market.district) + ) + message: "this capability needs governed state and district codes in market" + - check: | + ( + $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; + $exists($ra.validity.startsAt) and $exists($ra.validity.endsAt) + ) + message: "this capability needs a validity window; it reports prices over a date range" + +# The upstream is a GET, so this object becomes the query string. The token is +# not here and must never be: it comes from the adapter's authScheme query, +# whose value is read from an environment variable. This file is published. +# +# marketcode is sent when the payload names one and omitted otherwise, which is +# what the upstream expects: without it the query widens from one market to the +# whole district. +request: | + ( + $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; + + /* The upstream wants dd-MM-yyyy; the pack's validity is an ISO date. A + substring reorder rather than a date library, because these are dates + with no time and no zone, and $fromMillis would invent both. */ + $ddmmyyyy := function($iso) { + $substring($iso, 8, 2) & "-" & $substring($iso, 5, 2) & "-" & $substring($iso, 0, 4) + }; + + $base := { + "statecode": $ra.market.state, + "districtcode": $ra.market.district, + "commoditycode": $ra.supportedCommodities[0].code, + "from_date": $ddmmyyyy($ra.validity.startsAt), + "to_date": $ddmmyyyy($ra.validity.endsAt) + }; + + $exists($ra.market.marketCode) + ? $merge([$base, {"marketcode": $ra.market.marketCode}]) + : $base + ) + +# One resource per price record, in Direct mode. +# +# Direct is what the pack requires of an answer: the resource now CARRIES the +# observation rather than advertising that it could obtain one. It requires +# source, commodity, market, arrivalDate, prices and generatedAt, and all six +# are set below. +# +# The upstream's records use Title Case keys WITH SPACES -- `Modal Price`, not +# modalPrice -- so they need backticks, and its prices are STRINGS, so they need +# $number() before they satisfy the pack's numeric types. +response: | + ( + $records := $type(response) = "array" ? response + : $exists(response.data) ? response.data + : $exists(response.records) ? response.records + : []; + + $selected := beckn.message.contract.commitments[0]; + $ra := $selected.resources[0].resourceAttributes; + + /* Bound once because it is used twice -- for a resource's own id and for + the offer's reference to it. Two copies of one expression is how a + dangling reference gets reintroduced. */ + /* Built from CODES, not the names the upstream reports. A market name + carries spaces and a commodity name carries brackets -- "Kasdol APMC", + "Paddy(Common)" -- and an identifier that a consumer may put in a URL or + a filter should not. The codes are already in the payload, so they cost + nothing, and they are stable where a display name is not. + + The market code is optional: without it the query widened to the whole + district, so the district code is what identifies the scope. */ + $iso := function($ddmmyyyy) { + $substring($ddmmyyyy, 6, 4) & "-" & $substring($ddmmyyyy, 3, 2) & "-" & $substring($ddmmyyyy, 0, 2) + }; + + $scope := $exists($ra.market.marketCode) ? $ra.market.marketCode : $ra.market.district; + $resourceId := function($r) { + "res:agmarknet:" & $scope & ":" & $ra.supportedCommodities[0].code + & ":" & $iso($r.`Arrival Date`) + }; + + /* dd-MM-yyyy back to ISO, so the answer speaks the pack's date format + rather than the upstream's. */ + + /* Absent rather than present-and-empty: a consumer must be able to tell + "the market reported no minimum" from "the minimum was zero". */ + $priced := function($value) { $exists($value) ? $number($value) }; + + { + "context": { + "version": beckn.context.version, + "action": "on_select", + "networkId": beckn.context.networkId, + "transactionId": beckn.context.transactionId, + "messageId": beckn.context.messageId, + "timestamp": $now() + }, + "message": { + "contract": { + "commitments": [ + { + "status": { + "descriptor": { "code": "DRAFT", "name": "Draft" } + }, + /* The offer is echoed, but its references are not: the request + named an abstract price enquiry and the answer returns the + concrete observations. Leaving resourceIds as they arrived + would point the offer at an id appearing nowhere here. */ + "offer": $merge([ + $selected.offer, + { "resourceIds": [$map($records, function($r) { $resourceId($r) })] } + ]), + /* Wrapped: JSONata collapses a one-element sequence to a bare + value, so a single-record answer would return an object where + every other count returns a list. */ + "resources": [$map($records, function($r) { + { + "id": $resourceId($r), + /* Required by Commitment.resources in the Beckn v2 spec, + which defines no quantity property and carries no Quantity + schema at all -- a defect upstream. One resource is one + market's observation for one day, so one. */ + "quantity": 1, + "resourceAttributes": { + "@context": "https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld", + "@type": "openagrinet:MandiPrice", + "informationMode": "Direct", + "subjectCategories": $ra.subjectCategories, + "source": { + "sourceId": "agmarknet", + "sourceName": "Agmarknet Vistaar" + }, + "commodity": { + "code": $ra.supportedCommodities[0].code, + "name": $r.Commodity + }, + "commodityGroup": $r.Group, + "variety": $r.Variety, + "grade": $r.Grade, + "market": { + "marketName": $r.Market, + "marketCode": $ra.market.marketCode, + "district": $r.District, + "state": $r.State + }, + "arrivalDate": $iso($r.`Arrival Date`), + "prices": { + "minimum": $priced($r.`Min Price`), + "maximum": $priced($r.`Max Price`), + "modal": $number($r.`Modal Price`), + "currency": "INR", + "unit": $r.`Price Unit` + }, + "generatedAt": $now() + } + } + })] + } + ] + } + } + } + ) diff --git a/install/build-plugins.sh b/install/build-plugins.sh index 3bf07a01..8ac0d713 100755 --- a/install/build-plugins.sh +++ b/install/build-plugins.sh @@ -33,6 +33,7 @@ plugins=( "oanregistry" "jsonmapper" "weather" + "mandi" "manifestloader" "reqpreprocessor" "otelsetup" diff --git a/pkg/plugin/implementation/mandi/cmd/plugin.go b/pkg/plugin/implementation/mandi/cmd/plugin.go new file mode 100644 index 00000000..5b4fbb70 --- /dev/null +++ b/pkg/plugin/implementation/mandi/cmd/plugin.go @@ -0,0 +1,104 @@ +// Command plugin builds the mandi provider step as a loadable plugin. +// +// The filename of the built .so is the id a deployment names in providerSteps, +// so this package is mandi's whole public surface: a config map in, a step out. +package main + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/beckn-one/beckn-onix/pkg/log" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/mandi" +) + +// mandiProvider implements definition.ProviderStepProvider. +type mandiProvider struct{} + +// newStepFunc creates a new step. Indirected for tests. +var newStepFunc = mandi.New + +// parseConfig turns the plugin config map into a typed Config. Anything absent +// is left zero: mandi.New applies the defaults and validates the auth scheme, +// so those rules live in one place. +func (p mandiProvider) parseConfig(config map[string]string) (*mandi.Config, error) { + cfg := &mandi.Config{ + BindingKeys: splitList(config["bindingKeys"]), + // Absent means the Beckn v2 convention. See upstream.Config for why + // this is a default rather than something to set. + ProviderIDAt: config["providerIdAt"], + CapabilityCodeAt: config["capabilityCodeAt"], + AuthScheme: config["authScheme"], + UsernameEnv: config["usernameEnv"], + PasswordEnv: config["passwordEnv"], + HeaderName: config["headerName"], + HeaderValueEnv: config["headerValueEnv"], + QueryName: config["queryName"], + QueryValueEnv: config["queryValueEnv"], + } + + if raw, exists := config["maxResponseBytes"]; exists && raw != "" { + value, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid maxResponseBytes value '%s': %w", raw, err) + } + if value <= 0 { + return nil, fmt.Errorf("maxResponseBytes must be positive, got %d", value) + } + cfg.MaxResponseBytes = value + } + + return cfg, nil +} + +// New creates a new mandi provider step instance. +func (p mandiProvider) New(ctx context.Context, registry definition.ProviderRecordLookup, mapper definition.Mapper, config map[string]string) (definition.Step, func() error, error) { + if ctx == nil { + return nil, nil, errors.New("context cannot be nil") + } + + cfg, err := p.parseConfig(config) + if err != nil { + log.Errorf(ctx, err, "Failed to parse mandi configuration") + return nil, nil, fmt.Errorf("failed to parse mandi configuration: %w", err) + } + + step, closer, err := newStepFunc(ctx, registry, mapper, cfg) + if err != nil { + log.Errorf(ctx, err, "Failed to create mandi step") + return nil, nil, err + } + + log.Infof(ctx, "Mandi step created successfully") + return step, closer, nil +} + +// splitList reads a comma-separated config value, which is how a list reaches a +// plugin -- the config is map[string]string. Blanks are dropped and spaces +// trimmed, so a trailing comma or a wrapped line is not a config error. +// +// A comma is unambiguous here: a binding key separates its own halves with a +// pipe. +func splitList(raw string) []string { + if strings.TrimSpace(raw) == "" { + return nil + } + var out []string + for _, part := range strings.Split(raw, ",") { + if trimmed := strings.TrimSpace(part); trimmed != "" { + out = append(out, trimmed) + } + } + return out +} + +// Provider is the exported plugin instance. +var Provider = mandiProvider{} + +// Compile-time proof the provider satisfies the interface the manager asserts +// against. A mismatch is otherwise a runtime cast failure at startup. +var _ definition.ProviderStepProvider = Provider diff --git a/pkg/plugin/implementation/mandi/mandi.go b/pkg/plugin/implementation/mandi/mandi.go new file mode 100644 index 00000000..26e8b94b --- /dev/null +++ b/pkg/plugin/implementation/mandi/mandi.go @@ -0,0 +1,38 @@ +// Package mandi serves the network's market price capabilities. +// +// One package per schema pack family, so which plugin owns a capability is +// readable from its binding key: openagrinet:MandiPrice is mandi's, +// openagrinet:WeatherObservation is weather's. +// +// Almost nothing lives here, and that is the point. Recognising a capability, +// resolving the call plan, authenticating, calling with the registry's budget +// and translating in both directions are all internal/upstream's, because none +// of them differ by domain. What this package owns is its name, and +// prerequisites -- the work a mapping cannot express, which is domain knowledge +// by definition. +// +// The upstream this was written against is Agmarknet's Vistaar API, whose +// select takes governed codes for state, district, market and commodity plus a +// date range, all of which a MandiPrice payload carries. So the package is a +// name and nothing else: see prerequisites.go for why that is worth stating. +package mandi + +import ( + "context" + + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" +) + +// Config is upstream's, unchanged. Aliased here so a domain plugin's cmd package +// need not know where the machinery lives. +type Config = upstream.Config + +// New creates the mandi step. +// +// Which capabilities it answers to is configuration, with no default: a package +// serving a family cannot guess which of them a deployment has providers for. +func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper definition.Mapper, + cfg *Config) (definition.Step, func() error, error) { + return upstream.New(ctx, registry, mapper, prerequisites, cfg) +} diff --git a/pkg/plugin/implementation/mandi/mappings_test.go b/pkg/plugin/implementation/mandi/mappings_test.go new file mode 100644 index 00000000..1eae729f --- /dev/null +++ b/pkg/plugin/implementation/mandi/mappings_test.go @@ -0,0 +1,458 @@ +package mandi_test + +// mappings_test.go runs the shipped mandi mapping through the real mapper and +// the real provider step. It is the only test that proves the three pieces fit: +// a mapping is JSONata inside YAML fetched over HTTP, and nothing but running +// it establishes that what is published actually produces valid Beckn. +// +// An external test package on purpose -- it uses the plugins exactly as the +// adapter does, through their exported surface and nothing else. + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/mandi" +) + +// mappingsDir is where the shipped mappings live, relative to this package. +const mappingsDir = "../../../../config/mappings/agmarknet" + +// shippedMapping is the file this binding-action publishes: one file, both +// directions. The action segment of the name must match the action the registry +// entry declares -- a mismatch would apply a correct mapping to the wrong call, +// silently. +const shippedMapping = "mandi-price.select.yaml" + +// shippedCapability is what the pack calls this capability, and the second half +// of the binding key the registry indexes the provider's record by. +const shippedCapability = "openagrinet:MandiPrice" + +const shippedBindingKey = "agmarknet|" + shippedCapability + +// selectRequest is a MandiPrice select in OnDemand mode: it names the market and +// commodity it wants prices for, and carries no prices of its own -- the pack +// forbids that combination. +const selectRequest = `{ + "context": { + "version": "2.0.0", + "action": "select", + "networkId": "oan-dev", + "transactionId": "9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44", + "messageId": "7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22", + "timestamp": "2026-09-03T06:12:01.330Z" + }, + "message": { + "contract": { + "commitments": [ + { + "status": { "descriptor": { "code": "DRAFT", "name": "Draft" } }, + "resources": [ + { + "id": "res:agmarknet:price-enquiry", + "quantity": 1, + "resourceAttributes": { + "@context": "https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld", + "@type": "openagrinet:MandiPrice", + "informationMode": "OnDemand", + "subjectCategories": ["Market"], + "supportedCommodities": [{ "code": "2", "name": "Paddy(Common)" }], + "supportedPriceFields": ["Minimum", "Maximum", "Modal"], + "market": { + "marketName": "Kasdol APMC", + "marketCode": "2056", + "district": "96", + "state": "CG" + }, + "validity": { "startsAt": "2025-08-20", "endsAt": "2025-08-21" } + } + } + ], + "offer": { + "id": "offer:agmarknet:open-data", + "resourceIds": ["res:agmarknet:price-enquiry"], + "provider": { + "id": "agmarknet", + "descriptor": { "code": "AGMARKNET-01", "name": "Agmarknet Vistaar" } + } + } + } + ] + } + } +}` + +// providerResponse is a verbatim Agmarknet Vistaar answer, taken from the +// working example in the provider backend's own documentation. Two records, so +// the mapping is exercised on a list rather than a single object. +// +// Note what it is: Title Case keys WITH SPACES, and prices as STRINGS. Both are +// the reason the mapping needs backticks and $number, and pinning a real +// capture here is what keeps that honest. +const providerResponse = `[ + { + "Grade": "Non-FAQ", + "Group": "Cereals", + "State": "Chattisgarh", + "Market": "Kasdol APMC", + "Variety": "D.B.", + "District": "Balodabazar", + "Commodity": "Paddy(Common)", + "Max Price": "2100", + "Min Price": "1900", + "Price Unit": "Rs./Qtl", + "Modal Price": "2000", + "Arrival Date": "20-08-2025" + }, + { + "Grade": "FAQ", + "Group": "Cereals", + "State": "Chattisgarh", + "Market": "Kasdol APMC", + "Variety": "Common", + "District": "Balodabazar", + "Commodity": "Paddy(Common)", + "Price Unit": "Rs./Qtl", + "Modal Price": "2050", + "Arrival Date": "21-08-2025" + } +]` + +// serveMappings publishes the shipped mapping files over HTTP, which is how the +// mapper fetches them in production. +func serveMappings(t *testing.T) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := os.ReadFile(filepath.Join(mappingsDir, filepath.Base(r.URL.Path))) + if err != nil { + t.Errorf("could not read the mapping %q: %v", r.URL.Path, err) + w.WriteHeader(http.StatusNotFound) + return + } + fmt.Fprint(w, string(body)) + })) +} + +// stubRegistry answers with the call plan the live registry holds for this +// capability. +type stubRegistry struct{ plan *model.ProviderRecord } + +func (s *stubRegistry) ProviderRecord(context.Context, string) (*model.ProviderRecord, error) { + return s.plan, nil +} + +// runShipped drives the real step over the real mapping and returns the query +// the provider saw and the answer produced. +func runShipped(t *testing.T, request string) (url.Values, map[string]any) { + t.Helper() + + mappings := serveMappings(t) + defer mappings.Close() + + var gotQuery url.Values + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotQuery = r.URL.Query() + fmt.Fprint(w, providerResponse) + })) + defer upstream.Close() + + mapper, closeMapper, err := jsonmapper.New(context.Background(), &jsonmapper.Config{}) + if err != nil { + t.Fatalf("failed to build the mapper: %v", err) + } + defer closeMapper() + + registry := &stubRegistry{plan: &model.ProviderRecord{ + BindingKey: shippedBindingKey, + ParticipantID: "agmarknet", + CapabilityCode: shippedCapability, + BaseURL: upstream.URL, + Actions: map[string]model.ActionPlan{ + "select": {Method: http.MethodGet, Path: "/v1/fetch-agmarknet-vistaar", + Mappings: mappings.URL + "/" + shippedMapping, TimeoutMs: 30000, RetryMax: 3}, + }, + }} + + step, closeStep, err := mandi.New(context.Background(), registry, mapper, + &mandi.Config{BindingKeys: []string{shippedBindingKey}}) + if err != nil { + t.Fatalf("failed to build the step: %v", err) + } + defer closeStep() + + stepCtx := &model.StepContext{Context: t.Context(), Body: []byte(request)} + if err := step.Run(stepCtx); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + if len(stepCtx.ResponseBody) == 0 { + t.Fatal("the step produced no answer") + } + var answer map[string]any + if err := json.Unmarshal(stepCtx.ResponseBody, &answer); err != nil { + t.Fatalf("the answer is not JSON: %v\n%s", err, stepCtx.ResponseBody) + } + return gotQuery, answer +} + +func TestShippedMappingServesARealSelect(t *testing.T) { + gotQuery, answer := runShipped(t, selectRequest) + + // --- the request reached the provider as Agmarknet expects -------------- + // Every one of these comes off the payload. Nothing was resolved before the + // call, which is the whole claim of this plugin having no prerequisites. + for param, want := range map[string]string{ + "statecode": "CG", + "districtcode": "96", + "marketcode": "2056", + "commoditycode": "2", + // dd-MM-yyyy, not the ISO the payload carried. + "from_date": "20-08-2025", + "to_date": "21-08-2025", + } { + if got := gotQuery.Get(param); got != want { + t.Errorf("upstream query %s = %q, want %q", param, got, want) + } + } + // The credential is the adapter's business, never the mapping's. + if gotQuery.Has("token") { + t.Error("the mapping must not put a token in the query; authScheme does that") + } + + // --- the answer is Beckn ----------------------------------------------- + beckncontext, _ := answer["context"].(map[string]any) + if beckncontext["action"] != "on_select" { + t.Errorf("action = %v, want on_select", beckncontext["action"]) + } + if beckncontext["transactionId"] != "9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44" { + t.Errorf("transactionId = %v, want the one from the request", beckncontext["transactionId"]) + } + // A mapping transforms a payload; it does not assert who anyone is. + for _, field := range []string{"bapId", "bapUri", "bppId", "bppUri"} { + if _, present := beckncontext[field]; present { + t.Errorf("response context carries %q; a mapping must not assert identity", field) + } + } + + // Written out so the answer can be validated against the Beckn v2 spec and + // the MandiPrice pack by tooling outside Go. Skipped unless asked for. + if path := os.Getenv("MANDI_DUMP_ANSWER"); path != "" { + raw, _ := json.MarshalIndent(answer, "", " ") + if err := os.WriteFile(path, raw, 0o600); err != nil { + t.Fatalf("could not write the answer: %v", err) + } + } + + commitment := firstCommitment(t, answer) + if status := commitment["status"].(map[string]any)["descriptor"].(map[string]any); status["code"] != "DRAFT" { + t.Errorf("status = %v, want DRAFT -- the spec's enum is DRAFT, ACTIVE, CLOSED", status["code"]) + } + + // --- one resource per price record -------------------------------------- + resources, _ := commitment["resources"].([]any) + if len(resources) != 2 { + t.Fatalf("got %d resources, want 2 -- one per record the provider answered with", len(resources)) + } + + returned := make([]string, 0, len(resources)) + for _, entry := range resources { + resource, _ := entry.(map[string]any) + id, _ := resource["id"].(string) + // Codes, not names: no spaces or brackets in an identifier. + if !strings.HasPrefix(id, "res:agmarknet:2056:2:") { + t.Errorf("resource id = %q, want one built from the market and commodity codes", id) + } + if strings.ContainsAny(id, " ()") { + t.Errorf("resource id %q contains a space or bracket; use codes, not display names", id) + } + // Required by Commitment.resources in the spec even though the spec + // defines no quantity property. + if _, present := resource["quantity"]; !present { + t.Errorf("resource %s carries no quantity", id) + } + returned = append(returned, id) + } + + // The offer must reference what was actually returned, not what was asked + // for. This is the assertion that fails the moment the offer is echoed. + offer, _ := commitment["offer"].(map[string]any) + referenced, _ := offer["resourceIds"].([]any) + if len(referenced) != len(returned) { + t.Fatalf("offer references %d resources, want %d", len(referenced), len(returned)) + } + for _, reference := range referenced { + if !slices.Contains(returned, reference.(string)) { + t.Errorf("offer references %v, which is not among the resources returned", reference) + } + } + if offer["id"] != "offer:agmarknet:open-data" { + t.Errorf("offer id = %v, want the one the request offered", offer["id"]) + } + + // --- the MandiPrice pack, Direct mode ----------------------------------- + first, _ := resources[0].(map[string]any) + attributes, _ := first["resourceAttributes"].(map[string]any) + for _, f := range []struct{ key, want string }{ + {"@type", "openagrinet:MandiPrice"}, + {"informationMode", "Direct"}, + } { + if attributes[f.key] != f.want { + t.Errorf("%s = %v, want %v", f.key, attributes[f.key], f.want) + } + } + // Direct requires all six of these. + for _, required := range []string{"source", "commodity", "market", "arrivalDate", "prices", "generatedAt"} { + if attributes[required] == nil { + t.Errorf("resourceAttributes carries no %q", required) + } + } + // OnDemand's fields must NOT appear: the pack forbids prices alongside + // them, and an answer advertising a capability is a category error. + for _, absent := range []string{"supportedCommodities", "supportedPriceFields"} { + if _, present := attributes[absent]; present { + t.Errorf("a Direct answer must not carry %q", absent) + } + } + + // --- the prices, converted from strings --------------------------------- + prices, _ := attributes["prices"].(map[string]any) + for field, want := range map[string]float64{"minimum": 1900, "maximum": 2100, "modal": 2000} { + got, ok := prices[field].(float64) + if !ok { + t.Errorf("prices.%s = %#v, want a number -- the upstream sends strings", field, prices[field]) + continue + } + if got != want { + t.Errorf("prices.%s = %v, want %v", field, got, want) + } + } + if prices["currency"] != "INR" || prices["unit"] != "Rs./Qtl" { + t.Errorf("prices currency/unit = %v/%v, want INR/Rs./Qtl", prices["currency"], prices["unit"]) + } + + // arrivalDate is ISO in the answer, though the upstream reported dd-MM-yyyy. + if attributes["arrivalDate"] != "2025-08-20" { + t.Errorf("arrivalDate = %v, want 2025-08-20 in ISO", attributes["arrivalDate"]) + } + + // The pack's enum is Crop, Livestock, Weather, Market, Scheme, Knowledge, + // Service -- so "Market", not "MarketPrice". Echoed from the request, which + // is why getting it wrong there would produce an invalid answer here. + categories, _ := attributes["subjectCategories"].([]any) + if len(categories) != 1 || categories[0] != "Market" { + t.Errorf("subjectCategories = %v, want [Market] from the pack's enum", categories) + } + + market, _ := attributes["market"].(map[string]any) + if market["marketName"] != "Kasdol APMC" || market["state"] != "Chattisgarh" { + t.Errorf("market = %v, want the names the provider reported", market) + } + + // --- a record the market reported only partially ------------------------- + // The second record has no Min or Max Price. Those must be absent, not zero: + // a consumer must be able to tell "not reported" from "reported as zero". + second, _ := resources[1].(map[string]any) + secondPrices, _ := second["resourceAttributes"].(map[string]any)["prices"].(map[string]any) + for _, absent := range []string{"minimum", "maximum"} { + if _, present := secondPrices[absent]; present { + t.Errorf("prices.%s is present for a record that did not report it", absent) + } + } + if secondPrices["modal"] != float64(2050) { + t.Errorf("the second record's modal price = %v, want 2050", secondPrices["modal"]) + } +} + +// The pack leaves every field this upstream needs optional, so a spec-valid +// select can still be unanswerable. The mapping refuses those before the +// provider is called, with its own message. +func TestShippedMappingRefusesWhatItCannotServe(t *testing.T) { + for _, tc := range []struct{ name, drop, expect string }{ + {"no commodity code", "supportedCommodities", "commodity code"}, + {"no market codes", "market", "state and district"}, + {"no validity window", "validity", "validity window"}, + } { + t.Run(tc.name, func(t *testing.T) { + // Built by deleting a key from the decoded fixture rather than by + // editing its text: removing the last member of an object leaves a + // trailing comma, and the resulting parse error would look like a + // mapping failure. + var payload map[string]any + if err := json.Unmarshal([]byte(selectRequest), &payload); err != nil { + t.Fatalf("the fixture is not JSON: %v", err) + } + attributes := payload["message"].(map[string]any)["contract"].(map[string]any)["commitments"].([]any)[0].(map[string]any)["resources"].([]any)[0].(map[string]any)["resourceAttributes"].(map[string]any) + if _, present := attributes[tc.drop]; !present { + t.Fatalf("the fixture has no %q, so this case tests nothing", tc.drop) + } + delete(attributes, tc.drop) + body, err := json.Marshal(payload) + if err != nil { + t.Fatalf("could not rebuild the payload: %v", err) + } + + mappings := serveMappings(t) + defer mappings.Close() + + called := false + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + called = true + fmt.Fprint(w, providerResponse) + })) + defer upstream.Close() + + mapper, closeMapper, err := jsonmapper.New(context.Background(), &jsonmapper.Config{}) + if err != nil { + t.Fatalf("failed to build the mapper: %v", err) + } + defer closeMapper() + + registry := &stubRegistry{plan: &model.ProviderRecord{ + BindingKey: shippedBindingKey, BaseURL: upstream.URL, + Actions: map[string]model.ActionPlan{ + "select": {Method: http.MethodGet, Path: "/v1/fetch-agmarknet-vistaar", + Mappings: mappings.URL + "/" + shippedMapping, TimeoutMs: 30000}, + }, + }} + step, closeStep, err := mandi.New(context.Background(), registry, mapper, + &mandi.Config{BindingKeys: []string{shippedBindingKey}}) + if err != nil { + t.Fatalf("failed to build the step: %v", err) + } + defer closeStep() + + stepCtx := &model.StepContext{Context: t.Context(), Body: body} + if err := step.Run(stepCtx); err == nil { + t.Fatal("expected an unserviceable payload to be refused") + } else if !strings.Contains(err.Error(), tc.expect) { + t.Errorf("error %q should carry the mapping's own message about %q", err, tc.expect) + } + if called { + t.Error("the provider was called for a payload the mapping refuses") + } + }) + } +} + +// firstCommitment reaches the one commitment an answer carries. +func firstCommitment(t *testing.T, answer map[string]any) map[string]any { + t.Helper() + message, _ := answer["message"].(map[string]any) + contract, _ := message["contract"].(map[string]any) + commitments, _ := contract["commitments"].([]any) + if len(commitments) != 1 { + t.Fatalf("got %d commitments, want 1", len(commitments)) + } + commitment, _ := commitments[0].(map[string]any) + return commitment +} diff --git a/pkg/plugin/implementation/mandi/prerequisites.go b/pkg/plugin/implementation/mandi/prerequisites.go new file mode 100644 index 00000000..2027248b --- /dev/null +++ b/pkg/plugin/implementation/mandi/prerequisites.go @@ -0,0 +1,20 @@ +package mandi + +import "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" + +// prerequisites is what a mandi capability needs that its payload does not +// carry, keyed by binding key. +// +// Empty, and for a better reason than weather's: a market price select names +// the market it wants. The MandiPrice pack has no top-level location -- only +// market.marketCode, market.district and market.state -- so there is nothing +// to resolve. Agmarknet's Vistaar select takes exactly those codes, and the +// mapping reads them straight off the payload. +// +// An entry would be needed only for real I/O: a commodity name to resolve to a +// code, a token to exchange, a point to turn into a market. Each of those is a +// different upstream than the one this was written against, and each would +// bring the question of where the provider-to-function binding belongs -- see +// the note in weather/prerequisites.go and prefer keeping the payload explicit +// over adding an entry here. +var prerequisites = upstream.Prerequisites{} From 8fc61043afac233ee313abdc5432ecc8a4820c03 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Sun, 6 Sep 2026 18:31:06 +0530 Subject: [PATCH 073/130] docs(config): serve both capabilities from the reference config [#8] Adds mandi alongside weather in config/oan-provider-adapter.yaml, so the reference shows the thing that is actually interesting about this design: two domain packages in one pipeline, sharing the module, the registry client and the mapper, and sharing nothing else. The whole cost of the second capability is one providerSteps entry and one line in steps. No routing table, no new module, no new port. Which one answers is decided by the payload -- each step builds a binding key from it, serves the request if the key is its own, and passes it through untouched if not -- so the order they appear in does not matter either. mandi uses authScheme query, because Agmarknet's Vistaar API takes its token as a query parameter. The adapter holds the parameter's name and the name of the environment variable carrying the value, never the value, and redacts it from the URL it logs -- so a token cannot reach the log by way of the request. Verified by booting this config in an image that has both plugins: both ProviderStep plugins load, the pipeline initialises as [validateSign validateSchema weather mandi signAck], and the module registers at /. Worth noting the published adapter image does NOT yet carry mandi.so, so against that image this config fails at startup with "plugin mandi not found" until it is rebuilt from this branch. --- config/oan-provider-adapter.yaml | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/config/oan-provider-adapter.yaml b/config/oan-provider-adapter.yaml index 2bba72e1..c35f041f 100644 --- a/config/oan-provider-adapter.yaml +++ b/config/oan-provider-adapter.yaml @@ -4,6 +4,11 @@ # capability's call plan from the registry, calls the provider, and answers with # the mapped result. There is no callback -- the answer is the HTTP response. # +# Two capabilities are served here, weather and mandi, by two domain packages +# in one pipeline. They share this module, the registry client and the mapper, +# and share nothing else: which one answers is decided by the payload, not by +# the URL, the domain or the order they appear in. +# # Adding a provider is three things, and none of them is a Go change here: # 1. a registry row binding "|" to a call plan # 2. one mapping file per action, published at the URL that row names @@ -270,6 +275,24 @@ modules: # # maxResponseBytes: 4194304 # default: 4194304 (4 MiB) + # A second capability in the same pipeline, from a different domain + # package. Nothing about it is weather's business: a different + # upstream, a different mapping, a different set of prerequisites -- + # and the same two registry rows. This entry, plus "mandi" in steps + # below, is the entire cost of adding it. + - id: mandi + config: + bindingKeys: "agmarknet|openagrinet:MandiPrice" + + # Agmarknet's Vistaar API takes its token as a QUERY parameter, + # which is what authScheme query is for. The adapter holds the + # parameter's NAME and the name of the variable carrying the + # value -- never the value -- and redacts it from the URL it + # logs, so a token cannot reach the log by way of the request. + authScheme: query + queryName: token + queryValueEnv: MANDI_TOKEN + # Declaring a step above is not enough: THIS list is what runs. A step # that appears under providerSteps but not here never executes, and the # request falls through to the 404 above -- which looks like a registry @@ -277,7 +300,8 @@ modules: steps: - validateSign # the sender's key, from the registry - validateSchema # the pinned Beckn v2 spec - - weather # resolve, map out, call, map back + - weather # openagrinet:WeatherObservation, or pass through + - mandi # openagrinet:MandiPrice, or pass through - signAck # signs whatever the step answered with # ---------------------------------------------------------------------------- From b6e954cf4a900e75fbe8688883bf046516021408 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 11:41:57 +0530 Subject: [PATCH 074/130] refactor(mappings): echo the caller's @context in the mandi mapping too [#8] Same change as the weather mapping: the response half read a hardcoded pack URL and now takes it off the incoming select, so the file never has to know which URL is current and cannot disagree with the caller. --- config/mappings/agmarknet/mandi-price.select.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/config/mappings/agmarknet/mandi-price.select.yaml b/config/mappings/agmarknet/mandi-price.select.yaml index 2f0e70b3..5492275d 100644 --- a/config/mappings/agmarknet/mandi-price.select.yaml +++ b/config/mappings/agmarknet/mandi-price.select.yaml @@ -93,6 +93,12 @@ response: | : []; $selected := beckn.message.contract.commitments[0]; + + /* The caller's own @context, echoed back rather than restated here. A + mapping that hardcodes it has to be reissued whenever the pack URL + moves, and it can disagree with what the request actually declared. + Backticks because @ is an operator in JSONata. */ + $ctx := beckn.message.contract.commitments[0].resources[0].resourceAttributes.`@context`; $ra := $selected.resources[0].resourceAttributes; /* Bound once because it is used twice -- for a resource's own id and for @@ -159,7 +165,7 @@ response: | market's observation for one day, so one. */ "quantity": 1, "resourceAttributes": { - "@context": "https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld", + "@context": $ctx, "@type": "openagrinet:MandiPrice", "informationMode": "Direct", "subjectCategories": $ra.subjectCategories, From ba5f92293a0b32346bbde2d38b3996d3eb785f7d Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 23:49:09 +0530 Subject: [PATCH 075/130] fix(mappings): guard the price cast on shape, not just presence [#8] Agmarknet writes an unreported price as a marker rather than omitting the field -- "NR", "-", "". $exists() is true for all of them, so the existence-only guard handed them to $number() and it threw: D3030: unable to cast value to a number: "NR" (argument 1) That failed the WHOLE response half. One unreported cell in one row turned a good multi-row answer into an adapter error -- a 500 where the honest answer is "this row has no minimum", which is exactly the distinction the comment above the function says it exists to preserve. modal had no guard at all: it went through a bare $number while minimum and maximum went through $priced. It uses $priced now, so the three agree. The number branch in the new guard is not redundant, and I checked rather than assumed: $match throws T0410 on a non-string, so testing the regex first would break the day this upstream sends a real number instead of a quoted one. Verified every input shape -- absent, "1441", "1441.50", 1441, "NR", "-", "", "1,441" -- and only the numeric ones survive as numbers. Test drives the shipped mapping with markers in all three price fields of one row and real prices in another. It asserts the good row still arrives with numbers AND that the markers are absent rather than zero. Confirmed it fails against the old guard, reporting the D3030 above verbatim. runShipped is split so a test can vary what the upstream returns; the existing callers are unchanged. --- .../agmarknet/mandi-price.select.yaml | 22 ++++- .../implementation/mandi/mappings_test.go | 93 ++++++++++++++++++- 2 files changed, 111 insertions(+), 4 deletions(-) diff --git a/config/mappings/agmarknet/mandi-price.select.yaml b/config/mappings/agmarknet/mandi-price.select.yaml index 5492275d..0350caf5 100644 --- a/config/mappings/agmarknet/mandi-price.select.yaml +++ b/config/mappings/agmarknet/mandi-price.select.yaml @@ -126,8 +126,24 @@ response: | rather than the upstream's. */ /* Absent rather than present-and-empty: a consumer must be able to tell - "the market reported no minimum" from "the minimum was zero". */ - $priced := function($value) { $exists($value) ? $number($value) }; + "the market reported no minimum" from "the minimum was zero". + + GUARDS SHAPE, NOT JUST PRESENCE. Agmarknet writes an unreported price as + a marker -- "NR", "-", "" -- and $exists() is true for all of them, so + an existence-only guard handed them to $number() and it threw D3030. + That failed the WHOLE response: one unreported cell in one row turned a + good multi-row answer into an adapter error, which is the opposite of + the distinction this function exists to preserve. + + The number branch is not redundant. $match() throws T0410 on a + non-string, so testing the regex first would break the day this + upstream sends a real number instead of a quoted one. */ + $priced := function($value) { + $type($value) = "number" + ? $value + : ($type($value) = "string" and $match($value, /^[0-9]+(\.[0-9]+)?$/) + ? $number($value)) + }; { "context": { @@ -190,7 +206,7 @@ response: | "prices": { "minimum": $priced($r.`Min Price`), "maximum": $priced($r.`Max Price`), - "modal": $number($r.`Modal Price`), + "modal": $priced($r.`Modal Price`), "currency": "INR", "unit": $r.`Price Unit` }, diff --git a/pkg/plugin/implementation/mandi/mappings_test.go b/pkg/plugin/implementation/mandi/mappings_test.go index 1eae729f..69cad30a 100644 --- a/pkg/plugin/implementation/mandi/mappings_test.go +++ b/pkg/plugin/implementation/mandi/mappings_test.go @@ -155,6 +155,13 @@ func (s *stubRegistry) ProviderRecord(context.Context, string) (*model.ProviderR // runShipped drives the real step over the real mapping and returns the query // the provider saw and the answer produced. func runShipped(t *testing.T, request string) (url.Values, map[string]any) { + return runShippedWith(t, request, providerResponse) +} + +// runShippedWith is runShipped with the upstream's answer under the test's +// control, for the cases where what Agmarknet returns is the thing being +// exercised rather than the request that fetched it. +func runShippedWith(t *testing.T, request, providerBody string) (url.Values, map[string]any) { t.Helper() mappings := serveMappings(t) @@ -163,7 +170,7 @@ func runShipped(t *testing.T, request string) (url.Values, map[string]any) { var gotQuery url.Values upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { gotQuery = r.URL.Query() - fmt.Fprint(w, providerResponse) + fmt.Fprint(w, providerBody) })) defer upstream.Close() @@ -456,3 +463,87 @@ func firstCommitment(t *testing.T, answer map[string]any) map[string]any { commitment, _ := commitments[0].(map[string]any) return commitment } + +// resourcesOf returns the answer's resources as maps, so a test can look at +// each record the provider's rows became. +func resourcesOf(t *testing.T, answer map[string]any) []map[string]any { + t.Helper() + + raw, _ := firstCommitment(t, answer)["resources"].([]any) + out := make([]map[string]any, 0, len(raw)) + for _, r := range raw { + m, ok := r.(map[string]any) + if !ok { + t.Fatalf("resource is not an object: %#v", r) + } + out = append(out, m) + } + return out +} + +// Agmarknet writes an unreported price as a marker rather than omitting the +// field -- "NR", "-", "". $exists() is true for all of them, so an +// existence-only guard handed them to $number() and it threw D3030, which +// failed the WHOLE response: one unreported cell in one row turned a good +// multi-row answer into an adapter error. +func TestShippedMappingSurvivesUnreportedPrices(t *testing.T) { + t.Parallel() + + // Row one has a marker in each of the three price fields; row two is + // ordinary. The point is that row two still arrives. + const withMarkers = `[ + { + "Grade": "Non-FAQ", "Group": "Cereals", "State": "Chattisgarh", + "Market": "Kasdol APMC", "Variety": "D.B.", "District": "Balodabazar", + "Commodity": "Paddy(Common)", + "Min Price": "NR", "Max Price": "-", "Modal Price": "", + "Price Unit": "Rs./Qtl", "Arrival Date": "20-08-2025" + }, + { + "Grade": "FAQ", "Group": "Cereals", "State": "Chattisgarh", + "Market": "Kasdol APMC", "Variety": "Common", "District": "Balodabazar", + "Commodity": "Paddy(Common)", + "Min Price": "1900", "Max Price": "2100", "Modal Price": "2000", + "Price Unit": "Rs./Qtl", "Arrival Date": "21-08-2025" + } + ]` + + _, answer := runShippedWith(t, selectRequest, withMarkers) + + resources := resourcesOf(t, answer) + if len(resources) == 0 { + t.Fatal("the answer carries no resources; one unreported cell discarded the lot") + } + + // The priced row keeps its numbers, as numbers. + var priced map[string]any + for _, r := range resources { + ra := r["resourceAttributes"].(map[string]any) + if ra["arrivalDate"] == "2025-08-21" { + priced = ra["prices"].(map[string]any) + } + } + if priced == nil { + t.Fatal("the row with real prices is missing from the answer") + } + for _, field := range []string{"minimum", "maximum", "modal"} { + if _, ok := priced[field].(float64); !ok { + t.Errorf("prices.%s = %#v, want a number", field, priced[field]) + } + } + + // And a marker is absent rather than zero, which is the distinction the + // guard exists to preserve. + for _, r := range resources { + ra := r["resourceAttributes"].(map[string]any) + if ra["arrivalDate"] != "2025-08-20" { + continue + } + p := ra["prices"].(map[string]any) + for _, field := range []string{"minimum", "maximum", "modal"} { + if v, present := p[field]; present { + t.Errorf("prices.%s = %#v for an unreported price; absent is honest, zero is a lie", field, v) + } + } + } +} From ebc1c91d4d5ed4d0f66db34bd0461e91654b46d3 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Tue, 8 Sep 2026 00:00:25 +0530 Subject: [PATCH 076/130] fix(mappings): emit only records that can produce a conformant resource [#8] Three findings, one rule. A record that cannot be made conformant is dropped rather than emitted with a degenerate value -- absent is honest, and present-and-wrong is a lie in the shape of an answer that this adapter then signs. no Arrival Date $iso is substring-and-concatenate and JSONata casts undefined to "", so an absent date became the string "--". The pack declares format: date, so that is invalid, not merely ugly -- and arrivalDate is on the Direct required list, so it cannot be omitted either. The resource id degraded with it: res:agmarknet:2056:2:-- no Price Unit prices.required is [currency, unit], and JSONata drops an absent key rather than emitting null, so the unit silently vanished and took the resource's validity with it. NOT defaulted: Rs./Qtl and Rs./Kg are both real, so inventing one would misreport a price by a factor of a hundred. no usable price prices carries anyOf [minimum, maximum, modal], so a row whose three prices are all unreported markers has nothing to report and cannot satisfy it. Dropped rather than refused, because the other rows in the same answer are good and failing the request would discard them too -- the mistake the price guard in 55ca67d just fixed. The offer's resourceIds are filtered with them, so dropping a record cannot leave a dangling reference; that is asserted. TWO THINGS I GOT WRONG AND THE TESTS CAUGHT, both worth recording. The first version used $exists($match(...)) for the date. This engine returns an EMPTY ARRAY from $match when nothing matches, and $exists([]) is true -- so the predicate was constant-true and the filter did nothing. It reads correctly and does nothing at all, which is the worst combination. Worse, I had verified it against the WRONG ENGINE. My probes used the JavaScript jsonata package; the adapter uses github.com/jsonata-go/jsonata, where $exists([]) differs. Verified in the real engine this time: bare $match in a ternary and $count(...) > 0 both behave, only the $exists form does not. And the filter made the previous commit's test vacuous -- its all-markers row is now dropped, so the loop asserting "markers are absent" never ran. That test now uses a row with one real price and two markers, so it still asserts the distinction, plus a third row that is dropped for having none. --- .../agmarknet/mandi-price.select.yaml | 43 +++++- .../implementation/mandi/mappings_test.go | 137 ++++++++++++++---- 2 files changed, 149 insertions(+), 31 deletions(-) diff --git a/config/mappings/agmarknet/mandi-price.select.yaml b/config/mappings/agmarknet/mandi-price.select.yaml index 0350caf5..4c4237ff 100644 --- a/config/mappings/agmarknet/mandi-price.select.yaml +++ b/config/mappings/agmarknet/mandi-price.select.yaml @@ -145,6 +145,45 @@ response: | ? $number($value)) }; + /* EMIT ONLY RECORDS THAT CAN PRODUCE A CONFORMANT RESOURCE. + Absent is honest; present-and-degenerate is a lie in the shape of an + answer, and it is worse here than elsewhere because the answer is + signed. Three ways a record cannot be made conformant: + + no Arrival Date $iso is substring-and-concatenate, and JSONata casts + undefined to "" -- so an absent date became the + string "--", which the pack refuses (format: date) + and which degraded the resource id along with it. + arrivalDate is on the Direct required list, so it + cannot be omitted either. + + no Price Unit prices.required is [currency, unit]. JSONata drops + an absent key rather than emitting null, so the unit + silently vanished and took the whole resource's + validity with it. Not defaulted: Rs./Qtl and Rs./Kg + are both real, so inventing one would misreport a + price by a factor of a hundred. + + no usable price prices carries anyOf [minimum, maximum, modal], so a + row whose three prices are all unreported markers + has nothing to report and cannot satisfy it. + + Dropped rather than refused: the other rows in the same answer are good, + and failing the request would discard them too -- which is the mistake + the price guard above just fixed. */ + $conformant := function($r) { + /* $count(...) > 0 rather than $exists($match(...)): this engine returns + an EMPTY ARRAY from $match when nothing matches, and $exists([]) is + true -- so the $exists form silently accepted every record and the + filter did nothing. Caught by the test, not by reading. */ + $count($match($iso($r.`Arrival Date`), /^[0-9]{4}-[0-9]{2}-[0-9]{2}$/)) > 0 + and $exists($r.`Price Unit`) + and ($exists($priced($r.`Min Price`)) + or $exists($priced($r.`Max Price`)) + or $exists($priced($r.`Modal Price`))) + }; + $usable := $filter($records, $conformant); + { "context": { "version": beckn.context.version, @@ -167,12 +206,12 @@ response: | would point the offer at an id appearing nowhere here. */ "offer": $merge([ $selected.offer, - { "resourceIds": [$map($records, function($r) { $resourceId($r) })] } + { "resourceIds": [$map($usable, function($r) { $resourceId($r) })] } ]), /* Wrapped: JSONata collapses a one-element sequence to a bare value, so a single-record answer would return an object where every other count returns a list. */ - "resources": [$map($records, function($r) { + "resources": [$map($usable, function($r) { { "id": $resourceId($r), /* Required by Commitment.resources in the Beckn v2 spec, diff --git a/pkg/plugin/implementation/mandi/mappings_test.go b/pkg/plugin/implementation/mandi/mappings_test.go index 69cad30a..829ba1cf 100644 --- a/pkg/plugin/implementation/mandi/mappings_test.go +++ b/pkg/plugin/implementation/mandi/mappings_test.go @@ -489,61 +489,140 @@ func resourcesOf(t *testing.T, answer map[string]any) []map[string]any { func TestShippedMappingSurvivesUnreportedPrices(t *testing.T) { t.Parallel() - // Row one has a marker in each of the three price fields; row two is - // ordinary. The point is that row two still arrives. + // Row one has a marker in TWO price fields and a real modal, so it stays + // and its markers must come back absent. Row two is ordinary. Row three + // has markers in all three, so it has nothing to report and is dropped. const withMarkers = `[ { "Grade": "Non-FAQ", "Group": "Cereals", "State": "Chattisgarh", "Market": "Kasdol APMC", "Variety": "D.B.", "District": "Balodabazar", "Commodity": "Paddy(Common)", - "Min Price": "NR", "Max Price": "-", "Modal Price": "", + "Min Price": "NR", "Max Price": "-", "Modal Price": "2000", "Price Unit": "Rs./Qtl", "Arrival Date": "20-08-2025" }, { "Grade": "FAQ", "Group": "Cereals", "State": "Chattisgarh", "Market": "Kasdol APMC", "Variety": "Common", "District": "Balodabazar", "Commodity": "Paddy(Common)", - "Min Price": "1900", "Max Price": "2100", "Modal Price": "2000", + "Min Price": "1900", "Max Price": "2100", "Modal Price": "2050", "Price Unit": "Rs./Qtl", "Arrival Date": "21-08-2025" + }, + { + "Grade": "FAQ", "Group": "Cereals", "State": "Chattisgarh", + "Market": "Kasdol APMC", "Variety": "Common", "District": "Balodabazar", + "Commodity": "Paddy(Common)", + "Min Price": "NR", "Max Price": "NR", "Modal Price": "NR", + "Price Unit": "Rs./Qtl", "Arrival Date": "22-08-2025" } ]` _, answer := runShippedWith(t, selectRequest, withMarkers) - resources := resourcesOf(t, answer) - if len(resources) == 0 { - t.Fatal("the answer carries no resources; one unreported cell discarded the lot") + byDate := map[string]map[string]any{} + for _, r := range resourcesOf(t, answer) { + ra := r["resourceAttributes"].(map[string]any) + byDate[ra["arrivalDate"].(string)] = ra } - // The priced row keeps its numbers, as numbers. - var priced map[string]any - for _, r := range resources { - ra := r["resourceAttributes"].(map[string]any) - if ra["arrivalDate"] == "2025-08-21" { - priced = ra["prices"].(map[string]any) + // The row with a real modal survives -- one unreported cell must not + // discard it, and must not discard the rows beside it either. + partial, ok := byDate["2025-08-20"] + if !ok { + t.Fatal("the partially-priced row is missing; an unreported cell discarded it") + } + prices := partial["prices"].(map[string]any) + if _, isNum := prices["modal"].(float64); !isNum { + t.Errorf("prices.modal = %#v, want the reported number", prices["modal"]) + } + // Absent, not zero: a consumer must tell "no minimum reported" from + // "the minimum was zero". + for _, field := range []string{"minimum", "maximum"} { + if v, present := prices[field]; present { + t.Errorf("prices.%s = %#v for an unreported price; absent is honest, zero is a lie", field, v) } } - if priced == nil { - t.Fatal("the row with real prices is missing from the answer") + + if _, ok := byDate["2025-08-21"]; !ok { + t.Error("the fully-priced row is missing from the answer") + } + + // Nothing to report at all: the pack's prices.anyOf cannot be satisfied, + // so the row is dropped rather than emitted as an invalid resource. + if _, ok := byDate["2025-08-22"]; ok { + t.Error("a row whose every price is unreported was emitted; it cannot satisfy prices.anyOf") } - for _, field := range []string{"minimum", "maximum", "modal"} { - if _, ok := priced[field].(float64); !ok { - t.Errorf("prices.%s = %#v, want a number", field, priced[field]) +} + +// A record that cannot produce a conformant resource is dropped, not emitted +// with a degenerate value. Absent is honest; present-and-wrong is a lie in the +// shape of an answer, and it is signed. +func TestShippedMappingDropsRecordsItCannotMakeConformant(t *testing.T) { + t.Parallel() + + // One good row, then one for each way a record fails the pack. + const mixed = `[ + { + "Grade": "FAQ", "Group": "Cereals", "State": "Chattisgarh", + "Market": "Kasdol APMC", "Variety": "Common", "District": "Balodabazar", + "Commodity": "Paddy(Common)", + "Min Price": "1900", "Max Price": "2100", "Modal Price": "2000", + "Price Unit": "Rs./Qtl", "Arrival Date": "20-08-2025" + }, + { + "Grade": "FAQ", "Group": "Cereals", "State": "Chattisgarh", + "Market": "Kasdol APMC", "Variety": "Common", "District": "Balodabazar", + "Commodity": "Paddy(Common)", + "Min Price": "1800", "Max Price": "2000", "Modal Price": "1900", + "Price Unit": "Rs./Qtl" + }, + { + "Grade": "FAQ", "Group": "Cereals", "State": "Chattisgarh", + "Market": "Kasdol APMC", "Variety": "Common", "District": "Balodabazar", + "Commodity": "Paddy(Common)", + "Min Price": "1700", "Max Price": "1900", "Modal Price": "1800", + "Arrival Date": "23-08-2025" + } + ]` + + _, answer := runShippedWith(t, selectRequest, mixed) + + resources := resourcesOf(t, answer) + if len(resources) != 1 { + var dates []string + for _, r := range resources { + ra := r["resourceAttributes"].(map[string]any) + dates = append(dates, fmt.Sprint(ra["arrivalDate"])) } + t.Fatalf("got %d resources with dates %v, want only the conformant one", len(resources), dates) + } + + ra := resources[0]["resourceAttributes"].(map[string]any) + if ra["arrivalDate"] != "2025-08-20" { + t.Errorf("arrivalDate = %v, want the one good record", ra["arrivalDate"]) + } + // The degenerate date the old mapping produced, specifically. + if ra["arrivalDate"] == "--" { + t.Error(`arrivalDate = "--", which the pack refuses as format: date`) + } + if _, ok := ra["prices"].(map[string]any)["unit"]; !ok { + t.Error("prices.unit is missing, which the pack requires") } - // And a marker is absent rather than zero, which is the distinction the - // guard exists to preserve. + // The offer must not reference a resource the filter removed -- dropping a + // record and leaving its id in resourceIds would trade an invalid resource + // for a dangling reference. + ids := map[string]bool{} for _, r := range resources { - ra := r["resourceAttributes"].(map[string]any) - if ra["arrivalDate"] != "2025-08-20" { - continue - } - p := ra["prices"].(map[string]any) - for _, field := range []string{"minimum", "maximum", "modal"} { - if v, present := p[field]; present { - t.Errorf("prices.%s = %#v for an unreported price; absent is honest, zero is a lie", field, v) - } + ids[r["id"].(string)] = true + } + offer, _ := firstCommitment(t, answer)["offer"].(map[string]any) + referenced, _ := offer["resourceIds"].([]any) + if len(referenced) != len(resources) { + t.Errorf("offer references %d resources, want %d", len(referenced), len(resources)) + } + for _, ref := range referenced { + if !ids[fmt.Sprint(ref)] { + t.Errorf("the offer references %q, which is not among the answer's resources", ref) } } } From aaac1b441ee5a05453bca31404d2e251df56f483 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Tue, 8 Sep 2026 00:07:14 +0530 Subject: [PATCH 077/130] fix(mappings): state what the answer knows instead of echoing the request [#8] Two echoes, both of them claims this adapter signs. subjectCategories was $ra.subjectCategories. It is a closed enum on AgricultureResource and "Market" is what a MandiPrice resource IS -- the pack states it in both of its own examples, and the sibling weather mapping has always stated ["Weather"]. Echoing meant a caller sending ["Weather"] on a MandiPrice select got it faithfully republished over this adapter's signature, and nothing caught it: ["Weather"] is enum-legal, so it validates. Verified that -- it comes back VALID against the pack, which is what made the echo dangerous rather than merely untidy. market took marketName, district and state from the record but marketCode from the REQUEST. The review suggested taking the code from the record too; there is nothing to take. This upstream TAKES a market code as a query parameter and reports none back -- there is no such field anywhere in its response, which I checked against the PR's own fixture. So the code is dropped rather than sourced differently: - restating the requested code against a returned row asserts something unverified, and would misreport a provider that answered about a different market instead of surfacing it; - district-wide there is no requested code at all, while the rows come from several markets, so the single code was wrong for most of them. The pack requires only marketName and calls marketCode "when available". Here it is not available, and absent is the honest answer. One test covering both, driven by a request that states the WRONG category, so it fails on an echo of either field. Confirmed it fails against both old forms. --- .../agmarknet/mandi-price.select.yaml | 21 +++++++++- .../implementation/mandi/mappings_test.go | 41 +++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/config/mappings/agmarknet/mandi-price.select.yaml b/config/mappings/agmarknet/mandi-price.select.yaml index 4c4237ff..7f9e4d4a 100644 --- a/config/mappings/agmarknet/mandi-price.select.yaml +++ b/config/mappings/agmarknet/mandi-price.select.yaml @@ -223,7 +223,14 @@ response: | "@context": $ctx, "@type": "openagrinet:MandiPrice", "informationMode": "Direct", - "subjectCategories": $ra.subjectCategories, + /* Stated, not echoed. subjectCategories is a closed enum on + AgricultureResource, and "Market" is what a MandiPrice resource + IS -- the pack states it in both of its own examples. Echoing the + request meant a caller sending ["Weather"] on a MandiPrice select + got it faithfully republished over this adapter's signature, and + nothing caught it: the value is enum-legal, so it validates. The + sibling weather mapping has always stated ["Weather"]. */ + "subjectCategories": ["Market"], "source": { "sourceId": "agmarknet", "sourceName": "Agmarknet Vistaar" @@ -235,9 +242,19 @@ response: | "commodityGroup": $r.Group, "variety": $r.Variety, "grade": $r.Grade, + /* Every member comes from the RECORD, so the answer describes what + the provider reported rather than what was asked for. + + marketCode is absent, and that is not an omission. This upstream + TAKES a market code as a query parameter and reports none back -- + there is no such field anywhere in its response. Restating the + requested code against a returned row would assert something + unverified, and district-wide there is no requested code at all + while the rows come from several markets, so one code would have + been wrong for most of them. The pack requires only marketName + and calls marketCode "when available"; here it is not. */ "market": { "marketName": $r.Market, - "marketCode": $ra.market.marketCode, "district": $r.District, "state": $r.State }, diff --git a/pkg/plugin/implementation/mandi/mappings_test.go b/pkg/plugin/implementation/mandi/mappings_test.go index 829ba1cf..452b2075 100644 --- a/pkg/plugin/implementation/mandi/mappings_test.go +++ b/pkg/plugin/implementation/mandi/mappings_test.go @@ -626,3 +626,44 @@ func TestShippedMappingDropsRecordsItCannotMakeConformant(t *testing.T) { } } } + +// The answer must assert what it knows, not repeat what it was told. Both of +// these were echoes of the request, and an echo is a claim this adapter signs. +func TestShippedMappingStatesWhatItKnowsRatherThanEchoing(t *testing.T) { + t.Parallel() + + // A caller asking a MandiPrice question with the WRONG category. It is + // enum-legal, so nothing downstream would reject it -- which is exactly + // why echoing it was dangerous. + wrongCategory := strings.Replace(selectRequest, + `"subjectCategories": ["Market"]`, `"subjectCategories": ["Weather"]`, 1) + if wrongCategory == selectRequest { + t.Fatal("the fixture no longer states subjectCategories; this test needs updating") + } + + _, answer := runShippedWith(t, wrongCategory, providerResponse) + + for _, r := range resourcesOf(t, answer) { + ra := r["resourceAttributes"].(map[string]any) + + // Stated from the pack, not taken from the caller. + cats, _ := ra["subjectCategories"].([]any) + if len(cats) != 1 || cats[0] != "Market" { + t.Errorf(`subjectCategories = %#v, want ["Market"] regardless of what the request said`, ra["subjectCategories"]) + } + + market, _ := ra["market"].(map[string]any) + // The upstream reports no market code, so the answer must not claim one. + if code, present := market["marketCode"]; present { + t.Errorf("market.marketCode = %#v; this upstream reports no code, so asserting one is unfounded", code) + } + // And what is there comes from the record. + if market["marketName"] != "Kasdol APMC" || market["state"] != "Chattisgarh" { + t.Errorf("market = %#v, want the values the provider reported", market) + } + // marketName is the one member the pack requires. + if _, ok := market["marketName"]; !ok { + t.Error("market.marketName is missing, which the pack requires") + } + } +} From 3ffe2e88350428eeb82ae6f0748f32bf699229e0 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Tue, 8 Sep 2026 00:16:09 +0530 Subject: [PATCH 078/130] fix(mappings): distinguish colliding rows, and answer only what was asked [#8] Four findings in the mandi mapping. RESOURCE IDS COLLIDED. The id was scope:commodity:date, and Agmarknet reports several rows for the same market, commodity and date differing only by Variety and Grade -- this package's own fixture is exactly that pair, saved from collision only by having different arrival dates. Two distinct resources then shared one id and the offer referenced it twice, so a consumer resolving resourceIds could not tell which price it had. Market is in the id too: district-wide $scope is the district and the rows come from several markets inside it. Built with $join over a list, so an absent Variety or Grade drops out rather than leaving an empty segment. ONE COMMODITY, NOT THE FIRST OF SEVERAL. The guard, the outbound query and the commodity stamped on each resource all read supportedCommodities[0], so a caller sending three passed validation, was queried for one, and got a confident signed answer to a third of what it asked. Refused now, with the message saying to send them separately -- the same answer oanbinding gives at the commitment level. THE MARKET GUARD CHECKS WHAT IT PROMISED. Its message said "governed codes" while the check was $exists, and the pack describes district and state as "name or governed code" -- so the names form is pack-legal, passed, and went to Agmarknet verbatim. Agmarknet answered nothing and the caller received a signed, spec-valid "no prices" for a market that had prices. Now checks shape: a numeric district and a short alphabetic state, which is what this upstream takes. supportedPriceFields IS HONOURED. It was validated on the way in and ignored on the way out, so a caller asking for Modal alone got all three. Emitted conditionally now, and the conformance filter judges usability over the REQUESTED fields -- a row whose Modal is a marker cannot serve a request for Modal alone even though its Minimum is fine. Every construct was checked in the engine that actually runs it, github.com/jsonata-go/jsonata, after the $exists([]) surprise in a77b70f: $replace, $join over a list with gaps, `in`, a ternary with no else dropping its key, and $match discriminating 96 from Balodabazar and CG from Chattisgarh. Four tests, each confirmed to fail against the old form -- the id one reports both rows sharing res:agmarknet:2056:2:2025-08-20, and the price-fields one reports minimum and maximum arriving unasked. One existing expectation moved from "state and district" to "codes in market", which the new message carries. --- .../agmarknet/mandi-price.select.yaml | 76 ++++++-- .../implementation/mandi/mappings_test.go | 177 +++++++++++++++++- 2 files changed, 241 insertions(+), 12 deletions(-) diff --git a/config/mappings/agmarknet/mandi-price.select.yaml b/config/mappings/agmarknet/mandi-price.select.yaml index 7f9e4d4a..786259cc 100644 --- a/config/mappings/agmarknet/mandi-price.select.yaml +++ b/config/mappings/agmarknet/mandi-price.select.yaml @@ -28,15 +28,32 @@ required: - check: | ( $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; + /* Exactly one. Everything downstream reads supportedCommodities[0]: + this guard, the outbound query, and the commodity stamped on each + resource. So a caller sending three commodities passed validation, was + queried for the first, and got a confident signed answer to a third of + what it asked -- the same failure oanbinding refuses at the commitment + level, where guessing would silently serve part of a request. */ $exists($ra.supportedCommodities[0].code) + and $count($ra.supportedCommodities) = 1 ) - message: "this capability needs a commodity code in supportedCommodities[0].code" + message: "this capability needs exactly one commodity code in supportedCommodities; send several requests rather than have all but the first dropped" - check: | ( $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; - $exists($ra.market.state) and $exists($ra.market.district) + /* Shape, not just presence. The pack describes district and state as + "name or governed code", so the names form is pack-legal -- and the + existence-only check passed it, after which it went to Agmarknet + verbatim as codes. Agmarknet answered nothing, and the caller got a + signed, spec-valid "no prices" for a market that had prices. + + Agmarknet wants a numeric district and a short alphabetic state (96, + CG), so a district that is not all digits or a state longer than four + letters is a name and is refused with the reason. */ + $match($ra.market.district, /^[0-9]+$/) + and $match($ra.market.state, /^[A-Za-z]{2,4}$/) ) - message: "this capability needs governed state and district codes in market" + message: "this capability needs Agmarknet's codes in market: a numeric district and a short alphabetic state, not their names" - check: | ( $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; @@ -117,9 +134,31 @@ response: | }; $scope := $exists($ra.market.marketCode) ? $ra.market.marketCode : $ra.market.district; + /* SLUG for the parts that are names rather than codes, so an id has no + spaces and no case surprises. */ + $slug := function($v) { $exists($v) ? $replace($lowercase($v), " ", "-") }; + + /* Every field the upstream distinguishes rows by is in the id. + + It was scope:commodity:date only, and Agmarknet routinely reports + several rows for the same market, commodity and date differing by + Variety and Grade -- the fixture in this package is exactly that pair. + Two distinct resources then shared one id, and the offer referenced it + twice, so a consumer resolving resourceIds could not tell which price + it had. Market is in it too: district-wide $scope is the district, and + the rows come from several markets inside it. + + Built with $join over a list, because an absent Variety or Grade drops + out of an array literal rather than leaving an empty segment. */ $resourceId := function($r) { - "res:agmarknet:" & $scope & ":" & $ra.supportedCommodities[0].code - & ":" & $iso($r.`Arrival Date`) + "res:agmarknet:" & $join([ + $scope, + $ra.supportedCommodities[0].code, + $iso($r.`Arrival Date`), + $slug($r.Market), + $slug($r.Variety), + $slug($r.Grade) + ], ":") }; /* dd-MM-yyyy back to ISO, so the answer speaks the pack's date format @@ -138,6 +177,14 @@ response: | The number branch is not redundant. $match() throws T0410 on a non-string, so testing the regex first would break the day this upstream sends a real number instead of a quoted one. */ + /* The price fields the caller asked for. Absent means all three: the + pack requires supportedPriceFields on an OnDemand request, so absence + is a payload this mapping does not have to serve well -- but it should + not silently return nothing either. */ + $fields := $exists($ra.supportedPriceFields) + ? $ra.supportedPriceFields + : ["Minimum", "Maximum", "Modal"]; + $priced := function($value) { $type($value) = "number" ? $value @@ -178,9 +225,12 @@ response: | filter did nothing. Caught by the test, not by reading. */ $count($match($iso($r.`Arrival Date`), /^[0-9]{4}-[0-9]{2}-[0-9]{2}$/)) > 0 and $exists($r.`Price Unit`) - and ($exists($priced($r.`Min Price`)) - or $exists($priced($r.`Max Price`)) - or $exists($priced($r.`Modal Price`))) + /* Judged over the fields the caller asked for, not all three. A row + whose Modal is a marker cannot serve a request for Modal alone, + even though its Minimum is fine. */ + and (("Minimum" in $fields and $exists($priced($r.`Min Price`))) + or ("Maximum" in $fields and $exists($priced($r.`Max Price`))) + or ("Modal" in $fields and $exists($priced($r.`Modal Price`)))) }; $usable := $filter($records, $conformant); @@ -260,9 +310,13 @@ response: | }, "arrivalDate": $iso($r.`Arrival Date`), "prices": { - "minimum": $priced($r.`Min Price`), - "maximum": $priced($r.`Max Price`), - "modal": $priced($r.`Modal Price`), + /* Only what was asked for. supportedPriceFields was validated on the + way in and then ignored on the way out, so a caller asking for Modal + alone got all three. A ternary with no else yields nothing and the key + drops, which is the same mechanism an unreported price already uses. */ + "minimum": "Minimum" in $fields ? $priced($r.`Min Price`), + "maximum": "Maximum" in $fields ? $priced($r.`Max Price`), + "modal": "Modal" in $fields ? $priced($r.`Modal Price`), "currency": "INR", "unit": $r.`Price Unit` }, diff --git a/pkg/plugin/implementation/mandi/mappings_test.go b/pkg/plugin/implementation/mandi/mappings_test.go index 452b2075..6be53f46 100644 --- a/pkg/plugin/implementation/mandi/mappings_test.go +++ b/pkg/plugin/implementation/mandi/mappings_test.go @@ -386,7 +386,7 @@ func TestShippedMappingServesARealSelect(t *testing.T) { func TestShippedMappingRefusesWhatItCannotServe(t *testing.T) { for _, tc := range []struct{ name, drop, expect string }{ {"no commodity code", "supportedCommodities", "commodity code"}, - {"no market codes", "market", "state and district"}, + {"no market codes", "market", "codes in market"}, {"no validity window", "validity", "validity window"}, } { t.Run(tc.name, func(t *testing.T) { @@ -667,3 +667,178 @@ func TestShippedMappingStatesWhatItKnowsRatherThanEchoing(t *testing.T) { } } } + +// The guards now check what they claim to. Both of these payloads are legal +// against the pack and unanswerable by this upstream, which is the gap between +// "valid" and "serviceable" the required block exists to close. +func TestShippedMappingRefusesPayloadsItCannotAnswer(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(map[string]any) + expect string + }{ + { + // The pack describes district and state as "name or governed + // code", so this validates -- and went to Agmarknet verbatim as + // codes, which answered nothing. The caller then received a + // signed, spec-valid "no prices" for a market that had prices. + name: "names where the upstream wants codes", + mutate: func(ra map[string]any) { + ra["market"] = map[string]any{ + "marketName": "Kasdol APMC", + "district": "Balodabazar", + "state": "Chattisgarh", + } + }, + expect: "codes in market", + }, + { + // Everything downstream reads supportedCommodities[0], so this + // used to be queried for Paddy alone and answered confidently. + name: "more commodities than one request can serve", + mutate: func(ra map[string]any) { + ra["supportedCommodities"] = []any{ + map[string]any{"code": "2", "name": "Paddy(Common)"}, + map[string]any{"code": "3", "name": "Wheat"}, + } + }, + expect: "exactly one commodity", + }, + } { + t.Run(tc.name, func(t *testing.T) { + var payload map[string]any + if err := json.Unmarshal([]byte(selectRequest), &payload); err != nil { + t.Fatalf("the fixture is not JSON: %v", err) + } + ra := payload["message"].(map[string]any)["contract"].(map[string]any)["commitments"].([]any)[0].(map[string]any)["resources"].([]any)[0].(map[string]any)["resourceAttributes"].(map[string]any) + tc.mutate(ra) + body, err := json.Marshal(payload) + if err != nil { + t.Fatalf("could not rebuild the payload: %v", err) + } + + mappings := serveMappings(t) + defer mappings.Close() + + called := false + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + called = true + fmt.Fprint(w, providerResponse) + })) + defer upstream.Close() + + mapper, closeMapper, err := jsonmapper.New(context.Background(), &jsonmapper.Config{}) + if err != nil { + t.Fatalf("failed to build the mapper: %v", err) + } + defer closeMapper() + + registry := &stubRegistry{plan: &model.ProviderRecord{ + BindingKey: shippedBindingKey, BaseURL: upstream.URL, + Actions: map[string]model.ActionPlan{ + "select": {Method: http.MethodGet, Path: "/v1/fetch-agmarknet-vistaar", + Mappings: mappings.URL + "/" + shippedMapping, TimeoutMs: 30000}, + }, + }} + step, closeStep, err := mandi.New(context.Background(), registry, mapper, + &mandi.Config{BindingKeys: []string{shippedBindingKey}}) + if err != nil { + t.Fatalf("failed to build the step: %v", err) + } + defer closeStep() + + stepCtx := &model.StepContext{Context: t.Context(), Body: body} + if err := step.Run(stepCtx); err == nil { + t.Fatal("expected a payload this upstream cannot answer to be refused") + } else if !strings.Contains(err.Error(), tc.expect) { + t.Errorf("error %q should explain the refusal in terms of %q", err, tc.expect) + } + // The point of refusing early: the upstream is never troubled with + // a request that cannot produce an answer. + if called { + t.Error("the provider was called for a payload the mapping refuses") + } + }) + } +} + +// Agmarknet routinely reports several rows for the same market, commodity and +// date differing only by Variety and Grade. The id was built from +// scope:commodity:date, so those rows collided: two distinct resources under +// one id, referenced twice by the offer, and a consumer resolving resourceIds +// could not tell which price it had. +func TestShippedMappingGivesCollidingRowsDistinctIDs(t *testing.T) { + t.Parallel() + + // The same pair as this package's own fixture -- FAQ/Common and + // Non-FAQ/D.B. -- but sharing an arrival date, which is what the fixture + // was accidentally saved by not doing. + const sameDay = `[ + { + "Grade": "Non-FAQ", "Group": "Cereals", "State": "Chattisgarh", + "Market": "Kasdol APMC", "Variety": "D.B.", "District": "Balodabazar", + "Commodity": "Paddy(Common)", + "Min Price": "1900", "Max Price": "2100", "Modal Price": "2000", + "Price Unit": "Rs./Qtl", "Arrival Date": "20-08-2025" + }, + { + "Grade": "FAQ", "Group": "Cereals", "State": "Chattisgarh", + "Market": "Kasdol APMC", "Variety": "Common", "District": "Balodabazar", + "Commodity": "Paddy(Common)", + "Min Price": "1600", "Max Price": "1800", "Modal Price": "1700", + "Price Unit": "Rs./Qtl", "Arrival Date": "20-08-2025" + } + ]` + + _, answer := runShippedWith(t, selectRequest, sameDay) + + resources := resourcesOf(t, answer) + if len(resources) != 2 { + t.Fatalf("got %d resources, want both rows", len(resources)) + } + seen := map[string]int{} + for _, r := range resources { + seen[r["id"].(string)]++ + } + for id, n := range seen { + if n > 1 { + t.Errorf("id %q is shared by %d resources; the offer cannot reference one of them unambiguously", id, n) + } + } + if len(seen) != 2 { + t.Errorf("got %d distinct ids for 2 rows: %v", len(seen), seen) + } +} + +// supportedPriceFields was validated on the way in and ignored on the way out, +// so a caller asking for one field got all three. +func TestShippedMappingHonoursTheRequestedPriceFields(t *testing.T) { + t.Parallel() + + modalOnly := strings.Replace(selectRequest, + `"supportedPriceFields": ["Minimum", "Maximum", "Modal"]`, + `"supportedPriceFields": ["Modal"]`, 1) + if modalOnly == selectRequest { + t.Fatal("the fixture no longer lists all three price fields; this test needs updating") + } + + _, answer := runShippedWith(t, modalOnly, providerResponse) + + for _, r := range resourcesOf(t, answer) { + prices := r["resourceAttributes"].(map[string]any)["prices"].(map[string]any) + if _, ok := prices["modal"]; !ok { + t.Error("prices.modal is missing, and it is the field that was asked for") + } + for _, unasked := range []string{"minimum", "maximum"} { + if v, present := prices[unasked]; present { + t.Errorf("prices.%s = %#v was returned though the request did not ask for it", unasked, v) + } + } + // The pack requires these whatever was asked for. + for _, required := range []string{"currency", "unit"} { + if _, ok := prices[required]; !ok { + t.Errorf("prices.%s is missing, which the pack requires", required) + } + } + } +} From 9714b24e9007514519d5b2e55bf53b6570efd8f3 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Tue, 8 Sep 2026 00:21:00 +0530 Subject: [PATCH 079/130] docs: document MANDI_TOKEN, and correct two claims about the pack [#8] MANDI_TOKEN was documented nowhere. It appeared exactly once outside tests -- on the config line that names it -- with no README, no env example, no manifest. The failure mode is a plugin that loads cleanly, registers cleanly, passes startup validation, and then fails 100% of requests, because the step refuses to call an upstream unauthenticated and marks that as permanent. There cannot be a default: a credential is not something this file may hold. So the note sits at the point of use and says how to supply it under docker, kubernetes and a local run. This got MORE important since the review, not less. The error used to name the variable; 5bd4021 moved that name off the wire, because a network peer has no business learning which variables this deployment reads. The name is in the adapter's own log at error level now, so the config is the only place a deployer will find it in advance. TWO CLAIMS ABOUT THE PACK WERE WRONG, and both were ours rather than the pack's. The header said "NOTHING HERE IS OUTSIDE THE PACK". True of the response half -- every field it sets is declared and correctly placed, inside the closed market and prices sets -- and not true of the request half, which reads market and validity. Split, and the request side now says why: OnDemand describes what a provider can obtain, Direct describes an obtained reading, and a request is neither. The guard note said the pack "leaves market and validity optional". It excludes them. Corrected, with the consequence spelled out -- a payload that satisfies these guards cannot validate, and one that validates cannot satisfy them -- and with the reason nothing breaks today: the exclusion sits under if/then, which the validator parses and never evaluates. An accident to rely on rather than a design, now written down as such. Also notes which constants are the pack's and which are this provider's, since INR is stated while the unit beside it is read: Agmarknet reports a unit and no currency, and the pack requires currency, so it cannot be read or omitted. --- .../agmarknet/mandi-price.select.yaml | 50 ++++++++++++++++--- config/oan-provider-adapter.yaml | 19 +++++++ 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/config/mappings/agmarknet/mandi-price.select.yaml b/config/mappings/agmarknet/mandi-price.select.yaml index 786259cc..31dcbcc5 100644 --- a/config/mappings/agmarknet/mandi-price.select.yaml +++ b/config/mappings/agmarknet/mandi-price.select.yaml @@ -10,17 +10,42 @@ # mandi plugin has no prerequisites: a MandiPrice select names the market it # wants rather than a point to search from. # -# NOTHING HERE IS OUTSIDE THE PACK. openagrinet:MandiPrice v0.1 carries every -# field this answer sets. Where the upstream reports something the pack has no -# home for, it is dropped rather than invented. +# THE ANSWER IS WITHIN THE PACK; THE REQUEST IS NOT, AND CANNOT BE YET. +# +# Every field the response sets is declared by openagrinet:MandiPrice v0.1 and +# correctly placed, inside the closed market and prices property sets. Where +# the upstream reports something the pack has no home for, it is dropped rather +# than invented. +# +# The REQUEST half is a different matter, and an earlier version of this note +# claimed otherwise. It reads market and validity, and the pack's OnDemand +# branch EXCLUDES both -- see the note on the guards below. That is a gap in +# the pack rather than a mistake here: OnDemand describes what a provider can +# obtain and Direct describes an obtained reading, and a request is neither. # What this capability cannot serve, refused before the provider is called. # -# The pack requires none of these: a MandiPrice select is OnDemand, and that -# branch requires only supportedCommodities and supportedPriceFields. It leaves -# market and validity optional, and defines market.district and market.state as -# "name or governed code". So a payload can be perfectly valid and still be -# unanswerable by this upstream, which wants codes and a date range. +# THE PACK AND THESE GUARDS DISAGREE TODAY, and an earlier version of this note +# said the pack leaves market and validity OPTIONAL. It does not: the OnDemand +# branch EXCLUDES them, alongside source, commodity, commodityGroup, grade, +# variety, arrivalDate, prices and generatedAt. So a payload that satisfies +# these guards cannot validate against the pack, and one that validates cannot +# satisfy them. +# +# Nothing breaks in practice: the exclusion sits under if/then, which the +# validator parses and never evaluates. That is an accident to rely on rather +# than a design, and it is written down here so it is not mistaken for one. +# +# Resolving it is a pack change. For the market, coverageAreas is the likely +# home -- inherited from AgricultureResource, not excluded in OnDemand, and it +# accepts an AdministrativeAreaReference. For the date range there is no +# candidate at all: historyPeriod and updateFrequency are durations describing +# provider capability, not a window a caller asks for. +# +# What IS true, and is the reason this block exists: the pack defines +# market.district and market.state as "name or governed code", so a payload can +# be perfectly valid and still be unanswerable by this upstream, which wants +# codes and a date range. # # Refusing here names what is missing. Sending it anyway earns a 400 from # Agmarknet, or worse an empty result that reads as "no prices". @@ -317,6 +342,15 @@ response: | "minimum": "Minimum" in $fields ? $priced($r.`Min Price`), "maximum": "Maximum" in $fields ? $priced($r.`Max Price`), "modal": "Modal" in $fields ? $priced($r.`Modal Price`), + /* INR is a fact about THIS UPSTREAM, not about the pack, and it is + stated rather than read because Agmarknet reports no currency field + to read -- unlike the unit on the next line, which it does report. + The pack requires currency, so it cannot be omitted either. + + Worth being deliberate about the difference: on_select, DRAFT and the + @type values are PACK constants, correctly stated. INR and the + agmarknet sourceId are PROVIDER constants -- they change if this file + is pointed at another upstream, and the pack ones do not. */ "currency": "INR", "unit": $r.`Price Unit` }, diff --git a/config/oan-provider-adapter.yaml b/config/oan-provider-adapter.yaml index c35f041f..34730d21 100644 --- a/config/oan-provider-adapter.yaml +++ b/config/oan-provider-adapter.yaml @@ -293,6 +293,25 @@ modules: queryName: token queryValueEnv: MANDI_TOKEN + # MANDI_TOKEN MUST BE SET IN THE ADAPTER'S ENVIRONMENT. Without it + # every mandi call fails immediately and permanently: the step + # refuses to call an upstream unauthenticated, and marks the failure + # as one no retry can fix. + # + # There is no default and there cannot be one -- a credential is not + # something this file may hold. So the failure mode is a plugin that + # loads cleanly, registers cleanly, passes startup validation, and + # then fails 100% of requests. Worth knowing before deploying: + # + # docker: -e MANDI_TOKEN=... or an env_file + # kubernetes: a Secret, mounted as an environment variable + # local: MANDI_TOKEN=... ./server --config=... + # + # The error names the SCHEME rather than the variable, deliberately: + # this runs behind a signed network call and a peer has no business + # learning which variables this deployment reads. The variable name + # is in the adapter's own log at error level. + # Declaring a step above is not enough: THIS list is what runs. A step # that appears under providerSteps but not here never executes, and the # request falls through to the 404 above -- which looks like a registry From 6ae0b0d7f596aed1d52614324dd37e726b48090b Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Tue, 8 Sep 2026 13:34:29 +0530 Subject: [PATCH 080/130] test(mandi): cover the plugin entry point, which had no tests at all [#8] The rebase onto development brought a changed-file coverage gate, and this branch failed it: mandi/cmd/plugin.go was the only changed non-test Go file and sat at 0%, putting the diff at 3% against an 80% minimum. The gap is the one the review already identified. mandi/cmd is a near-verbatim copy of weather/cmd, but weather/cmd has five test functions and this had none -- so the response cap's bounds checking and the auth-scheme rejection were untested in this copy while passing in the other. Not a copy of weather's tests, because the two files are not identical where it counts: mandi carries queryName and queryValueEnv, and query auth is the whole reason this capability needs its own entry -- Agmarknet's Vistaar API takes its token as a query parameter. That case is covered explicitly. Beyond weather's set: splitList is tested directly rather than only through bindingKeys, an empty maxResponseBytes is asserted to read as unset rather than as malformed (a rendered config with an unset variable produces exactly that), and New's success path asserts the closer it returns actually reaches the step's own closer. parseConfig, New and splitList are now at 100%; the changed-line gate reports 100%. This does not deduplicate the two cmd packages, which is the standing review suggestion and is deliberately still open. --- .../implementation/mandi/cmd/plugin_test.go | 282 ++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100644 pkg/plugin/implementation/mandi/cmd/plugin_test.go diff --git a/pkg/plugin/implementation/mandi/cmd/plugin_test.go b/pkg/plugin/implementation/mandi/cmd/plugin_test.go new file mode 100644 index 00000000..262f73a6 --- /dev/null +++ b/pkg/plugin/implementation/mandi/cmd/plugin_test.go @@ -0,0 +1,282 @@ +package main + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/mandi" +) + +type stubRegistry struct{} + +func (stubRegistry) ProviderRecord(context.Context, string) (*model.ProviderRecord, error) { + return nil, nil +} + +type stubMapper struct{} + +func (stubMapper) Verify(context.Context, string, any) error { return nil } + +func (stubMapper) Transform(context.Context, string, definition.Direction, any) ([]byte, error) { + return nil, nil +} + +func TestParseConfig(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + config map[string]string + expected *mandi.Config + expectedErr string + }{ + { + // Everything absent is left zero: mandi.New defaults it, so the + // rules are defined in exactly one place. + name: "leaves everything unset for New to default", + config: map[string]string{}, + expected: &mandi.Config{}, + }, + { + // Query auth is why this capability has its own entry rather than + // sharing weather's: Agmarknet's Vistaar API takes its token as a + // QUERY parameter, and these two keys are the only place that is + // expressible. + name: "reads the query auth scheme this capability needs", + config: map[string]string{ + "bindingKeys": "agmarknet|openagrinet:MandiPrice", + "authScheme": "query", + "queryName": "api-key", + "queryValueEnv": "MANDI_TOKEN", + }, + expected: &mandi.Config{ + BindingKeys: []string{"agmarknet|openagrinet:MandiPrice"}, + AuthScheme: "query", + QueryName: "api-key", + QueryValueEnv: "MANDI_TOKEN", + }, + }, + { + name: "reads every supported setting", + config: map[string]string{ + "bindingKeys": "other|capability", + "authScheme": "basic", + "usernameEnv": "U", + "passwordEnv": "P", + "headerName": "X-Key", + "headerValueEnv": "V", + "queryName": "q", + "queryValueEnv": "Q", + "maxResponseBytes": "2048", + }, + expected: &mandi.Config{ + BindingKeys: []string{"other|capability"}, + AuthScheme: "basic", + UsernameEnv: "U", + PasswordEnv: "P", + HeaderName: "X-Key", + HeaderValueEnv: "V", + QueryName: "q", + QueryValueEnv: "Q", + MaxResponseBytes: 2048, + }, + }, + { + name: "rejects a malformed response cap", + config: map[string]string{"maxResponseBytes": "lots"}, + expectedErr: "invalid maxResponseBytes value 'lots'", + }, + { + name: "rejects a non-positive response cap", + config: map[string]string{"maxResponseBytes": "0"}, + expectedErr: "maxResponseBytes must be positive", + }, + { + // Present but empty is not the same as malformed. A rendered + // config with an unset variable produces this, and it should read + // as "unset" rather than failing startup. + name: "treats an empty response cap as unset", + config: map[string]string{"maxResponseBytes": ""}, + expected: &mandi.Config{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got, err := mandiProvider{}.parseConfig(tc.config) + + if tc.expectedErr != "" { + if err == nil { + t.Fatalf("expected error %q but got none", tc.expectedErr) + } + if !strings.Contains(err.Error(), tc.expectedErr) { + t.Errorf("expected error containing %q, got %q", tc.expectedErr, err.Error()) + } + return + } + + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !reflect.DeepEqual(got, tc.expected) { + t.Errorf("expected config %+v, got %+v", tc.expected, got) + } + }) + } +} + +// A plugin config is map[string]string, so a list arrives comma-separated -- +// the convention reqpreprocessor and schemav2validator already use. Binding keys +// separate their own halves with a pipe, so a comma is unambiguous. +func TestSplitList(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + raw string + want []string + }{ + {name: "empty is nil, not a one-element list of nothing", raw: "", want: nil}, + {name: "whitespace only is nil", raw: " ", want: nil}, + {name: "one value", raw: "a|openagrinet:One", want: []string{"a|openagrinet:One"}}, + { + name: "several, with blanks and spacing a wrapped config line produces", + raw: "a|openagrinet:One, b|openagrinet:Two ,, ", + want: []string{"a|openagrinet:One", "b|openagrinet:Two"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + if got := splitList(tc.raw); !reflect.DeepEqual(got, tc.want) { + t.Errorf("splitList(%q) = %v, want %v", tc.raw, got, tc.want) + } + }) + } +} + +// The override is two keys, both or neither. Absent leaves the step on the +// Beckn v2 convention, which is what every deployment should be running. +func TestParseConfigReadsTheBindingKeyOverride(t *testing.T) { + t.Parallel() + + cfg, err := mandiProvider{}.parseConfig(map[string]string{ + "bindingKeys": "a|openagrinet:One", + "providerIdAt": "who.provider", + "capabilityCodeAt": "what[].type", + }) + if err != nil { + t.Fatalf("parseConfig() returned an unexpected error: %v", err) + } + if cfg.ProviderIDAt != "who.provider" || cfg.CapabilityCodeAt != "what[].type" { + t.Errorf("override = %q / %q, want the configured paths", cfg.ProviderIDAt, cfg.CapabilityCodeAt) + } +} + +// Absent leaves them empty, and upstream reads that as "use the convention". +func TestParseConfigLeavesTheOverrideUnsetByDefault(t *testing.T) { + t.Parallel() + + cfg, err := mandiProvider{}.parseConfig(map[string]string{"bindingKeys": "a|openagrinet:One"}) + if err != nil { + t.Fatalf("parseConfig() returned an unexpected error: %v", err) + } + if cfg.ProviderIDAt != "" || cfg.CapabilityCodeAt != "" { + t.Errorf("override = %q / %q, want both empty", cfg.ProviderIDAt, cfg.CapabilityCodeAt) + } +} + +func TestNew(t *testing.T) { + t.Parallel() + + t.Run("rejects a nil context", func(t *testing.T) { + t.Parallel() + + //nolint:staticcheck // deliberately passing a nil context to assert the guard. + _, _, err := mandiProvider{}.New(nil, stubRegistry{}, stubMapper{}, map[string]string{}) + if err == nil { + t.Fatal("expected an error for a nil context, got none") + } + }) + + t.Run("rejects an unparseable config", func(t *testing.T) { + t.Parallel() + + _, _, err := mandiProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, + map[string]string{"maxResponseBytes": "lots"}) + if err == nil { + t.Fatal("expected an error for an invalid cap, got none") + } + }) + + t.Run("propagates an invalid auth scheme from New", func(t *testing.T) { + t.Parallel() + + _, _, err := mandiProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, + map[string]string{"authScheme": "oauth"}) + if err == nil { + t.Fatal("expected an unknown auth scheme to be refused") + } + }) + + // A domain plugin serves a family of capabilities, so it cannot guess which + // of them a deployment has providers for. Refused at startup rather than + // answering to nothing. + t.Run("refuses a config naming no capability", func(t *testing.T) { + t.Parallel() + + if _, _, err := (mandiProvider{}).New(context.Background(), stubRegistry{}, stubMapper{}, + map[string]string{}); err == nil { + t.Fatal("expected a config with no bindingKeys to be refused") + } + }) + + t.Run("returns the step and its closer on a config that loads", func(t *testing.T) { + // Not parallel: it swaps the package-level newStepFunc. + closed := false + original := newStepFunc + newStepFunc = func(context.Context, definition.ProviderRecordLookup, definition.Mapper, + *mandi.Config) (definition.Step, func() error, error) { + return nil, func() error { closed = true; return nil }, nil + } + defer func() { newStepFunc = original }() + + _, closer, err := mandiProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, + map[string]string{"bindingKeys": "agmarknet|openagrinet:MandiPrice"}) + if err != nil { + t.Fatalf("New() returned an unexpected error: %v", err) + } + if closer == nil { + t.Fatal("New() returned no closer, so nothing can release the step") + } + if err := closer(); err != nil { + t.Errorf("closer() returned %v, want nil", err) + } + if !closed { + t.Error("closer() did not reach the step's own closer") + } + }) + + t.Run("propagates a failure from the step constructor", func(t *testing.T) { + original := newStepFunc + wanted := errors.New("upstream refused the config") + newStepFunc = func(context.Context, definition.ProviderRecordLookup, definition.Mapper, + *mandi.Config) (definition.Step, func() error, error) { + return nil, nil, wanted + } + defer func() { newStepFunc = original }() + + _, _, err := mandiProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, + map[string]string{"bindingKeys": "agmarknet|openagrinet:MandiPrice"}) + if !errors.Is(err, wanted) { + t.Errorf("New() error = %v, want it to wrap %v", err, wanted) + } + }) +} From 75f87f054ae373e6365dde4dcc4f4d64dc3629e8 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Tue, 8 Sep 2026 15:18:00 +0530 Subject: [PATCH 081/130] docs(mappings): put each comment above the code it describes [#8] Three blocks had drifted from what they document, and the file's own history is the reason it matters: a resource-id collision had to be fixed here once already, so a future edit to $resourceId reading the comment above it would have found a date-conversion rationale instead. $resourceId's two blocks -- "bound once because it is used twice" and "built from CODES, not the names the upstream reports" -- sat above $iso, which converts dd-MM-yyyy to ISO and has nothing to do with either. They now join the comment $resourceId already had. $iso's own comment was orphaned further down with no code beneath it at all. It is back above $iso. The market-code paragraph moved to $scope rather than travelling with the rest: "without it the query widened to the whole district, so the district code is what identifies the scope" describes the $scope binding specifically, not how the id is composed. No expression changed -- the shipped-mapping tests serve this exact file over HTTP and still pass. --- .../agmarknet/mandi-price.select.yaml | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/config/mappings/agmarknet/mandi-price.select.yaml b/config/mappings/agmarknet/mandi-price.select.yaml index 31dcbcc5..d10a0ce8 100644 --- a/config/mappings/agmarknet/mandi-price.select.yaml +++ b/config/mappings/agmarknet/mandi-price.select.yaml @@ -143,27 +143,30 @@ response: | $ctx := beckn.message.contract.commitments[0].resources[0].resourceAttributes.`@context`; $ra := $selected.resources[0].resourceAttributes; - /* Bound once because it is used twice -- for a resource's own id and for - the offer's reference to it. Two copies of one expression is how a - dangling reference gets reintroduced. */ - /* Built from CODES, not the names the upstream reports. A market name - carries spaces and a commodity name carries brackets -- "Kasdol APMC", - "Paddy(Common)" -- and an identifier that a consumer may put in a URL or - a filter should not. The codes are already in the payload, so they cost - nothing, and they are stable where a display name is not. - - The market code is optional: without it the query widened to the whole - district, so the district code is what identifies the scope. */ + /* dd-MM-yyyy back to ISO, so the answer speaks the pack's date format + rather than the upstream's. */ $iso := function($ddmmyyyy) { $substring($ddmmyyyy, 6, 4) & "-" & $substring($ddmmyyyy, 3, 2) & "-" & $substring($ddmmyyyy, 0, 2) }; + /* The market code is optional: without it the query widened to the whole + district, so the district code is what identifies the scope. */ $scope := $exists($ra.market.marketCode) ? $ra.market.marketCode : $ra.market.district; /* SLUG for the parts that are names rather than codes, so an id has no spaces and no case surprises. */ $slug := function($v) { $exists($v) ? $replace($lowercase($v), " ", "-") }; - /* Every field the upstream distinguishes rows by is in the id. + /* Bound once because it is used twice -- for a resource's own id and for + the offer's reference to it. Two copies of one expression is how a + dangling reference gets reintroduced. + + Built from CODES, not the names the upstream reports. A market name + carries spaces and a commodity name carries brackets -- "Kasdol APMC", + "Paddy(Common)" -- and an identifier that a consumer may put in a URL or + a filter should not. The codes are already in the payload, so they cost + nothing, and they are stable where a display name is not. + + Every field the upstream distinguishes rows by is in the id. It was scope:commodity:date only, and Agmarknet routinely reports several rows for the same market, commodity and date differing by @@ -186,9 +189,6 @@ response: | ], ":") }; - /* dd-MM-yyyy back to ISO, so the answer speaks the pack's date format - rather than the upstream's. */ - /* Absent rather than present-and-empty: a consumer must be able to tell "the market reported no minimum" from "the minimum was zero". From a4337f30920ee645cb9790f99d0591e1b7eb9820 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 13:22:43 +0530 Subject: [PATCH 082/130] fix(schemav2validator): keep the JSON-LD keys a schema declares [#16] Extended validation removed @context and @type from every object before validating it. That suited a schema which closes itself with additionalProperties:false and never mentions either key, and it made a schema pack that DECLARES @type and lists it in required impossible to satisfy: the payload carried @type, the schema required it, and the validator had just taken it out, so a conforming payload was rejected for a missing field it had supplied. Decide per key, by asking the schema whether it declares that key -- as a property or in required, anywhere in its composition tree, since the OAN packs declare @type one level down in allOf. Both schema styles then validate without a config switch and without either having to know about the other. Keeping @type also means its const is now checked, so a resource whose @type is not the one the capability declares no longer passes silently. [#16] Please enter the commit message for your changes. Lines starting [#16] with '#' will be ignored, and an empty message aborts the commit. [#16] [#16] Date: Mon Sep 7 13:22:43 2026 +0530 [#16] [#16] interactive rebase in progress; onto 983affb [#16] Last command done (1 command done): [#16] reword e19177c fix(schemav2validator): keep the JSON-LD keys a schema declares [#16] [#16] Next commands to do (5 remaining commands): [#16] reword 247022c refactor(plugins): name the capability plugins after their capabilities [#16] [#16] reword ec87958 feat(config): validate resource attributes against their schema packs [#16] [#16] You are currently editing a commit while rebasing branch 'tmp/rebase-16' on '983affb'. [#16] [#16] Changes to be committed: [#16] modified: pkg/plugin/implementation/schemav2validator/extended_schema.go [#16] modified: pkg/plugin/implementation/schemav2validator/extended_schema_test.go [#16] --- .../schemav2validator/extended_schema.go | 73 ++++++- .../schemav2validator/extended_schema_test.go | 193 ++++++++++++++++++ 2 files changed, 259 insertions(+), 7 deletions(-) diff --git a/pkg/plugin/implementation/schemav2validator/extended_schema.go b/pkg/plugin/implementation/schemav2validator/extended_schema.go index a002b019..654ba2d7 100644 --- a/pkg/plugin/implementation/schemav2validator/extended_schema.go +++ b/pkg/plugin/implementation/schemav2validator/extended_schema.go @@ -9,6 +9,7 @@ import ( "net/url" "os" "path/filepath" + "slices" "strings" "sync" "time" @@ -498,6 +499,69 @@ func isAllowedDomain(u *url.URL, allowedDomains []string) bool { return false } +// jsonLDKeys are the JSON-LD control keys that travel inside a domain object +// rather than beside it. +var jsonLDKeys = []string{"@context", "@type"} + +// stripUnaccountedJSONLDKeys removes those JSON-LD keys the target schema does +// not declare, and keeps the ones it does. +// +// The two schema styles in use need opposite treatment, and removing both keys +// unconditionally only served the first: +// +// - a schema that closes itself with additionalProperties:false and never +// mentions @type rejects the payload if @type is left in; +// - a schema pack that declares @type and lists it in required rejects the +// payload if @type is taken out. +// +// Asking the schema, per key, satisfies both without a config switch and +// without either style having to know about the other. +func stripUnaccountedJSONLDKeys(schema *openapi3.SchemaRef, data map[string]interface{}) map[string]interface{} { + domainData := make(map[string]interface{}, len(data)) + for k, v := range data { + if slices.Contains(jsonLDKeys, k) && !schemaDeclaresProperty(schema, k, map[*openapi3.Schema]bool{}) { + continue + } + domainData[k] = v + } + return domainData +} + +// schemaDeclaresProperty reports whether name is declared as a property, or +// listed as required, anywhere in a schema's composition tree. +// +// allOf, anyOf, oneOf and the then/else branches can each introduce a property, +// so all of them are walked -- the OAN packs declare @type one level down, in +// allOf. "not" is skipped because naming a property there forbids it rather +// than permitting it, and "if" is skipped because it only selects a branch. +// seen guards against schemas that reference themselves. +func schemaDeclaresProperty(ref *openapi3.SchemaRef, name string, seen map[*openapi3.Schema]bool) bool { + if ref == nil || ref.Value == nil || seen[ref.Value] { + return false + } + seen[ref.Value] = true + + if _, ok := ref.Value.Properties[name]; ok { + return true + } + if slices.Contains(ref.Value.Required, name) { + return true + } + for _, group := range []openapi3.SchemaRefs{ref.Value.AllOf, ref.Value.AnyOf, ref.Value.OneOf} { + for _, sub := range group { + if schemaDeclaresProperty(sub, name, seen) { + return true + } + } + } + for _, sub := range []*openapi3.SchemaRef{ref.Value.Then, ref.Value.Else} { + if schemaDeclaresProperty(sub, name, seen) { + return true + } + } + return false +} + // validateReferencedObject validates a single object with @context. func (c *schemaCache) validateReferencedObject( ctx context.Context, @@ -552,13 +616,8 @@ func (c *schemaCache) validateReferencedObject( return model.NewCodedErrorWithCause("SCH_INVALID_ENTITY_TYPE", err.Error(), obj.Path, err) } - // Strip JSON-LD metadata before validation - domainData := make(map[string]interface{}, len(obj.Data)-2) - for k, v := range obj.Data { - if k != "@context" && k != "@type" { - domainData[k] = v - } - } + // Strip only the JSON-LD keys this schema does not account for itself. + domainData := stripUnaccountedJSONLDKeys(schema, obj.Data) // Validate domain-specific data against schema opts := []openapi3.SchemaValidationOption{ diff --git a/pkg/plugin/implementation/schemav2validator/extended_schema_test.go b/pkg/plugin/implementation/schemav2validator/extended_schema_test.go index 6adc99be..e38325b1 100644 --- a/pkg/plugin/implementation/schemav2validator/extended_schema_test.go +++ b/pkg/plugin/implementation/schemav2validator/extended_schema_test.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "reflect" + "sort" "sync/atomic" "testing" "time" @@ -1377,3 +1378,195 @@ func TestLoadSchemaFromPath_TTLExpiry_FetchesFresh(t *testing.T) { assert.NoError(t, err) assert.Equal(t, "Schema v2", doc2.Info.Title, "expected v2 after TTL expiry — global URIMapCache not bypassed") } + +// packStyleSchema mirrors how the OAN schema packs are shaped: the capability +// declares @type one level down in allOf and lists it as required, and nothing +// closes the object with additionalProperties:false. +const packStyleSchema = `openapi: 3.1.0 +info: + title: Pack Style + version: 1.0.0 +components: + schemas: + WeatherObservation: + type: object + x-jsonld: + "@context": https://schemas.example.org/schema/WeatherObservation/v0.1/context.jsonld + "@type": openagrinet:WeatherObservation + allOf: + - type: object + required: + - informationMode + properties: + informationMode: + type: string + enum: [OnDemand, Direct] + - type: object + required: + - "@type" + properties: + "@type": + type: string + const: openagrinet:WeatherObservation` + +func writeTempSchema(t *testing.T, content string) string { + t.Helper() + f, err := os.CreateTemp("", "test-schema-*.yaml") + assert.NoError(t, err) + t.Cleanup(func() { os.Remove(f.Name()) }) + _, err = f.Write([]byte(content)) + assert.NoError(t, err) + assert.NoError(t, f.Close()) + return f.Name() +} + +// A pack that requires @type must receive it. This is the case that could not +// validate while both JSON-LD keys were removed unconditionally: the payload +// carries @type, the schema requires it, and stripping it produced a spurious +// "@type is required". +func TestValidateReferencedObject_PackStyleKeepsAtType(t *testing.T) { + cache := newSchemaCache(10) + path := writeTempSchema(t, packStyleSchema) + + obj := referencedObject{ + Path: "message.catalogs[0].resources[0].resourceAttributes", + Context: path, + Type: "openagrinet:WeatherObservation", + Data: map[string]interface{}{ + "@context": "https://schemas.example.org/schema/WeatherObservation/v0.1/context.jsonld", + "@type": "openagrinet:WeatherObservation", + "informationMode": "OnDemand", + }, + } + + err := cache.validateReferencedObject(context.Background(), obj, 1*time.Hour, 30*time.Second, nil, false) + assert.NoError(t, err) +} + +// The same pack must still reject a payload whose @type is not the one the +// capability declares -- keeping the key means its const is now checked, which +// stripping it silently skipped. +func TestValidateReferencedObject_PackStyleWrongAtTypeRejected(t *testing.T) { + cache := newSchemaCache(10) + path := writeTempSchema(t, packStyleSchema) + + obj := referencedObject{ + Path: "message.catalogs[0].resources[0].resourceAttributes", + Context: path, + Type: "openagrinet:WeatherObservation", + Data: map[string]interface{}{ + "@type": "openagrinet:MandiPrice", + "informationMode": "OnDemand", + }, + } + + err := cache.validateReferencedObject(context.Background(), obj, 1*time.Hour, 30*time.Second, nil, false) + assert.Error(t, err) +} + +func TestStripUnaccountedJSONLDKeys(t *testing.T) { + declaresType := &openapi3.SchemaRef{Value: &openapi3.Schema{ + AllOf: openapi3.SchemaRefs{ + {Value: &openapi3.Schema{ + Required: []string{"@type"}, + Properties: openapi3.Schemas{"@type": {Value: &openapi3.Schema{}}}, + }}, + }, + }} + declaresNeither := &openapi3.SchemaRef{Value: &openapi3.Schema{ + Properties: openapi3.Schemas{"field1": {Value: &openapi3.Schema{}}}, + }} + declaresBoth := &openapi3.SchemaRef{Value: &openapi3.Schema{ + Properties: openapi3.Schemas{ + "@context": {Value: &openapi3.Schema{}}, + "@type": {Value: &openapi3.Schema{}}, + }, + }} + + data := map[string]interface{}{ + "@context": "https://example.com/context.jsonld", + "@type": "openagrinet:WeatherObservation", + "field1": "value1", + } + + tests := []struct { + name string + schema *openapi3.SchemaRef + want []string + }{ + {"pack declares @type, so only @context goes", declaresType, []string{"@type", "field1"}}, + {"schema declares neither, so both go", declaresNeither, []string{"field1"}}, + {"schema declares both, so neither goes", declaresBoth, []string{"@context", "@type", "field1"}}, + {"nil schema is treated as declaring nothing", nil, []string{"field1"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := stripUnaccountedJSONLDKeys(tt.schema, data) + keys := make([]string, 0, len(got)) + for k := range got { + keys = append(keys, k) + } + sort.Strings(keys) + assert.Equal(t, tt.want, keys) + // the input must not be mutated -- obj.Data is shared with the caller + assert.Len(t, data, 3) + }) + } +} + +func TestSchemaDeclaresProperty(t *testing.T) { + leaf := func(required ...string) *openapi3.SchemaRef { + return &openapi3.SchemaRef{Value: &openapi3.Schema{Required: required}} + } + + cyclic := &openapi3.SchemaRef{Value: &openapi3.Schema{}} + cyclic.Value.AllOf = openapi3.SchemaRefs{cyclic} + + tests := []struct { + name string + schema *openapi3.SchemaRef + want bool + }{ + {"nil ref", nil, false}, + {"nil value", &openapi3.SchemaRef{}, false}, + {"declared directly as a property", &openapi3.SchemaRef{Value: &openapi3.Schema{ + Properties: openapi3.Schemas{"@type": {Value: &openapi3.Schema{}}}, + }}, true}, + {"required directly", leaf("@type"), true}, + {"required inside allOf", &openapi3.SchemaRef{Value: &openapi3.Schema{ + AllOf: openapi3.SchemaRefs{leaf("other"), leaf("@type")}, + }}, true}, + {"required inside anyOf", &openapi3.SchemaRef{Value: &openapi3.Schema{ + AnyOf: openapi3.SchemaRefs{leaf("@type")}, + }}, true}, + {"required inside oneOf", &openapi3.SchemaRef{Value: &openapi3.Schema{ + OneOf: openapi3.SchemaRefs{leaf("@type")}, + }}, true}, + {"required inside then", &openapi3.SchemaRef{Value: &openapi3.Schema{ + Then: leaf("@type"), + }}, true}, + {"required inside else", &openapi3.SchemaRef{Value: &openapi3.Schema{ + Else: leaf("@type"), + }}, true}, + // naming a property under "not" forbids it, so it must not count as declared + {"named under not does not count", &openapi3.SchemaRef{Value: &openapi3.Schema{ + Not: leaf("@type"), + }}, false}, + // "if" only selects a branch; it does not permit the property + {"named under if does not count", &openapi3.SchemaRef{Value: &openapi3.Schema{ + If: leaf("@type"), + }}, false}, + {"absent everywhere", &openapi3.SchemaRef{Value: &openapi3.Schema{ + AllOf: openapi3.SchemaRefs{leaf("informationMode")}, + }}, false}, + {"self-referencing schema terminates", cyclic, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := schemaDeclaresProperty(tt.schema, "@type", map[*openapi3.Schema]bool{}) + assert.Equal(t, tt.want, got) + }) + } +} From e524cf72194354bdf31a73cbd3e7d8a7f8465701 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 13:23:03 +0530 Subject: [PATCH 083/130] refactor(plugins): name the capability plugins after their capabilities [#16] weather and mandi named a domain; the payloads they serve name a capability. Renaming the packages to WeatherObservation and MandiPrice puts the two in the same vocabulary, so a binding key, a schema pack and the plugin that answers for it all read the same. The .so basename is the plugin id the adapter config refers to, so the directory rename carries the ids with it -- providerSteps and steps in config/oan-provider-adapter.yaml move together with the packages, and build-plugins.sh with them. No behaviour changes: the whole suite passes, and both plugins still build as loadable shared objects. Both package docs claimed one package per schema pack FAMILY, which the new names contradict. They now say what is true -- one package per capability, named for the capability, with the binding keys it answers to still configuration. [#16] Please enter the commit message for your changes. Lines starting [#16] with '#' will be ignored, and an empty message aborts the commit. [#16] [#16] Date: Mon Sep 7 13:23:03 2026 +0530 [#16] [#16] interactive rebase in progress; onto 983affb [#16] Last commands done (2 commands done): [#16] reword e19177c fix(schemav2validator): keep the JSON-LD keys a schema declares [#16] [#16] reword 247022c refactor(plugins): name the capability plugins after their capabilities [#16] [#16] Next commands to do (4 remaining commands): [#16] reword ec87958 feat(config): validate resource attributes against their schema packs [#16] [#16] reword bc8979b refactor(config): resolve capability schemas from the payload's @context [#16] [#16] You are currently editing a commit while rebasing branch 'tmp/rebase-16' on '983affb'. [#16] [#16] Changes to be committed: [#16] modified: config/oan-provider-adapter.yaml [#16] modified: install/build-plugins.sh [#16] renamed: pkg/plugin/implementation/mandi/mandi.go -> pkg/plugin/implementation/MandiPrice/MandiPrice.go [#16] renamed: pkg/plugin/implementation/mandi/cmd/plugin.go -> pkg/plugin/implementation/MandiPrice/cmd/plugin.go [#16] renamed: pkg/plugin/implementation/mandi/mappings_test.go -> pkg/plugin/implementation/MandiPrice/mappings_test.go [#16] renamed: pkg/plugin/implementation/mandi/prerequisites.go -> pkg/plugin/implementation/MandiPrice/prerequisites.go [#16] renamed: pkg/plugin/implementation/weather/weather.go -> pkg/plugin/implementation/WeatherObservation/WeatherObservation.go [#16] renamed: pkg/plugin/implementation/weather/cmd/plugin.go -> pkg/plugin/implementation/WeatherObservation/cmd/plugin.go [#16] renamed: pkg/plugin/implementation/weather/cmd/plugin_test.go -> pkg/plugin/implementation/WeatherObservation/cmd/plugin_test.go [#16] renamed: pkg/plugin/implementation/weather/mappings_test.go -> pkg/plugin/implementation/WeatherObservation/mappings_test.go [#16] renamed: pkg/plugin/implementation/weather/prerequisites.go -> pkg/plugin/implementation/WeatherObservation/prerequisites.go [#16] --- config/oan-provider-adapter.yaml | 12 ++++++------ install/build-plugins.sh | 4 ++-- .../mandi.go => MandiPrice/MandiPrice.go} | 10 +++++----- .../{mandi => MandiPrice}/cmd/plugin.go | 10 +++++----- .../{mandi => MandiPrice}/cmd/plugin_test.go | 18 +++++++++--------- .../{mandi => MandiPrice}/mappings_test.go | 16 ++++++++-------- .../{mandi => MandiPrice}/prerequisites.go | 2 +- .../WeatherObservation.go} | 13 ++++++++----- .../cmd/plugin.go | 10 +++++----- .../cmd/plugin_test.go | 12 ++++++------ .../mappings_test.go | 8 ++++---- .../prerequisites.go | 2 +- 12 files changed, 60 insertions(+), 57 deletions(-) rename pkg/plugin/implementation/{mandi/mandi.go => MandiPrice/MandiPrice.go} (81%) rename pkg/plugin/implementation/{mandi => MandiPrice}/cmd/plugin.go (91%) rename pkg/plugin/implementation/{mandi => MandiPrice}/cmd/plugin_test.go (95%) rename pkg/plugin/implementation/{mandi => MandiPrice}/mappings_test.go (98%) rename pkg/plugin/implementation/{mandi => MandiPrice}/prerequisites.go (98%) rename pkg/plugin/implementation/{weather/weather.go => WeatherObservation/WeatherObservation.go} (68%) rename pkg/plugin/implementation/{weather => WeatherObservation}/cmd/plugin.go (91%) rename pkg/plugin/implementation/{weather => WeatherObservation}/cmd/plugin_test.go (94%) rename pkg/plugin/implementation/{weather => WeatherObservation}/mappings_test.go (98%) rename pkg/plugin/implementation/{weather => WeatherObservation}/prerequisites.go (97%) diff --git a/config/oan-provider-adapter.yaml b/config/oan-provider-adapter.yaml index 34730d21..87bcd08c 100644 --- a/config/oan-provider-adapter.yaml +++ b/config/oan-provider-adapter.yaml @@ -228,7 +228,7 @@ modules: # rather than calling the provider unauthenticated. # ------------------------------------------------------------------ providerSteps: - - id: weather + - id: WeatherObservation config: # REQUIRED. Comma-separated, because a plugin config value is a # string and one provider may serve several capabilities. A @@ -278,9 +278,9 @@ modules: # A second capability in the same pipeline, from a different domain # package. Nothing about it is weather's business: a different # upstream, a different mapping, a different set of prerequisites -- - # and the same two registry rows. This entry, plus "mandi" in steps - # below, is the entire cost of adding it. - - id: mandi + # and the same two registry rows. This entry, plus "MandiPrice" + # in steps below, is the entire cost of adding it. + - id: MandiPrice config: bindingKeys: "agmarknet|openagrinet:MandiPrice" @@ -319,8 +319,8 @@ modules: steps: - validateSign # the sender's key, from the registry - validateSchema # the pinned Beckn v2 spec - - weather # openagrinet:WeatherObservation, or pass through - - mandi # openagrinet:MandiPrice, or pass through + - WeatherObservation # its binding key, or pass through + - MandiPrice # its binding key, or pass through - signAck # signs whatever the step answered with # ---------------------------------------------------------------------------- diff --git a/install/build-plugins.sh b/install/build-plugins.sh index 8ac0d713..141cb3b0 100755 --- a/install/build-plugins.sh +++ b/install/build-plugins.sh @@ -32,8 +32,8 @@ plugins=( "dediregistry" "oanregistry" "jsonmapper" - "weather" - "mandi" + "WeatherObservation" + "MandiPrice" "manifestloader" "reqpreprocessor" "otelsetup" diff --git a/pkg/plugin/implementation/mandi/mandi.go b/pkg/plugin/implementation/MandiPrice/MandiPrice.go similarity index 81% rename from pkg/plugin/implementation/mandi/mandi.go rename to pkg/plugin/implementation/MandiPrice/MandiPrice.go index 26e8b94b..38fff611 100644 --- a/pkg/plugin/implementation/mandi/mandi.go +++ b/pkg/plugin/implementation/MandiPrice/MandiPrice.go @@ -1,8 +1,8 @@ -// Package mandi serves the network's market price capabilities. +// Package MandiPrice serves the network's market price capabilities. // -// One package per schema pack family, so which plugin owns a capability is -// readable from its binding key: openagrinet:MandiPrice is mandi's, -// openagrinet:WeatherObservation is weather's. +// One package per capability, named for the capability it serves, so which +// plugin owns a payload is readable from its binding key without a lookup: +// openagrinet:MandiPrice is this one's, openagrinet:WeatherObservation is not. // // Almost nothing lives here, and that is the point. Recognising a capability, // resolving the call plan, authenticating, calling with the registry's budget @@ -15,7 +15,7 @@ // select takes governed codes for state, district, market and commodity plus a // date range, all of which a MandiPrice payload carries. So the package is a // name and nothing else: see prerequisites.go for why that is worth stating. -package mandi +package MandiPrice import ( "context" diff --git a/pkg/plugin/implementation/mandi/cmd/plugin.go b/pkg/plugin/implementation/MandiPrice/cmd/plugin.go similarity index 91% rename from pkg/plugin/implementation/mandi/cmd/plugin.go rename to pkg/plugin/implementation/MandiPrice/cmd/plugin.go index 5b4fbb70..8dcce23f 100644 --- a/pkg/plugin/implementation/mandi/cmd/plugin.go +++ b/pkg/plugin/implementation/MandiPrice/cmd/plugin.go @@ -13,20 +13,20 @@ import ( "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/mandi" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/MandiPrice" ) // mandiProvider implements definition.ProviderStepProvider. type mandiProvider struct{} // newStepFunc creates a new step. Indirected for tests. -var newStepFunc = mandi.New +var newStepFunc = MandiPrice.New // parseConfig turns the plugin config map into a typed Config. Anything absent -// is left zero: mandi.New applies the defaults and validates the auth scheme, +// is left zero: MandiPrice.New applies the defaults and validates the auth scheme, // so those rules live in one place. -func (p mandiProvider) parseConfig(config map[string]string) (*mandi.Config, error) { - cfg := &mandi.Config{ +func (p mandiProvider) parseConfig(config map[string]string) (*MandiPrice.Config, error) { + cfg := &MandiPrice.Config{ BindingKeys: splitList(config["bindingKeys"]), // Absent means the Beckn v2 convention. See upstream.Config for why // this is a default rather than something to set. diff --git a/pkg/plugin/implementation/mandi/cmd/plugin_test.go b/pkg/plugin/implementation/MandiPrice/cmd/plugin_test.go similarity index 95% rename from pkg/plugin/implementation/mandi/cmd/plugin_test.go rename to pkg/plugin/implementation/MandiPrice/cmd/plugin_test.go index 262f73a6..0a3c1a50 100644 --- a/pkg/plugin/implementation/mandi/cmd/plugin_test.go +++ b/pkg/plugin/implementation/MandiPrice/cmd/plugin_test.go @@ -9,7 +9,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/mandi" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/MandiPrice" ) type stubRegistry struct{} @@ -32,15 +32,15 @@ func TestParseConfig(t *testing.T) { testCases := []struct { name string config map[string]string - expected *mandi.Config + expected *MandiPrice.Config expectedErr string }{ { - // Everything absent is left zero: mandi.New defaults it, so the + // Everything absent is left zero: MandiPrice.New defaults it, so the // rules are defined in exactly one place. name: "leaves everything unset for New to default", config: map[string]string{}, - expected: &mandi.Config{}, + expected: &MandiPrice.Config{}, }, { // Query auth is why this capability has its own entry rather than @@ -54,7 +54,7 @@ func TestParseConfig(t *testing.T) { "queryName": "api-key", "queryValueEnv": "MANDI_TOKEN", }, - expected: &mandi.Config{ + expected: &MandiPrice.Config{ BindingKeys: []string{"agmarknet|openagrinet:MandiPrice"}, AuthScheme: "query", QueryName: "api-key", @@ -74,7 +74,7 @@ func TestParseConfig(t *testing.T) { "queryValueEnv": "Q", "maxResponseBytes": "2048", }, - expected: &mandi.Config{ + expected: &MandiPrice.Config{ BindingKeys: []string{"other|capability"}, AuthScheme: "basic", UsernameEnv: "U", @@ -102,7 +102,7 @@ func TestParseConfig(t *testing.T) { // as "unset" rather than failing startup. name: "treats an empty response cap as unset", config: map[string]string{"maxResponseBytes": ""}, - expected: &mandi.Config{}, + expected: &MandiPrice.Config{}, }, } @@ -243,7 +243,7 @@ func TestNew(t *testing.T) { closed := false original := newStepFunc newStepFunc = func(context.Context, definition.ProviderRecordLookup, definition.Mapper, - *mandi.Config) (definition.Step, func() error, error) { + *MandiPrice.Config) (definition.Step, func() error, error) { return nil, func() error { closed = true; return nil }, nil } defer func() { newStepFunc = original }() @@ -268,7 +268,7 @@ func TestNew(t *testing.T) { original := newStepFunc wanted := errors.New("upstream refused the config") newStepFunc = func(context.Context, definition.ProviderRecordLookup, definition.Mapper, - *mandi.Config) (definition.Step, func() error, error) { + *MandiPrice.Config) (definition.Step, func() error, error) { return nil, nil, wanted } defer func() { newStepFunc = original }() diff --git a/pkg/plugin/implementation/mandi/mappings_test.go b/pkg/plugin/implementation/MandiPrice/mappings_test.go similarity index 98% rename from pkg/plugin/implementation/mandi/mappings_test.go rename to pkg/plugin/implementation/MandiPrice/mappings_test.go index 6be53f46..ef91a40c 100644 --- a/pkg/plugin/implementation/mandi/mappings_test.go +++ b/pkg/plugin/implementation/MandiPrice/mappings_test.go @@ -1,4 +1,4 @@ -package mandi_test +package MandiPrice_test // mappings_test.go runs the shipped mandi mapping through the real mapper and // the real provider step. It is the only test that proves the three pieces fit: @@ -22,8 +22,8 @@ import ( "testing" "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/MandiPrice" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/mandi" ) // mappingsDir is where the shipped mappings live, relative to this package. @@ -191,8 +191,8 @@ func runShippedWith(t *testing.T, request, providerBody string) (url.Values, map }, }} - step, closeStep, err := mandi.New(context.Background(), registry, mapper, - &mandi.Config{BindingKeys: []string{shippedBindingKey}}) + step, closeStep, err := MandiPrice.New(context.Background(), registry, mapper, + &MandiPrice.Config{BindingKeys: []string{shippedBindingKey}}) if err != nil { t.Fatalf("failed to build the step: %v", err) } @@ -431,8 +431,8 @@ func TestShippedMappingRefusesWhatItCannotServe(t *testing.T) { Mappings: mappings.URL + "/" + shippedMapping, TimeoutMs: 30000}, }, }} - step, closeStep, err := mandi.New(context.Background(), registry, mapper, - &mandi.Config{BindingKeys: []string{shippedBindingKey}}) + step, closeStep, err := MandiPrice.New(context.Background(), registry, mapper, + &MandiPrice.Config{BindingKeys: []string{shippedBindingKey}}) if err != nil { t.Fatalf("failed to build the step: %v", err) } @@ -740,8 +740,8 @@ func TestShippedMappingRefusesPayloadsItCannotAnswer(t *testing.T) { Mappings: mappings.URL + "/" + shippedMapping, TimeoutMs: 30000}, }, }} - step, closeStep, err := mandi.New(context.Background(), registry, mapper, - &mandi.Config{BindingKeys: []string{shippedBindingKey}}) + step, closeStep, err := MandiPrice.New(context.Background(), registry, mapper, + &MandiPrice.Config{BindingKeys: []string{shippedBindingKey}}) if err != nil { t.Fatalf("failed to build the step: %v", err) } diff --git a/pkg/plugin/implementation/mandi/prerequisites.go b/pkg/plugin/implementation/MandiPrice/prerequisites.go similarity index 98% rename from pkg/plugin/implementation/mandi/prerequisites.go rename to pkg/plugin/implementation/MandiPrice/prerequisites.go index 2027248b..d583092c 100644 --- a/pkg/plugin/implementation/mandi/prerequisites.go +++ b/pkg/plugin/implementation/MandiPrice/prerequisites.go @@ -1,4 +1,4 @@ -package mandi +package MandiPrice import "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" diff --git a/pkg/plugin/implementation/weather/weather.go b/pkg/plugin/implementation/WeatherObservation/WeatherObservation.go similarity index 68% rename from pkg/plugin/implementation/weather/weather.go rename to pkg/plugin/implementation/WeatherObservation/WeatherObservation.go index b899e45b..65820e84 100644 --- a/pkg/plugin/implementation/weather/weather.go +++ b/pkg/plugin/implementation/WeatherObservation/WeatherObservation.go @@ -1,15 +1,18 @@ -// Package weather serves the network's weather capabilities. +// Package WeatherObservation serves the network's weather capabilities. // -// One package per schema pack family, so which plugin owns a capability is -// readable from its binding key: openagrinet:WeatherObservation and -// openagrinet:WeatherAdvisory are weather's, openagrinet:MandiPrice is not. +// One package per capability, named for the capability it serves, so which +// plugin owns a payload is readable from its binding key without a lookup: +// openagrinet:WeatherObservation is this one's, openagrinet:MandiPrice is not. +// Which keys it answers to is still configuration -- a deployment can point it +// at a related pack such as openagrinet:WeatherAdvisory -- but the name says +// what it was built against. // // Almost nothing lives here. Recognising a capability, resolving the call plan, // authenticating, calling with the registry's budget and translating in both // directions are all internal/upstream's, because none of them differ by domain. // What this package owns is its name, and prerequisites -- the work a mapping // cannot express, which is domain knowledge by definition. -package weather +package WeatherObservation import ( "context" diff --git a/pkg/plugin/implementation/weather/cmd/plugin.go b/pkg/plugin/implementation/WeatherObservation/cmd/plugin.go similarity index 91% rename from pkg/plugin/implementation/weather/cmd/plugin.go rename to pkg/plugin/implementation/WeatherObservation/cmd/plugin.go index e78c17ef..9a53bf97 100644 --- a/pkg/plugin/implementation/weather/cmd/plugin.go +++ b/pkg/plugin/implementation/WeatherObservation/cmd/plugin.go @@ -9,20 +9,20 @@ import ( "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/weather" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/WeatherObservation" ) // weatherProvider implements definition.ProviderStepProvider. type weatherProvider struct{} // newStepFunc creates a new step. Indirected for tests. -var newStepFunc = weather.New +var newStepFunc = WeatherObservation.New // parseConfig turns the plugin config map into a typed Config. Anything absent -// is left zero: weather.New applies the defaults and validates the auth +// is left zero: WeatherObservation.New applies the defaults and validates the auth // scheme, so those rules live in one place. -func (p weatherProvider) parseConfig(config map[string]string) (*weather.Config, error) { - cfg := &weather.Config{ +func (p weatherProvider) parseConfig(config map[string]string) (*WeatherObservation.Config, error) { + cfg := &WeatherObservation.Config{ BindingKeys: splitList(config["bindingKeys"]), // Absent means the Beckn v2 convention. See upstream.Config for why // this is a default rather than something to set. diff --git a/pkg/plugin/implementation/weather/cmd/plugin_test.go b/pkg/plugin/implementation/WeatherObservation/cmd/plugin_test.go similarity index 94% rename from pkg/plugin/implementation/weather/cmd/plugin_test.go rename to pkg/plugin/implementation/WeatherObservation/cmd/plugin_test.go index e04b9191..f95323da 100644 --- a/pkg/plugin/implementation/weather/cmd/plugin_test.go +++ b/pkg/plugin/implementation/WeatherObservation/cmd/plugin_test.go @@ -9,7 +9,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/weather" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/WeatherObservation" ) type stubRegistry struct{} @@ -32,15 +32,15 @@ func TestParseConfig(t *testing.T) { testCases := []struct { name string config map[string]string - expected *weather.Config + expected *WeatherObservation.Config expectedErr string }{ { - // Everything absent is left zero: weather.New defaults it, so the + // Everything absent is left zero: WeatherObservation.New defaults it, so the // rules are defined in exactly one place. name: "leaves everything unset for New to default", config: map[string]string{}, - expected: &weather.Config{}, + expected: &WeatherObservation.Config{}, }, { name: "reads every supported setting", @@ -53,7 +53,7 @@ func TestParseConfig(t *testing.T) { "headerValueEnv": "V", "maxResponseBytes": "2048", }, - expected: &weather.Config{ + expected: &WeatherObservation.Config{ BindingKeys: []string{"other|capability"}, AuthScheme: "basic", UsernameEnv: "U", @@ -222,7 +222,7 @@ func TestNew(t *testing.T) { t.Cleanup(func() { newStepFunc = original }) wantErr := errors.New("boom") - newStepFunc = func(context.Context, definition.ProviderRecordLookup, definition.Mapper, *weather.Config) (definition.Step, func() error, error) { + newStepFunc = func(context.Context, definition.ProviderRecordLookup, definition.Mapper, *WeatherObservation.Config) (definition.Step, func() error, error) { return nil, nil, wantErr } diff --git a/pkg/plugin/implementation/weather/mappings_test.go b/pkg/plugin/implementation/WeatherObservation/mappings_test.go similarity index 98% rename from pkg/plugin/implementation/weather/mappings_test.go rename to pkg/plugin/implementation/WeatherObservation/mappings_test.go index 4c275765..b25c04d1 100644 --- a/pkg/plugin/implementation/weather/mappings_test.go +++ b/pkg/plugin/implementation/WeatherObservation/mappings_test.go @@ -1,4 +1,4 @@ -package weather_test +package WeatherObservation_test // mappings_test.go runs the shipped mapping files through the real mapper and // the real provider step. It is the only test that proves the three pieces fit: @@ -22,8 +22,8 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/WeatherObservation" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/weather" ) // mappingsDir is where the shipped mappings live, relative to this package. @@ -142,8 +142,8 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { }, }} - step, closeStep, err := weather.New(context.Background(), registry, mapper, - &weather.Config{BindingKeys: []string{shippedBindingKey}}) + step, closeStep, err := WeatherObservation.New(context.Background(), registry, mapper, + &WeatherObservation.Config{BindingKeys: []string{shippedBindingKey}}) if err != nil { t.Fatalf("failed to build the step: %v", err) } diff --git a/pkg/plugin/implementation/weather/prerequisites.go b/pkg/plugin/implementation/WeatherObservation/prerequisites.go similarity index 97% rename from pkg/plugin/implementation/weather/prerequisites.go rename to pkg/plugin/implementation/WeatherObservation/prerequisites.go index 0c7b947a..3492d577 100644 --- a/pkg/plugin/implementation/weather/prerequisites.go +++ b/pkg/plugin/implementation/WeatherObservation/prerequisites.go @@ -1,4 +1,4 @@ -package weather +package WeatherObservation import "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" From 586ed23cc1184e5832f7463514bab70ed63a92eb Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 13:23:24 +0530 Subject: [PATCH 084/130] feat(config): validate resource attributes against their schema packs [#16] The base validator treats resourceAttributes as a free-form object -- the envelope is correct whatever a capability puts inside it. Extended validation checks the inside: it walks the payload for objects carrying @context and @type, resolves the schema @type names, and validates against it. Until the JSON-LD strip was fixed no pack could pass, so the keys were configured and the layer was off. Resolution is local, not a fetch per payload. Every schema under extendedSchema_localSchemaPath is loaded at startup and looked up by @type, so a select costs no network call and works with no egress. The schemas are published elsewhere and deliberately not copied in here; a copy would drift, and pinning a revision in adapter config would make a deployment decision on the deployment's behalf. The comment states the layout to mount, $ref targets included. A missing directory fails startup rather than degrading, which is the behaviour to want -- the alternative is accepting unvalidated payloads because a mount was forgotten. The allowlist is narrowed from raw.githubusercontent.com to the host the packs' own @context names, so a local miss fails loudly instead of quietly fetching from elsewhere. Recorded in the comment because a green result is otherwise misleading: the validator library parses if/then/else but never evaluates it, so a pack's conditional rules are not enforced. In the OAN packs that is everything predicated on informationMode. [#16] Please enter the commit message for your changes. Lines starting [#16] with '#' will be ignored, and an empty message aborts the commit. [#16] [#16] Date: Mon Sep 7 13:23:24 2026 +0530 [#16] [#16] interactive rebase in progress; onto 983affb [#16] Last commands done (3 commands done): [#16] reword 247022c refactor(plugins): name the capability plugins after their capabilities [#16] [#16] reword ec87958 feat(config): validate resource attributes against their schema packs [#16] [#16] Next commands to do (3 remaining commands): [#16] reword bc8979b refactor(config): resolve capability schemas from the payload's @context [#16] [#16] reword 6385051 refactor: name the registry and binding packages for what they are [#16] [#16] You are currently editing a commit while rebasing branch 'tmp/rebase-16' on '983affb'. [#16] [#16] Changes to be committed: [#16] modified: config/oan-provider-adapter.yaml [#16] --- config/oan-provider-adapter.yaml | 72 ++++++++++++++++++++++++++++---- 1 file changed, 64 insertions(+), 8 deletions(-) diff --git a/config/oan-provider-adapter.yaml b/config/oan-provider-adapter.yaml index 87bcd08c..d7e1812e 100644 --- a/config/oan-provider-adapter.yaml +++ b/config/oan-provider-adapter.yaml @@ -154,22 +154,78 @@ modules: signValidator: id: signvalidator - # Base Beckn v2 schema validation against the pinned LTS spec. The - # extended layer fetches each resource's own @context and validates - # against that -- a network call per payload and a second thing that - # can fail -- so it is off, and the extendedSchema_* keys below only - # take effect if it is switched on. + # ------------------------------------------------------------------ + # schemaValidator -- two layers, both on. + # + # BASE validates the envelope against the pinned Beckn v2 LTS spec. + # To it, resourceAttributes is a free-form object: the envelope is + # correct whatever a capability puts inside. + # + # EXTENDED validates that inside. It walks the payload for every + # object carrying both @context and @type, resolves the schema that + # @type names, and validates the object against it. That is what makes + # a wrong unit or a missing required attribute a rejected payload + # rather than a provider's problem to discover later. + # + # What extended validation DOES enforce: types, string formats + # (date-time, duration, uri), enum, const, required, minItems, + # additionalProperties, not, and allOf/anyOf/oneOf. + # + # What it does NOT: if/then/else. The validator library parses those + # keywords but never evaluates them, so a pack's conditional rules -- + # in the OAN packs, everything predicated on informationMode -- are + # not checked. Worth knowing before treating a pass here as full + # conformance to a pack. + # ------------------------------------------------------------------ schemaValidator: id: schemav2validator config: type: url location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" cacheTTL: "3600" - extendedSchema_enabled: "false" - extendedSchema_cacheTTL: "86400" + + extendedSchema_enabled: "true" + + # Where the capability schemas are read from. Set, so resolution + # is a memory lookup and no payload costs a network call. + # + # Every *.yaml under this directory is loaded at STARTUP and keyed + # by /attributes.yaml, with the version segment dropped. + # An object's @type is then looked up directly -- the part after + # the colon, so openagrinet:MandiPrice finds MandiPrice. $refs + # between schemas resolve out of the same memory. + # + # The layout to place there, $ref targets included: + # + # WeatherObservation/v0.1/attributes.yaml + # MandiPrice/v0.1/attributes.yaml + # AgricultureResource/v0.1/attributes.yaml # the packs' base + # Address/v2.0/attributes.yaml # schema.beckn.io + # Descriptor/v2.1/attributes.yaml # schema.beckn.io + # GeoJSONGeometry/v2.0/attributes.yaml # schema.beckn.io + # Location/v2.0/attributes.yaml # schema.beckn.io + # + # The schemas are NOT in this repository: they are published, and + # a second copy here would drift from them. The deployment places + # them at this path, which is also why no source URL appears in + # this file -- which revision to ship is a deployment decision, + # not one frozen into adapter config. + # + # A missing directory FAILS STARTUP rather than degrading, which + # is what you want: the alternative is an adapter that accepts + # unvalidated payloads because a mount was forgotten. An empty one + # only warns, and then every payload falls through to the network. + extendedSchema_localSchemaPath: "/app/config/schemas" + + # The network fallback, reached only on a local miss. Restricted + # to the host the packs' own @context names, so a miss fails + # loudly instead of quietly fetching a schema from somewhere else. + extendedSchema_allowedDomains: "schemas.openagrinet.global" + + # The three keys below apply to that fallback only. + extendedSchema_cacheTTL: "86400" # 24h extendedSchema_maxCacheSize: "100" extendedSchema_downloadTimeout: "30" - extendedSchema_allowedDomains: "raw.githubusercontent.com" # ------------------------------------------------------------------ # jsonmapper -- the JSONata mapper. From 4fdf394d4be5724bde6b8f91fbd8205e0e178b1d Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 14:17:15 +0530 Subject: [PATCH 085/130] refactor(config): resolve capability schemas from the payload's @context [#16] Extended validation was pointed at a mounted directory of schema files, because the @context the payloads declared -- schemas.openagrinet.global -- does not resolve, and a failed fetch rejects the payload. That put the burden on every deployment to place the right files at the right path, and made the adapter refuse to start when one did not. The published packs do serve context.jsonld, so the fetch the validator already knows how to do works once @context names them: @context .../network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/context.jsonld fetched .../network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/attributes.yaml So localSchemaPath is gone, and with it the mount, the directory and the script that filled it. The revision a payload is judged against now comes from the payload, which is a better place for it than adapter config: nothing here can go stale against what the network publishes. The allowlist moves to raw.githubusercontent.com, the host that @context now resolves to. It is doing real work rather than naming a host that never answered -- an @context on any other host is refused before a fetch is attempted. Two costs, both stated in the config: this adapter now needs egress to that host, and the first payload after a restart pays for the fetch. Measured at about 2s, cached for 24h after that. Verified in oan-local: publish and select both pass, the first payload logs "fetching from network" and later ones "LRU cache hit", a foreign @context is refused with SCH_INVALID_JSONLD_CONTEXT, and the collection is 51 of 51. [#16] Please enter the commit message for your changes. Lines starting [#16] with '#' will be ignored, and an empty message aborts the commit. [#16] [#16] Date: Mon Sep 7 14:17:15 2026 +0530 [#16] [#16] interactive rebase in progress; onto 983affb [#16] Last commands done (4 commands done): [#16] reword ec87958 feat(config): validate resource attributes against their schema packs [#16] [#16] reword bc8979b refactor(config): resolve capability schemas from the payload's @context [#16] [#16] Next commands to do (2 remaining commands): [#16] reword 6385051 refactor: name the registry and binding packages for what they are [#16] [#16] reword 9ebb5c1 docs(config): placeholder the subscriber id, keep the old value in a comment [#16] [#16] You are currently editing a commit while rebasing branch 'tmp/rebase-16' on '983affb'. [#16] [#16] Changes to be committed: [#16] modified: config/oan-provider-adapter.yaml [#16] --- config/oan-provider-adapter.yaml | 53 ++++++++++++-------------------- 1 file changed, 20 insertions(+), 33 deletions(-) diff --git a/config/oan-provider-adapter.yaml b/config/oan-provider-adapter.yaml index d7e1812e..d3694a63 100644 --- a/config/oan-provider-adapter.yaml +++ b/config/oan-provider-adapter.yaml @@ -167,6 +167,10 @@ modules: # a wrong unit or a missing required attribute a rejected payload # rather than a provider's problem to discover later. # + # The schemas are not in this repository and are not mounted: they + # are fetched from the @context a payload declares, so the revision + # is the payload's choice and nothing here can go stale against it. + # # What extended validation DOES enforce: types, string formats # (date-time, duration, uri), enum, const, required, minItems, # additionalProperties, not, and allOf/anyOf/oneOf. @@ -186,43 +190,26 @@ modules: extendedSchema_enabled: "true" - # Where the capability schemas are read from. Set, so resolution - # is a memory lookup and no payload costs a network call. - # - # Every *.yaml under this directory is loaded at STARTUP and keyed - # by /attributes.yaml, with the version segment dropped. - # An object's @type is then looked up directly -- the part after - # the colon, so openagrinet:MandiPrice finds MandiPrice. $refs - # between schemas resolve out of the same memory. + # Resolution is a FETCH of the @context each resource declares, + # not a directory mounted here. The validator swaps context.jsonld + # for attributes.yaml to get the schema beside it: # - # The layout to place there, $ref targets included: + # @context .../schema/MandiPrice/v0.1/context.jsonld + # fetched .../schema/MandiPrice/v0.1/attributes.yaml # - # WeatherObservation/v0.1/attributes.yaml - # MandiPrice/v0.1/attributes.yaml - # AgricultureResource/v0.1/attributes.yaml # the packs' base - # Address/v2.0/attributes.yaml # schema.beckn.io - # Descriptor/v2.1/attributes.yaml # schema.beckn.io - # GeoJSONGeometry/v2.0/attributes.yaml # schema.beckn.io - # Location/v2.0/attributes.yaml # schema.beckn.io + # So a payload names the pack revision it wants to be judged + # against, and no copy of the schemas here can drift from the + # published ones. The packs' relative $refs (into + # AgricultureResource) resolve against that same base; their + # absolute ones resolve directly, against whichever host they name. # - # The schemas are NOT in this repository: they are published, and - # a second copy here would drift from them. The deployment places - # them at this path, which is also why no source URL appears in - # this file -- which revision to ship is a deployment decision, - # not one frozen into adapter config. - # - # A missing directory FAILS STARTUP rather than degrading, which - # is what you want: the alternative is an adapter that accepts - # unvalidated payloads because a mount was forgotten. An empty one - # only warns, and then every payload falls through to the network. - extendedSchema_localSchemaPath: "/app/config/schemas" - - # The network fallback, reached only on a local miss. Restricted - # to the host the packs' own @context names, so a miss fails - # loudly instead of quietly fetching a schema from somewhere else. - extendedSchema_allowedDomains: "schemas.openagrinet.global" + # Fetched once per @context and cached for the TTL below, so only + # the first payload after a restart pays for it. A fetch that + # FAILS rejects the payload -- it does not skip validation, which + # is the right way round, but it does mean this adapter needs + # egress to the host allowed below. + extendedSchema_allowedDomains: "raw.githubusercontent.com" - # The three keys below apply to that fallback only. extendedSchema_cacheTTL: "86400" # 24h extendedSchema_maxCacheSize: "100" extendedSchema_downloadTimeout: "30" From 4d18317353c428ac7c29729e070c082d6c9db6c8 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 17:15:52 +0530 Subject: [PATCH 086/130] refactor: name the registry and binding packages for what they are [#16] Three renames, and one rule applied across the tree. pkg/plugin/implementation/oanregistry -> sunbirdRegistry pkg/plugin/implementation/internal/oanbinding -> internal/capabilitybinding config/oan-provider-adapter.yaml -> config/provider-adapter.yaml capabilitybinding rather than keybinding, which was the suggestion: this package derives a {ParticipantID, CapabilityCode} pair and renders it as the participant|capability key the registry indexes on. "Capability binding" is already the term the code, the registry schema and the config all use, so the package now says it. "Key" would have been actively misleading -- this codebase uses key for signing keys, which is a different thing resolved by a different plugin. The organisation name is gone from every filename, identifier, comment, error string and trace span name. Two categories were deliberately left alone because they are DATA rather than naming: - openagrinet:WeatherObservation and its siblings are capability codes on the wire. Changing them would change the protocol. - participant ids and names inside captured registry fixtures (provider.oan.local, "OAN provider layer adapter") are verbatim responses. Editing them would make the fixture stop matching what a registry returns, which is the only reason the fixture is worth having. Two consequences worth knowing. The .so basename is the plugin id, so `id: oanregistry` becomes `id: sunbirdRegistry` and any deployment's config moves with the image. And pluginID is a telemetry attribute, so traces and metrics from this plugin now report sunbirdRegistry -- it follows the rename rather than reporting a name that no longer exists. The two cache key prefixes changed with it, oan_lookup_ and oan_provider_ to registry_lookup_ and registry_provider_. They are cache namespaces, not metric names, so the cost is one cold cache cycle. Full suite green at 63 packages, vet clean, and sunbirdRegistry.so builds. [#16] Please enter the commit message for your changes. Lines starting [#16] with '#' will be ignored, and an empty message aborts the commit. [#16] [#16] Date: Mon Sep 7 17:15:52 2026 +0530 [#16] [#16] interactive rebase in progress; onto 983affb [#16] Last commands done (5 commands done): [#16] reword bc8979b refactor(config): resolve capability schemas from the payload's @context [#16] [#16] reword 6385051 refactor: name the registry and binding packages for what they are [#16] [#16] Next command to do (1 remaining command): [#16] reword 9ebb5c1 docs(config): placeholder the subscriber id, keep the old value in a comment [#16] [#16] You are currently editing a commit while rebasing branch 'tmp/rebase-16' on '983affb'. [#16] [#16] Changes to be committed: [#16] modified: config/mappings/agmarknet/mandi-price.select.yaml [#16] renamed: config/oan-provider-adapter.yaml -> config/provider-adapter.yaml [#16] modified: install/build-plugins.sh [#16] modified: pkg/plugin/definition/mapper.go [#16] modified: pkg/plugin/implementation/WeatherObservation/mappings_test.go [#16] renamed: pkg/plugin/implementation/internal/oanbinding/oanbinding.go -> pkg/plugin/implementation/internal/capabilitybinding/capabilitybinding.go [#16] renamed: pkg/plugin/implementation/internal/oanbinding/oanbinding_test.go -> pkg/plugin/implementation/internal/capabilitybinding/capabilitybinding_test.go [#16] renamed: pkg/plugin/implementation/internal/oanbinding/paths.go -> pkg/plugin/implementation/internal/capabilitybinding/paths.go [#16] modified: pkg/plugin/implementation/internal/upstream/upstream.go [#16] modified: pkg/plugin/implementation/internal/upstream/upstream_test.go [#16] modified: pkg/plugin/implementation/jsonmapper/README.md [#16] modified: pkg/plugin/implementation/schemav2validator/extended_schema.go [#16] modified: pkg/plugin/implementation/schemav2validator/extended_schema_test.go [#16] renamed: pkg/plugin/implementation/oanregistry/README.md -> pkg/plugin/implementation/sunbirdRegistry/README.md [#16] renamed: pkg/plugin/implementation/oanregistry/cmd/plugin.go -> pkg/plugin/implementation/sunbirdRegistry/cmd/plugin.go [#16] renamed: pkg/plugin/implementation/oanregistry/cmd/plugin_test.go -> pkg/plugin/implementation/sunbirdRegistry/cmd/plugin_test.go [#16] renamed: pkg/plugin/implementation/oanregistry/providerrecord.go -> pkg/plugin/implementation/sunbirdRegistry/providerrecord.go [#16] renamed: pkg/plugin/implementation/oanregistry/providerrecord_test.go -> pkg/plugin/implementation/sunbirdRegistry/providerrecord_test.go [#16] renamed: pkg/plugin/implementation/oanregistry/oanregistry.go -> pkg/plugin/implementation/sunbirdRegistry/sunbirdRegistry.go [#16] renamed: pkg/plugin/implementation/oanregistry/oanregistry_test.go -> pkg/plugin/implementation/sunbirdRegistry/sunbirdRegistry_test.go [#16] --- .../agmarknet/mandi-price.select.yaml | 5 ++- ...der-adapter.yaml => provider-adapter.yaml} | 6 +-- install/build-plugins.sh | 2 +- pkg/plugin/definition/mapper.go | 2 +- .../WeatherObservation/mappings_test.go | 2 +- .../capabilitybinding.go} | 16 +++---- .../capabilitybinding_test.go} | 4 +- .../paths.go | 6 +-- .../internal/upstream/upstream.go | 20 ++++----- .../internal/upstream/upstream_test.go | 4 +- .../implementation/jsonmapper/README.md | 2 +- .../schemav2validator/extended_schema.go | 2 +- .../schemav2validator/extended_schema_test.go | 2 +- .../README.md | 6 +-- .../cmd/plugin.go | 42 +++++++++--------- .../cmd/plugin_test.go | 44 +++++++++---------- .../providerrecord.go | 10 ++--- .../providerrecord_test.go | 4 +- .../sunbirdRegistry.go} | 28 ++++++------ .../sunbirdRegistry_test.go} | 12 ++--- 20 files changed, 110 insertions(+), 109 deletions(-) rename config/{oan-provider-adapter.yaml => provider-adapter.yaml} (99%) rename pkg/plugin/implementation/internal/{oanbinding/oanbinding.go => capabilitybinding/capabilitybinding.go} (84%) rename pkg/plugin/implementation/internal/{oanbinding/oanbinding_test.go => capabilitybinding/capabilitybinding_test.go} (99%) rename pkg/plugin/implementation/internal/{oanbinding => capabilitybinding}/paths.go (95%) rename pkg/plugin/implementation/{oanregistry => sunbirdRegistry}/README.md (99%) rename pkg/plugin/implementation/{oanregistry => sunbirdRegistry}/cmd/plugin.go (75%) rename pkg/plugin/implementation/{oanregistry => sunbirdRegistry}/cmd/plugin_test.go (77%) rename pkg/plugin/implementation/{oanregistry => sunbirdRegistry}/providerrecord.go (98%) rename pkg/plugin/implementation/{oanregistry => sunbirdRegistry}/providerrecord_test.go (99%) rename pkg/plugin/implementation/{oanregistry/oanregistry.go => sunbirdRegistry/sunbirdRegistry.go} (96%) rename pkg/plugin/implementation/{oanregistry/oanregistry_test.go => sunbirdRegistry/sunbirdRegistry_test.go} (99%) diff --git a/config/mappings/agmarknet/mandi-price.select.yaml b/config/mappings/agmarknet/mandi-price.select.yaml index d10a0ce8..81e11115 100644 --- a/config/mappings/agmarknet/mandi-price.select.yaml +++ b/config/mappings/agmarknet/mandi-price.select.yaml @@ -57,8 +57,9 @@ required: this guard, the outbound query, and the commodity stamped on each resource. So a caller sending three commodities passed validation, was queried for the first, and got a confident signed answer to a third of - what it asked -- the same failure oanbinding refuses at the commitment - level, where guessing would silently serve part of a request. */ + what it asked -- the same failure capabilitybinding refuses at the + commitment level, where guessing would silently serve part of a + request. */ $exists($ra.supportedCommodities[0].code) and $count($ra.supportedCommodities) = 1 ) diff --git a/config/oan-provider-adapter.yaml b/config/provider-adapter.yaml similarity index 99% rename from config/oan-provider-adapter.yaml rename to config/provider-adapter.yaml index d3694a63..ddd3df18 100644 --- a/config/oan-provider-adapter.yaml +++ b/config/provider-adapter.yaml @@ -19,7 +19,7 @@ # serves. It does NOT hold where a provider lives or how long to wait for it -- # that is the registry's ProviderSchema row, read per request. Repointing a # provider is a registry write, not an edit here and a restart. -appName: "oan-provider-adapter" +appName: "provider-adapter" log: level: debug @@ -80,7 +80,7 @@ modules: plugins: # ------------------------------------------------------------------ - # oanregistry -- the OAN Registry (SunbirdRC) client. + # sunbirdRegistry -- the SunbirdRC registry client. # # Serves both halves of the lookup: the sender's signing key for # validateSign, and the capability call plans the provider steps @@ -89,7 +89,7 @@ modules: # other. # ------------------------------------------------------------------ registry: - id: oanregistry + id: sunbirdRegistry config: # REQUIRED, and the only key with no default. Include the API # version prefix; the plugin appends /{entity}/search. diff --git a/install/build-plugins.sh b/install/build-plugins.sh index 141cb3b0..b7e64256 100755 --- a/install/build-plugins.sh +++ b/install/build-plugins.sh @@ -30,7 +30,7 @@ plugins=( "publisher" "registry" "dediregistry" - "oanregistry" + "sunbirdRegistry" "jsonmapper" "WeatherObservation" "MandiPrice" diff --git a/pkg/plugin/definition/mapper.go b/pkg/plugin/definition/mapper.go index e5c7be95..e1a3790e 100644 --- a/pkg/plugin/definition/mapper.go +++ b/pkg/plugin/definition/mapper.go @@ -17,7 +17,7 @@ const ( // Mapper transforms a document with a mapping fetched from a reference. // -// It exists so that translating between OAN's Beckn payloads and a provider's +// It exists so that translating between the network's Beckn payloads and a provider's // own shape is configuration rather than code: a new provider ships mapping // files, not a new transformation routine. The mapper itself knows nothing // about any provider, and nothing about what a mapping says -- it fetches, diff --git a/pkg/plugin/implementation/WeatherObservation/mappings_test.go b/pkg/plugin/implementation/WeatherObservation/mappings_test.go index b25c04d1..d8688193 100644 --- a/pkg/plugin/implementation/WeatherObservation/mappings_test.go +++ b/pkg/plugin/implementation/WeatherObservation/mappings_test.go @@ -43,7 +43,7 @@ const shippedBindingKey = "mausamgram|openagrinet:WeatherObservation" const shippedMapping = "weather-observation.select.yaml" -// selectRequest is the verbatim /select captured from the OAN network. +// selectRequest is the verbatim /select captured from the network. const selectRequest = `{ "context": { "version": "2.0.0", "action": "select", "networkId": "da.gov.in/vistaar", diff --git a/pkg/plugin/implementation/internal/oanbinding/oanbinding.go b/pkg/plugin/implementation/internal/capabilitybinding/capabilitybinding.go similarity index 84% rename from pkg/plugin/implementation/internal/oanbinding/oanbinding.go rename to pkg/plugin/implementation/internal/capabilitybinding/capabilitybinding.go index b53290ef..e72b1dd7 100644 --- a/pkg/plugin/implementation/internal/oanbinding/oanbinding.go +++ b/pkg/plugin/implementation/internal/capabilitybinding/capabilitybinding.go @@ -1,10 +1,10 @@ -// Package oanbinding derives the capability binding a Beckn request is asking +// Package capabilitybinding derives the capability binding a Beckn request is asking // for, so a provider step can tell whether the request is its work and, if it // is, which registry row describes the call. // // It is shared by every provider step rather than living in one, because the -// binding is a property of the OAN network's payloads and not of any provider. -package oanbinding +// binding is a property of the network's payloads and not of any provider. +package capabilitybinding import ( "encoding/json" @@ -19,7 +19,7 @@ const separator = "|" // ErrNoBinding reports a payload that names no capability binding. It is not a // fault: a request for something else entirely reaches a provider step too, and // the step's answer is to do nothing. -var ErrNoBinding = errors.New("oanbinding: payload names no capability binding") +var ErrNoBinding = errors.New("capabilitybinding: payload names no capability binding") // Binding identifies one provider capability. type Binding struct { @@ -47,7 +47,7 @@ func (b Binding) Key() string { func From(paths Paths, body []byte) (Binding, error) { var payload any if err := json.Unmarshal(body, &payload); err != nil { - return Binding{}, fmt.Errorf("oanbinding: payload could not be read: %w", err) + return Binding{}, fmt.Errorf("capabilitybinding: payload could not be read: %w", err) } // Before distinctness: N commitments naming the SAME provider and type @@ -64,7 +64,7 @@ func From(paths Paths, body []byte) (Binding, error) { // the paragraph above says is refused. if commitments := countAt(payload, paths.ProviderID); commitments > 1 { return Binding{}, fmt.Errorf( - "oanbinding: payload carries %d commitments; one request maps to one call, "+ + "capabilitybinding: payload carries %d commitments; one request maps to one call, "+ "so send them separately rather than have all but the first dropped", commitments) } @@ -76,11 +76,11 @@ func From(paths Paths, body []byte) (Binding, error) { return Binding{}, ErrNoBinding } if len(providers) > 1 { - return Binding{}, fmt.Errorf("oanbinding: payload names %d providers (%s); one request maps to one call", + return Binding{}, fmt.Errorf("capabilitybinding: payload names %d providers (%s); one request maps to one call", len(providers), strings.Join(providers, ", ")) } if len(types) > 1 { - return Binding{}, fmt.Errorf("oanbinding: payload names %d resource types (%s); one request maps to one call", + return Binding{}, fmt.Errorf("capabilitybinding: payload names %d resource types (%s); one request maps to one call", len(types), strings.Join(types, ", ")) } diff --git a/pkg/plugin/implementation/internal/oanbinding/oanbinding_test.go b/pkg/plugin/implementation/internal/capabilitybinding/capabilitybinding_test.go similarity index 99% rename from pkg/plugin/implementation/internal/oanbinding/oanbinding_test.go rename to pkg/plugin/implementation/internal/capabilitybinding/capabilitybinding_test.go index 1a9d0836..43213053 100644 --- a/pkg/plugin/implementation/internal/oanbinding/oanbinding_test.go +++ b/pkg/plugin/implementation/internal/capabilitybinding/capabilitybinding_test.go @@ -1,4 +1,4 @@ -package oanbinding +package capabilitybinding import ( "encoding/json" @@ -7,7 +7,7 @@ import ( "testing" ) -// realSelectPayload is a verbatim /select request captured from the OAN network +// realSelectPayload is a verbatim /select request captured from the network // on 29 Aug 2026. It is the reason this package reads through contract and // commitments rather than off message directly: the design notes showed the // shallower message.offer.provider.id, and the wire does not. diff --git a/pkg/plugin/implementation/internal/oanbinding/paths.go b/pkg/plugin/implementation/internal/capabilitybinding/paths.go similarity index 95% rename from pkg/plugin/implementation/internal/oanbinding/paths.go rename to pkg/plugin/implementation/internal/capabilitybinding/paths.go index 5c47deb2..f1f89a2a 100644 --- a/pkg/plugin/implementation/internal/oanbinding/paths.go +++ b/pkg/plugin/implementation/internal/capabilitybinding/paths.go @@ -1,4 +1,4 @@ -package oanbinding +package capabilitybinding import ( "fmt" @@ -38,11 +38,11 @@ func (p Paths) Validate() error { "capabilityCodeAt": p.CapabilityCode, } { if strings.TrimSpace(path) == "" { - return fmt.Errorf("oanbinding: %s is empty", name) + return fmt.Errorf("capabilitybinding: %s is empty", name) } for _, segment := range strings.Split(path, ".") { if strings.TrimSpace(strings.TrimSuffix(segment, arrayMarker)) == "" { - return fmt.Errorf("oanbinding: %s (%q) has a blank segment", name, path) + return fmt.Errorf("capabilitybinding: %s (%q) has a blank segment", name, path) } } } diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index 40d28fda..2d70ea0d 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -33,7 +33,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/oanbinding" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/capabilitybinding" ) // Defaults applied when the registry or the operator leaves a setting out. @@ -164,7 +164,7 @@ type Config struct { // safe for concurrent use. type Step struct { config *Config - paths oanbinding.Paths + paths capabilitybinding.Paths prerequisites Prerequisites registry definition.ProviderRecordLookup mapper definition.Mapper @@ -218,19 +218,19 @@ func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper d // Both halves or neither: overriding one and leaving the other on the default // is a half-configured deployment that would match nothing, and it would do so // silently on every request rather than once at startup. -func bindingPaths(cfg *Config) (oanbinding.Paths, error) { +func bindingPaths(cfg *Config) (capabilitybinding.Paths, error) { if cfg.ProviderIDAt == "" && cfg.CapabilityCodeAt == "" { - return oanbinding.BecknV2, nil + return capabilitybinding.BecknV2, nil } if cfg.ProviderIDAt == "" { - return oanbinding.Paths{}, errors.New("upstream: capabilityCodeAt is set without providerIdAt") + return capabilitybinding.Paths{}, errors.New("upstream: capabilityCodeAt is set without providerIdAt") } if cfg.CapabilityCodeAt == "" { - return oanbinding.Paths{}, errors.New("upstream: providerIdAt is set without capabilityCodeAt") + return capabilitybinding.Paths{}, errors.New("upstream: providerIdAt is set without capabilityCodeAt") } - paths := oanbinding.Paths{ProviderID: cfg.ProviderIDAt, CapabilityCode: cfg.CapabilityCodeAt} + paths := capabilitybinding.Paths{ProviderID: cfg.ProviderIDAt, CapabilityCode: cfg.CapabilityCodeAt} if err := paths.Validate(); err != nil { - return oanbinding.Paths{}, err + return capabilitybinding.Paths{}, err } return paths, nil } @@ -283,8 +283,8 @@ func applyDefaults(cfg *Config) error { // pipeline and each recognises its own work, so adding a provider is one more // entry rather than a change to a routing table. func (s *Step) Run(ctx *model.StepContext) error { - binding, err := oanbinding.From(s.paths, ctx.Body) - if errors.Is(err, oanbinding.ErrNoBinding) { + binding, err := capabilitybinding.From(s.paths, ctx.Body) + if errors.Is(err, capabilitybinding.ErrNoBinding) { return nil } if err != nil { diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go index 33530e33..2e871dea 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -17,7 +17,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/oanbinding" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/capabilitybinding" ) const selectBody = `{ @@ -1086,7 +1086,7 @@ func TestNewUsesTheBecknConventionByDefault(t *testing.T) { t.Parallel() step := newStep(t, &stubRegistry{}, &stubMapper{}) - if step.paths != oanbinding.BecknV2 { + if step.paths != capabilitybinding.BecknV2 { t.Errorf("paths = %+v, want the Beckn v2 convention", step.paths) } } diff --git a/pkg/plugin/implementation/jsonmapper/README.md b/pkg/plugin/implementation/jsonmapper/README.md index 1f1d5913..e837331a 100644 --- a/pkg/plugin/implementation/jsonmapper/README.md +++ b/pkg/plugin/implementation/jsonmapper/README.md @@ -13,7 +13,7 @@ what a mapping says, and nothing about the payloads passing through. Anything specific to a network or a provider belongs in the caller, which is what lets one mapper serve all of them. -Its first caller is the OAN provider flow, where it translates between Beckn +Its first caller is the provider flow, where it translates between Beckn payloads and each provider's own request and response shapes -- so adding a provider is one mapping file and a registry row rather than another transformation routine. diff --git a/pkg/plugin/implementation/schemav2validator/extended_schema.go b/pkg/plugin/implementation/schemav2validator/extended_schema.go index 654ba2d7..cf9cc91f 100644 --- a/pkg/plugin/implementation/schemav2validator/extended_schema.go +++ b/pkg/plugin/implementation/schemav2validator/extended_schema.go @@ -531,7 +531,7 @@ func stripUnaccountedJSONLDKeys(schema *openapi3.SchemaRef, data map[string]inte // listed as required, anywhere in a schema's composition tree. // // allOf, anyOf, oneOf and the then/else branches can each introduce a property, -// so all of them are walked -- the OAN packs declare @type one level down, in +// so all of them are walked -- the capability packs declare @type one level down, in // allOf. "not" is skipped because naming a property there forbids it rather // than permitting it, and "if" is skipped because it only selects a branch. // seen guards against schemas that reference themselves. diff --git a/pkg/plugin/implementation/schemav2validator/extended_schema_test.go b/pkg/plugin/implementation/schemav2validator/extended_schema_test.go index e38325b1..8f7e378c 100644 --- a/pkg/plugin/implementation/schemav2validator/extended_schema_test.go +++ b/pkg/plugin/implementation/schemav2validator/extended_schema_test.go @@ -1379,7 +1379,7 @@ func TestLoadSchemaFromPath_TTLExpiry_FetchesFresh(t *testing.T) { assert.Equal(t, "Schema v2", doc2.Info.Title, "expected v2 after TTL expiry — global URIMapCache not bypassed") } -// packStyleSchema mirrors how the OAN schema packs are shaped: the capability +// packStyleSchema mirrors how the capability schema packs are shaped: the capability // declares @type one level down in allOf and lists it as required, and nothing // closes the object with additionalProperties:false. const packStyleSchema = `openapi: 3.1.0 diff --git a/pkg/plugin/implementation/oanregistry/README.md b/pkg/plugin/implementation/sunbirdRegistry/README.md similarity index 99% rename from pkg/plugin/implementation/oanregistry/README.md rename to pkg/plugin/implementation/sunbirdRegistry/README.md index cbce9a81..459d51fe 100644 --- a/pkg/plugin/implementation/oanregistry/README.md +++ b/pkg/plugin/implementation/sunbirdRegistry/README.md @@ -1,6 +1,6 @@ -# OAN Registry Plugin +# SunbirdRC Registry Plugin -A **registry type plugin** for Beckn-ONIX that reads the OAN Registry, a +A **registry type plugin** for Beckn-ONIX that reads a SunbirdRC registry, a [SunbirdRC](https://docs.sunbirdrc.dev/) deployment. ## Overview @@ -31,7 +31,7 @@ request spends waiting before it can even be rejected. ```yaml registry: - id: oanregistry + id: sunbirdRegistry config: url: http://registry:8081/api/v1 entity: Participant diff --git a/pkg/plugin/implementation/oanregistry/cmd/plugin.go b/pkg/plugin/implementation/sunbirdRegistry/cmd/plugin.go similarity index 75% rename from pkg/plugin/implementation/oanregistry/cmd/plugin.go rename to pkg/plugin/implementation/sunbirdRegistry/cmd/plugin.go index 461432a6..5992bf4b 100644 --- a/pkg/plugin/implementation/oanregistry/cmd/plugin.go +++ b/pkg/plugin/implementation/sunbirdRegistry/cmd/plugin.go @@ -9,33 +9,33 @@ import ( "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/oanregistry" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/sunbirdRegistry" ) // Defaults for settings an operator leaves out. Only parseConfig can tell // "absent" from "explicitly zero" -- retry_max of 0 is a legitimate "do not // retry" -- so they are applied here. The values themselves live in the -// oanregistry package so there is exactly one place to change them. +// sunbirdRegistry package so there is exactly one place to change them. const ( - defaultEntity = oanregistry.DefaultEntity - defaultProviderEntity = oanregistry.DefaultProviderEntity - defaultTimeout = oanregistry.DefaultTimeoutSeconds - defaultRetryMax = oanregistry.DefaultRetryMax - defaultRetryWaitMin = oanregistry.DefaultRetryWaitMin - defaultRetryWaitMax = oanregistry.DefaultRetryWaitMax + defaultEntity = sunbirdRegistry.DefaultEntity + defaultProviderEntity = sunbirdRegistry.DefaultProviderEntity + defaultTimeout = sunbirdRegistry.DefaultTimeoutSeconds + defaultRetryMax = sunbirdRegistry.DefaultRetryMax + defaultRetryWaitMin = sunbirdRegistry.DefaultRetryWaitMin + defaultRetryWaitMax = sunbirdRegistry.DefaultRetryWaitMax ) -// oanRegistryProvider implements the RegistryLookupProvider interface for the -// OAN registry plugin. -type oanRegistryProvider struct{} +// sunbirdRegistryProvider implements the RegistryLookupProvider interface for the +// registry plugin. +type sunbirdRegistryProvider struct{} -// newOANRegistryFunc creates a new OAN registry client. Indirected for tests. -var newOANRegistryFunc = oanregistry.New +// newSunbirdRegistryFunc creates a new registry client. Indirected for tests. +var newSunbirdRegistryFunc = sunbirdRegistry.New -// parseConfig parses the configuration map into an oanregistry.Config, starting +// parseConfig parses the configuration map into an sunbirdRegistry.Config, starting // from the defaults and overriding whatever the operator supplied. -func (o oanRegistryProvider) parseConfig(config map[string]string) (*oanregistry.Config, error) { - cfg := &oanregistry.Config{ +func (o sunbirdRegistryProvider) parseConfig(config map[string]string) (*sunbirdRegistry.Config, error) { + cfg := &sunbirdRegistry.Config{ URL: config["url"], Entity: defaultEntity, ProviderEntity: defaultProviderEntity, @@ -135,8 +135,8 @@ func (o oanRegistryProvider) parseConfig(config map[string]string) (*oanregistry return cfg, nil } -// New creates a new OAN registry plugin instance. -func (o oanRegistryProvider) New(ctx context.Context, cache definition.Cache, config map[string]string) (definition.RegistryLookup, func() error, error) { +// New creates a new registry plugin instance. +func (o sunbirdRegistryProvider) New(ctx context.Context, cache definition.Cache, config map[string]string) (definition.RegistryLookup, func() error, error) { if ctx == nil { return nil, nil, errors.New("context cannot be nil") } @@ -144,12 +144,12 @@ func (o oanRegistryProvider) New(ctx context.Context, cache definition.Cache, co cfg, err := o.parseConfig(config) if err != nil { log.Errorf(ctx, err, "Failed to parse OAN registry configuration") - return nil, nil, fmt.Errorf("failed to parse oan registry configuration: %w", err) + return nil, nil, fmt.Errorf("failed to parse registry configuration: %w", err) } log.Debugf(ctx, "OAN registry config mapped: %+v", cfg) - client, closer, err := newOANRegistryFunc(ctx, cache, cfg) + client, closer, err := newSunbirdRegistryFunc(ctx, cache, cfg) if err != nil { log.Errorf(ctx, err, "Failed to create OAN registry instance") return nil, nil, err @@ -160,4 +160,4 @@ func (o oanRegistryProvider) New(ctx context.Context, cache definition.Cache, co } // Provider is the exported plugin instance. -var Provider = oanRegistryProvider{} +var Provider = sunbirdRegistryProvider{} diff --git a/pkg/plugin/implementation/oanregistry/cmd/plugin_test.go b/pkg/plugin/implementation/sunbirdRegistry/cmd/plugin_test.go similarity index 77% rename from pkg/plugin/implementation/oanregistry/cmd/plugin_test.go rename to pkg/plugin/implementation/sunbirdRegistry/cmd/plugin_test.go index 8d36d67a..fb2dfec5 100644 --- a/pkg/plugin/implementation/oanregistry/cmd/plugin_test.go +++ b/pkg/plugin/implementation/sunbirdRegistry/cmd/plugin_test.go @@ -9,11 +9,11 @@ import ( "time" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/oanregistry" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/sunbirdRegistry" ) -func defaultConfig() *oanregistry.Config { - return &oanregistry.Config{ +func defaultConfig() *sunbirdRegistry.Config { + return &sunbirdRegistry.Config{ Entity: defaultEntity, ProviderEntity: defaultProviderEntity, Timeout: defaultTimeout, @@ -26,7 +26,7 @@ func defaultConfig() *oanregistry.Config { func TestParseConfig(t *testing.T) { t.Parallel() - withDefaults := func(apply func(*oanregistry.Config)) *oanregistry.Config { + withDefaults := func(apply func(*sunbirdRegistry.Config)) *sunbirdRegistry.Config { cfg := defaultConfig() apply(cfg) return cfg @@ -35,13 +35,13 @@ func TestParseConfig(t *testing.T) { testCases := []struct { name string config map[string]string - expected *oanregistry.Config + expected *sunbirdRegistry.Config expectedErr string }{ { name: "applies defaults when only a URL is given", config: map[string]string{"url": "http://registry:8081/api/v1"}, - expected: withDefaults(func(c *oanregistry.Config) { c.URL = "http://registry:8081/api/v1" }), + expected: withDefaults(func(c *sunbirdRegistry.Config) { c.URL = "http://registry:8081/api/v1" }), }, { name: "reads every supported setting", @@ -54,7 +54,7 @@ func TestParseConfig(t *testing.T) { "retry_wait_min": "200ms", "retry_wait_max": "1s", }, - expected: &oanregistry.Config{ + expected: &sunbirdRegistry.Config{ URL: "http://registry:8081/api/v1", Entity: "Subscriber", ProviderEntity: defaultProviderEntity, @@ -71,7 +71,7 @@ func TestParseConfig(t *testing.T) { "url": "http://registry:8081", "providerEntity": "ProviderCapability", }, - expected: withDefaults(func(c *oanregistry.Config) { + expected: withDefaults(func(c *sunbirdRegistry.Config) { c.URL = "http://registry:8081" c.ProviderEntity = "ProviderCapability" }), @@ -82,7 +82,7 @@ func TestParseConfig(t *testing.T) { "url": "http://registry:8081", "providerEntity": "", }, - expected: withDefaults(func(c *oanregistry.Config) { + expected: withDefaults(func(c *sunbirdRegistry.Config) { c.URL = "http://registry:8081" }), }, @@ -91,18 +91,18 @@ func TestParseConfig(t *testing.T) { // participant keeps verifying. name: "leaves caching disabled when no TTL is set", config: map[string]string{"url": "http://registry:8081"}, - expected: withDefaults(func(c *oanregistry.Config) { c.URL = "http://registry:8081" }), + expected: withDefaults(func(c *sunbirdRegistry.Config) { c.URL = "http://registry:8081" }), }, { // Distinct from "unset", which yields the default of 1. name: "honours an explicit retry_max of zero", config: map[string]string{"url": "http://registry:8081", "retry_max": "0"}, - expected: withDefaults(func(c *oanregistry.Config) { c.URL = "http://registry:8081"; c.RetryMax = 0 }), + expected: withDefaults(func(c *sunbirdRegistry.Config) { c.URL = "http://registry:8081"; c.RetryMax = 0 }), }, { name: "ignores empty values and keeps the defaults", config: map[string]string{"url": "http://registry:8081", "entity": "", "timeout": ""}, - expected: withDefaults(func(c *oanregistry.Config) { c.URL = "http://registry:8081" }), + expected: withDefaults(func(c *sunbirdRegistry.Config) { c.URL = "http://registry:8081" }), }, { name: "rejects a non-numeric timeout", @@ -144,7 +144,7 @@ func TestParseConfig(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - got, err := oanRegistryProvider{}.parseConfig(tc.config) + got, err := sunbirdRegistryProvider{}.parseConfig(tc.config) if tc.expectedErr != "" { if err == nil { @@ -173,7 +173,7 @@ func TestNew(t *testing.T) { t.Parallel() //nolint:staticcheck // deliberately passing a nil context to assert the guard. - _, _, err := oanRegistryProvider{}.New(nil, nil, map[string]string{"url": "http://registry:8081"}) + _, _, err := sunbirdRegistryProvider{}.New(nil, nil, map[string]string{"url": "http://registry:8081"}) if err == nil { t.Fatal("expected an error for a nil context, got none") } @@ -182,7 +182,7 @@ func TestNew(t *testing.T) { t.Run("rejects a missing URL", func(t *testing.T) { t.Parallel() - _, _, err := oanRegistryProvider{}.New(context.Background(), nil, map[string]string{}) + _, _, err := sunbirdRegistryProvider{}.New(context.Background(), nil, map[string]string{}) if err == nil { t.Fatal("expected an error for a missing URL, got none") } @@ -191,7 +191,7 @@ func TestNew(t *testing.T) { t.Run("rejects an unparseable config", func(t *testing.T) { t.Parallel() - _, _, err := oanRegistryProvider{}.New(context.Background(), nil, map[string]string{ + _, _, err := sunbirdRegistryProvider{}.New(context.Background(), nil, map[string]string{ "url": "http://registry:8081", "timeout": "soon", }) @@ -203,7 +203,7 @@ func TestNew(t *testing.T) { t.Run("builds a client from a valid config", func(t *testing.T) { t.Parallel() - client, closer, err := oanRegistryProvider{}.New(context.Background(), nil, map[string]string{ + client, closer, err := sunbirdRegistryProvider{}.New(context.Background(), nil, map[string]string{ "url": "http://registry:8081/api/v1", }) if err != nil { @@ -220,20 +220,20 @@ func TestNew(t *testing.T) { } }) - // Deliberately NOT parallel: this swaps the package-level newOANRegistryFunc, + // Deliberately NOT parallel: this swaps the package-level newSunbirdRegistryFunc, // so running it alongside its parallel siblings would race on that variable. // Go never schedules a non-parallel subtest concurrently with parallel ones, // which is what makes this safe -- do not add t.Parallel() "for consistency". t.Run("propagates a client construction failure", func(t *testing.T) { - original := newOANRegistryFunc - t.Cleanup(func() { newOANRegistryFunc = original }) + original := newSunbirdRegistryFunc + t.Cleanup(func() { newSunbirdRegistryFunc = original }) wantErr := errors.New("boom") - newOANRegistryFunc = func(context.Context, definition.Cache, *oanregistry.Config) (*oanregistry.Client, func() error, error) { + newSunbirdRegistryFunc = func(context.Context, definition.Cache, *sunbirdRegistry.Config) (*sunbirdRegistry.Client, func() error, error) { return nil, nil, wantErr } - _, _, err := oanRegistryProvider{}.New(context.Background(), nil, map[string]string{"url": "http://registry:8081"}) + _, _, err := sunbirdRegistryProvider{}.New(context.Background(), nil, map[string]string{"url": "http://registry:8081"}) if !errors.Is(err, wantErr) { t.Fatalf("expected the underlying error to be propagated, got: %v", err) } diff --git a/pkg/plugin/implementation/oanregistry/providerrecord.go b/pkg/plugin/implementation/sunbirdRegistry/providerrecord.go similarity index 98% rename from pkg/plugin/implementation/oanregistry/providerrecord.go rename to pkg/plugin/implementation/sunbirdRegistry/providerrecord.go index cf3dd5a6..03e621d0 100644 --- a/pkg/plugin/implementation/oanregistry/providerrecord.go +++ b/pkg/plugin/implementation/sunbirdRegistry/providerrecord.go @@ -1,10 +1,10 @@ -package oanregistry +package sunbirdRegistry // providerrecord.go resolves a capability binding into a call plan: what to // call, how to call it, and which mappings translate in and out. // -// This is the second thing the OAN registry is asked for, and it is a different -// question from the signing-key lookup in oanregistry.go. That one asks "who +// This is the second thing the registry is asked for, and it is a different +// question from the signing-key lookup in sunbirdRegistry.go. That one asks "who // sent this", keyed by an inbound Authorization header. This one asks "who do I // call next", keyed by a binding taken from the request body. Different subject, // different cache, different meaning of failure -- so they share transport and @@ -95,7 +95,7 @@ func searchURLFor(baseURL, entity string) string { func (c *Client) ProviderRecord(ctx context.Context, bindingKey string) (*model.ProviderRecord, error) { start := time.Now() tracer := otel.Tracer(telemetry.ScopeName, trace.WithInstrumentationVersion(telemetry.ScopeVersion)) - ctx, span := tracer.Start(ctx, "oan registry provider record") + ctx, span := tracer.Start(ctx, "registry provider record") defer span.End() if bindingKey == "" { @@ -373,7 +373,7 @@ func decodeRecords[T any](body []byte) ([]T, error) { // one cache but have different subjects and lifetimes, and a collision would // serve one as the other. func providerRecordCacheKey(bindingKey string) string { - return "oan_provider_" + bindingKey + return "registry_provider_" + bindingKey } func (c *Client) cachedProviderRecord(ctx context.Context, tracer trace.Tracer, key string) (*model.ProviderRecord, bool) { diff --git a/pkg/plugin/implementation/oanregistry/providerrecord_test.go b/pkg/plugin/implementation/sunbirdRegistry/providerrecord_test.go similarity index 99% rename from pkg/plugin/implementation/oanregistry/providerrecord_test.go rename to pkg/plugin/implementation/sunbirdRegistry/providerrecord_test.go index 2f83315f..aab2c4c8 100644 --- a/pkg/plugin/implementation/oanregistry/providerrecord_test.go +++ b/pkg/plugin/implementation/sunbirdRegistry/providerrecord_test.go @@ -1,4 +1,4 @@ -package oanregistry +package sunbirdRegistry import ( "context" @@ -591,7 +591,7 @@ func TestProviderRecordCacheKeyIsDistinctFromTheKeyLookupCacheKey(t *testing.T) if _, err := resolvePlan(t, client); err != nil { t.Fatalf("ProviderRecord() returned an unexpected error: %v", err) } - if strings.HasPrefix(cache.setKey, "oan_lookup_") { + if strings.HasPrefix(cache.setKey, "registry_lookup_") { t.Errorf("provider plan cache key %q shares the signing-key namespace", cache.setKey) } } diff --git a/pkg/plugin/implementation/oanregistry/oanregistry.go b/pkg/plugin/implementation/sunbirdRegistry/sunbirdRegistry.go similarity index 96% rename from pkg/plugin/implementation/oanregistry/oanregistry.go rename to pkg/plugin/implementation/sunbirdRegistry/sunbirdRegistry.go index 950829cc..0d598eb5 100644 --- a/pkg/plugin/implementation/oanregistry/oanregistry.go +++ b/pkg/plugin/implementation/sunbirdRegistry/sunbirdRegistry.go @@ -1,9 +1,9 @@ -// Package oanregistry resolves participant signing keys from the OAN Registry -// (a SunbirdRC deployment) so inbound Beckn signatures can be verified. +// Package sunbirdRegistry resolves participant signing keys from a SunbirdRC +// registry so inbound Beckn signatures can be verified. // // It implements definition.RegistryLookup only. Onboarding, key publication and // status changes all happen through the registry's own Participant APIs. -package oanregistry +package sunbirdRegistry import ( "context" @@ -129,13 +129,13 @@ func classify(err error) string { } const ( - pluginID = "oanregistry" + pluginID = "sunbirdRegistry" pluginType = "registry" operationLookup = "lookup" operationProviderRecord = "provider_record" ) -// Config holds configuration parameters for the OAN registry client. +// Config holds configuration parameters for the registry client. type Config struct { // URL is the registry base including any API version prefix, // e.g. "http://registry:8081/api/v1". @@ -156,7 +156,7 @@ type Config struct { MaxResponseBytes int64 `yaml:"maxResponseBytes" json:"maxResponseBytes"` } -// Client resolves participants from the OAN registry. It is safe for concurrent +// Client resolves participants from the registry. It is safe for concurrent // use: every field is set once in New and never mutated afterwards. type Client struct { searchURL string @@ -232,23 +232,23 @@ type searchRequest struct { Filters map[string]eqFilter `json:"filters"` } -// validate checks if the provided OAN registry configuration is valid. +// validate checks if the provided registry configuration is valid. func validate(cfg *Config) error { if cfg == nil { - return fmt.Errorf("oan registry config cannot be nil") + return fmt.Errorf("registry config cannot be nil") } if cfg.URL == "" { - return fmt.Errorf("oan registry URL cannot be empty") + return fmt.Errorf("registry URL cannot be empty") } // url.Parse accepts almost anything, so check the parts that actually have // to be there. Catching "registry:8081" (no scheme) at startup is far // cheaper than watching every lookup fail once traffic arrives. parsed, err := url.Parse(cfg.URL) if err != nil { - return fmt.Errorf("invalid oan registry URL %q: %w", cfg.URL, err) + return fmt.Errorf("invalid registry URL %q: %w", cfg.URL, err) } if parsed.Scheme == "" || parsed.Host == "" { - return fmt.Errorf("oan registry URL %q must include a scheme and host, e.g. http://:/api/v1", cfg.URL) + return fmt.Errorf("registry URL %q must include a scheme and host, e.g. http://:/api/v1", cfg.URL) } return nil } @@ -359,7 +359,7 @@ func New(ctx context.Context, cache definition.Cache, cfg *Config) (*Client, fun func (c *Client) Lookup(ctx context.Context, req *model.Subscription) ([]model.Subscription, error) { start := time.Now() tracer := otel.Tracer(telemetry.ScopeName, trace.WithInstrumentationVersion(telemetry.ScopeVersion)) - ctx, span := tracer.Start(ctx, "oan registry lookup") + ctx, span := tracer.Start(ctx, "registry lookup") defer span.End() // M2: an empty key id would match any record whose OSID is absent. Unreachable @@ -371,7 +371,7 @@ func (c *Client) Lookup(ctx context.Context, req *model.Subscription) ([]model.S return nil, nil } - cacheKey := fmt.Sprintf("oan_lookup_%s_%s", req.SubscriberID, req.KeyID) + cacheKey := fmt.Sprintf("registry_lookup_%s_%s", req.SubscriberID, req.KeyID) if cached, ok := c.cached(ctx, tracer, cacheKey); ok { log.Debugf(ctx, "OAN registry lookup cache hit for key: %s", cacheKey) span.SetAttributes(telemetry.AttrErrorType.String(outcomeCacheHit)) @@ -547,7 +547,7 @@ func toSubscription(p participant, k key, status string) model.Subscription { validFrom, _ := parseTime(k.ValidFrom) validUntil, _ := parseTime(k.ValidUntil) - // Domain is absent from the OAN record and so is left unset. Nothing on the + // Domain is absent from the registry record and so is left unset. Nothing on the // signature-validation path reads it. return model.Subscription{ Subscriber: model.Subscriber{ diff --git a/pkg/plugin/implementation/oanregistry/oanregistry_test.go b/pkg/plugin/implementation/sunbirdRegistry/sunbirdRegistry_test.go similarity index 99% rename from pkg/plugin/implementation/oanregistry/oanregistry_test.go rename to pkg/plugin/implementation/sunbirdRegistry/sunbirdRegistry_test.go index bc6f438a..ddbe9711 100644 --- a/pkg/plugin/implementation/oanregistry/oanregistry_test.go +++ b/pkg/plugin/implementation/sunbirdRegistry/sunbirdRegistry_test.go @@ -1,4 +1,4 @@ -package oanregistry +package sunbirdRegistry import ( "context" @@ -144,12 +144,12 @@ func TestValidate(t *testing.T) { { name: "should return error for nil config", config: nil, - expectedErr: "oan registry config cannot be nil", + expectedErr: "registry config cannot be nil", }, { name: "should return error for empty URL", config: &Config{URL: ""}, - expectedErr: "oan registry URL cannot be empty", + expectedErr: "registry URL cannot be empty", }, { name: "should succeed for valid config", @@ -1169,7 +1169,7 @@ func TestLookupCachesUsableResults(t *testing.T) { if cache.setTTL != ttl { t.Errorf("expected the configured TTL %v, got %v", ttl, cache.setTTL) } - if expected := fmt.Sprintf("oan_lookup_%s_%s", testParticipantID, testOSID); cache.setKey != expected { + if expected := fmt.Sprintf("registry_lookup_%s_%s", testParticipantID, testOSID); cache.setKey != expected { t.Errorf("expected cache key %q, got %q", expected, cache.setKey) } @@ -1358,7 +1358,7 @@ func assertOutcomeAttribute(t *testing.T, m metricdata.Metrics, outcome string) } // TestLookupAgainstCapturedRegistryResponse runs the plugin against a verbatim -// response captured from the real OAN registry on 31 Aug 2026, reformatted for +// response captured from a live registry on 31 Aug 2026, reformatted for // readability with field order and values untouched. // // It pins the deployed shape: the data envelope, one flat level with the keys @@ -1482,7 +1482,7 @@ func TestLookupAgainstCapturedRegistryResponse(t *testing.T) { } // TestLookupAgainstCurrentRegistryResponse runs the plugin against a verbatim -// response captured from an OAN registry on 2 Sep 2026, after the Participant +// response captured from a live registry on 2 Sep 2026, after the Participant // schema dropped three things from a published key. // // It pins the shape a registry writes TODAY, and every difference from the From 78d42b9303f691057eb2b94c1d000e4c6271b837 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 18:02:45 +0530 Subject: [PATCH 087/130] docs(config): drop the network name from the reference config's comments [#16] The plugins are generic and their comments should read that way: jsonmapper knows nothing about any provider, and the validator's if/then gap applies to any pack, not to one network's. This commit used to also placeholder the subscriberId. That work is now in the base branch, so what is left here is the comment wording alone, and the message says so rather than claiming a change that is no longer in the diff. --- config/provider-adapter.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config/provider-adapter.yaml b/config/provider-adapter.yaml index ddd3df18..e892811b 100644 --- a/config/provider-adapter.yaml +++ b/config/provider-adapter.yaml @@ -1,4 +1,4 @@ -# OAN provider adapter. +# Provider adapter. # # Serves the Beckn actions synchronously: verifies the sender, resolves the # capability's call plan from the registry, calls the provider, and answers with @@ -177,9 +177,9 @@ modules: # # What it does NOT: if/then/else. The validator library parses those # keywords but never evaluates them, so a pack's conditional rules -- - # in the OAN packs, everything predicated on informationMode -- are - # not checked. Worth knowing before treating a pass here as full - # conformance to a pack. + # in the capability packs, everything predicated on informationMode + # -- are not checked. Worth knowing before treating a pass here as + # full conformance to a pack. # ------------------------------------------------------------------ schemaValidator: id: schemav2validator @@ -217,7 +217,7 @@ modules: # ------------------------------------------------------------------ # jsonmapper -- the JSONata mapper. # - # Generic, and named for what it is rather than for OAN: it knows + # Generic, and named for what it is rather than for a network: it knows # nothing about any provider. It fetches whatever URL the registry's # mappings field names, compiles the JSONata, caches the compiled # form, and runs it in both directions. From 64a1de28da78f91846c06dd1a29599ac53b9fe58 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Tue, 8 Sep 2026 00:55:38 +0530 Subject: [PATCH 088/130] fix(schemav2validator): read only http and https for a payload-chosen schema [#16] The allowlist was consulted once, on the top-level @context. Every document fetched to resolve that document's $refs met no check at all, and freshReadFromURI falls through to os.ReadFile for any scheme but http and https. So a $ref of "file:///etc/passwd" -- or a bare path, which parses with no scheme -- was an instruction from the network to open this container's disk and parse it as a schema. Measured: one capability pack pulls 15 documents, so 14 of the 15 reads were unchecked. A loader on this path now refuses anything but http and https, for the entry document and every $ref under it. The base spec loader keeps the file fallthrough deliberately: its location is operator-configured, where a local file is the point. localSchema mode keeps it for the same reason. This does not restrict which HOSTS a $ref may reach, and that is deliberate rather than missed: the packs $ref two external spec hosts, so enforcing the allowlist on refs needs those named in it as well, or no pack loads at all. TestValidateReferencedObject_AllowsARefToAHostOutsideTheAllowlist pins that, so the wider change cannot be made without noticing. Two existing tests asserted the old behaviour rather than a requirement, and say the new one now; several others used a local temp file as a fixture and either load it in operator mode or serve it over http, which is what production does anyway. --- .../schemav2validator/extended_schema.go | 30 ++ .../schemav2validator/extended_schema_test.go | 385 +++++++++++++----- 2 files changed, 303 insertions(+), 112 deletions(-) diff --git a/pkg/plugin/implementation/schemav2validator/extended_schema.go b/pkg/plugin/implementation/schemav2validator/extended_schema.go index cf9cc91f..2fdcb614 100644 --- a/pkg/plugin/implementation/schemav2validator/extended_schema.go +++ b/pkg/plugin/implementation/schemav2validator/extended_schema.go @@ -323,6 +323,15 @@ func (c *schemaCache) loadSchemaFromPath(ctx context.Context, schemaPath string, loader := newFreshLoader() loader.Context = ctx + if !localSchema { + // The schema location on this path is derived from a payload's + // @context, so every read it causes is network-directed. Installed + // here rather than at the one @context check because that check runs + // once, on the entry document: the $refs inside whatever comes back + // are resolved by the loader and meet no check at all. One pack pulls + // 15 documents across 3 hosts, so this is the majority of the reads. + loader.ReadFromURIFunc = payloadDirectedReader + } var doc *openapi3.T var err error @@ -409,6 +418,27 @@ func (c *schemaCache) loadSchemaFromPath(ctx context.Context, schemaPath string, return doc, nil } +// payloadDirectedReader reads a schema document for a location that a payload +// chose, refusing any scheme but http and https. +// +// freshReadFromURI falls through to os.ReadFile for every other scheme, so +// without this a $ref of "file:///etc/passwd" -- or a bare path, which parses +// with no scheme at all -- is an instruction from the network to read this +// container's disk and parse it as a schema. The base spec loader keeps that +// fallthrough deliberately: its location is operator-configured, where a local +// file is the point. Here it never is. +// +// This does NOT restrict which hosts may be reached; isAllowedDomain still +// guards only the entry @context. Enforcing the allowlist here as well is the +// right shape, but the packs $ref two external spec hosts, so it needs those +// named in the allowlist or no pack loads at all. +func payloadDirectedReader(loader *openapi3.Loader, u *url.URL) ([]byte, error) { + if u.Scheme != "http" && u.Scheme != "https" { + return nil, fmt.Errorf("refusing to read schema from %q: only http and https are read for a location a payload chose", u.String()) + } + return freshReadFromURI(loader, u) +} + // findReferencedObjects recursively finds domain-specific objects with @context. func findReferencedObjects(data interface{}, path string) []referencedObject { var results []referencedObject diff --git a/pkg/plugin/implementation/schemav2validator/extended_schema_test.go b/pkg/plugin/implementation/schemav2validator/extended_schema_test.go index 8f7e378c..39beebff 100644 --- a/pkg/plugin/implementation/schemav2validator/extended_schema_test.go +++ b/pkg/plugin/implementation/schemav2validator/extended_schema_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "reflect" "sort" + "strings" "sync/atomic" "testing" "time" @@ -156,10 +157,10 @@ func TestHashURL(t *testing.T) { t.Run(tt.name, func(t *testing.T) { hash1 := hashURL(tt.url) hash2 := hashURL(tt.url) - + // Same URL should produce same hash assert.Equal(t, hash1, hash2) - + // Hash should be 64 characters (SHA256 hex) assert.Equal(t, 64, len(hash1)) }) @@ -247,23 +248,23 @@ func TestNewSchemaCache(t *testing.T) { func TestSchemaCache_GetSet(t *testing.T) { cache := newSchemaCache(10) - + // Create a simple schema doc doc := &openapi3.T{ OpenAPI: "3.1.0", } - + urlHash := hashURL("https://example.com/schema.yaml") ttl := 1 * time.Hour - + // Test Set cache.set(urlHash, doc, ttl) - + // Test Get - should find it retrieved, found := cache.get(urlHash) assert.True(t, found) assert.Equal(t, doc, retrieved) - + // Test Get - non-existent key _, found = cache.get("non-existent-hash") assert.False(t, found) @@ -271,28 +272,28 @@ func TestSchemaCache_GetSet(t *testing.T) { func TestSchemaCache_LRUEviction(t *testing.T) { cache := newSchemaCache(2) // Small cache for testing - + doc1 := &openapi3.T{OpenAPI: "3.1.0"} doc2 := &openapi3.T{OpenAPI: "3.1.1"} doc3 := &openapi3.T{OpenAPI: "3.1.2"} - + ttl := 1 * time.Hour - + // Add first two items cache.set("hash1", doc1, ttl) cache.set("hash2", doc2, ttl) - + // Access first item to make it more recent cache.get("hash1") - + // Add third item - should evict hash2 (least recently used) cache.set("hash3", doc3, ttl) - + // Verify hash1 and hash3 exist, hash2 was evicted _, found1 := cache.get("hash1") _, found2 := cache.get("hash2") _, found3 := cache.get("hash3") - + assert.True(t, found1, "hash1 should exist (recently accessed)") assert.False(t, found2, "hash2 should be evicted (LRU)") assert.True(t, found3, "hash3 should exist (just added)") @@ -300,20 +301,20 @@ func TestSchemaCache_LRUEviction(t *testing.T) { func TestSchemaCache_TTLExpiry(t *testing.T) { cache := newSchemaCache(10) - + doc := &openapi3.T{OpenAPI: "3.1.0"} urlHash := "test-hash" - + // Set with very short TTL cache.set(urlHash, doc, 1*time.Millisecond) - + // Should be found immediately _, found := cache.get(urlHash) assert.True(t, found) - + // Wait for expiry time.Sleep(10 * time.Millisecond) - + // Should not be found after expiry _, found = cache.get(urlHash) assert.False(t, found) @@ -321,23 +322,23 @@ func TestSchemaCache_TTLExpiry(t *testing.T) { func TestSchemaCache_CleanupExpired(t *testing.T) { cache := newSchemaCache(10) - + doc := &openapi3.T{OpenAPI: "3.1.0"} - + // Add items with short TTL cache.set("hash1", doc, 1*time.Millisecond) cache.set("hash2", doc, 1*time.Millisecond) cache.set("hash3", doc, 1*time.Hour) // This one won't expire - + // Wait for expiry time.Sleep(10 * time.Millisecond) - + // Cleanup expired count := cache.cleanupExpired() - + // Should have cleaned up 2 expired items assert.Equal(t, 2, count) - + // Verify only hash3 remains cache.mu.RLock() assert.Equal(t, 1, len(cache.schemas)) @@ -448,7 +449,7 @@ func TestFindReferencedObjects_PathBuilding(t *testing.T) { } objects := findReferencedObjects(data, "message") - + assert.Equal(t, 1, len(objects)) assert.Equal(t, "message.order.beckn:orderItems[0].beckn:acceptedOffer.beckn:offerAttributes", objects[0].Path) assert.Equal(t, "ChargingOffer", objects[0].Type) @@ -459,11 +460,11 @@ func TestFindReferencedObjects_PathBuilding(t *testing.T) { func TestLoadSchemaFromPath_LocalFile(t *testing.T) { cache := newSchemaCache(10) ctx := context.Background() - + tmpFile, err := os.CreateTemp("", "test-schema-*.yaml") assert.NoError(t, err) defer os.Remove(tmpFile.Name()) - + schemaContent := `openapi: 3.1.0 info: title: Test Schema @@ -475,58 +476,74 @@ components: properties: field1: type: string` - + _, err = tmpFile.Write([]byte(schemaContent)) assert.NoError(t, err) tmpFile.Close() - + + // localSchema=false means the location came from a payload's @context -- + // the only way the production caller passes it. A local file is not + // something the network may ask this process to open, so it is refused. doc, err := cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, false) + if err == nil { + t.Fatal("a payload-directed load opened a local file") + } + assert.Contains(t, err.Error(), "only http and https are read") + assert.Nil(t, doc) + + // localSchema=true is an operator naming a path in the adapter's own + // config, which is the one case where opening a file is the intent. + doc, err = cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, true) assert.NoError(t, err) assert.NotNil(t, doc) assert.Equal(t, "3.1.0", doc.OpenAPI) } func TestLoadSchemaFromPath_CacheHit(t *testing.T) { + // A temp file is just the fixture here, so this loads in operator mode: + // a payload-directed load refuses local files by design. cache := newSchemaCache(10) ctx := context.Background() - + tmpFile, err := os.CreateTemp("", "test-schema-*.yaml") assert.NoError(t, err) defer os.Remove(tmpFile.Name()) - + schemaContent := `openapi: 3.1.0 info: title: Test Schema version: 1.0.0` - + tmpFile.Write([]byte(schemaContent)) tmpFile.Close() - - doc1, err := cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, false) + + doc1, err := cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, true) assert.NoError(t, err) - doc2, err := cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, false) + doc2, err := cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, true) assert.NoError(t, err) - + assert.Equal(t, doc1, doc2) } func TestLoadSchemaFromPath_InvalidPath(t *testing.T) { cache := newSchemaCache(10) ctx := context.Background() - + _, err := cache.loadSchemaFromPath(ctx, "/nonexistent/schema.yaml", 1*time.Hour, 30*time.Second, false) assert.Error(t, err) } func TestFindSchemaByType_DirectMatch(t *testing.T) { + // A temp file is just the fixture here, so this loads in operator mode: + // a payload-directed load refuses local files by design. cache := newSchemaCache(10) ctx := context.Background() - + tmpFile, err := os.CreateTemp("", "test-schema-*.yaml") assert.NoError(t, err) defer os.Remove(tmpFile.Name()) - + schemaContent := `openapi: 3.1.0 info: title: Test Schema @@ -538,11 +555,11 @@ components: properties: field1: type: string` - + tmpFile.Write([]byte(schemaContent)) tmpFile.Close() - - doc, err := cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, false) + + doc, err := cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, true) assert.NoError(t, err) schema, err := findSchemaByType(ctx, doc, "TestType") @@ -551,13 +568,15 @@ components: } func TestFindSchemaByType_NotFound(t *testing.T) { + // A temp file is just the fixture here, so this loads in operator mode: + // a payload-directed load refuses local files by design. cache := newSchemaCache(10) ctx := context.Background() - + tmpFile, err := os.CreateTemp("", "test-schema-*.yaml") assert.NoError(t, err) defer os.Remove(tmpFile.Name()) - + schemaContent := `openapi: 3.1.0 info: title: Test Schema @@ -566,11 +585,11 @@ components: schemas: TestType: type: object` - + tmpFile.Write([]byte(schemaContent)) tmpFile.Close() - - doc, err := cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, false) + + doc, err := cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, true) assert.NoError(t, err) _, err = findSchemaByType(ctx, doc, "NonExistentType") @@ -581,11 +600,7 @@ components: func TestValidateReferencedObject_Valid(t *testing.T) { cache := newSchemaCache(10) ctx := context.Background() - - tmpFile, err := os.CreateTemp("", "test-schema-*.yaml") - assert.NoError(t, err) - defer os.Remove(tmpFile.Name()) - + schemaContent := `openapi: 3.1.0 info: title: Test Schema @@ -603,33 +618,28 @@ components: type: string required: - field1` - - tmpFile.Write([]byte(schemaContent)) - tmpFile.Close() - + + ctxURL := serveTempSchema(t, schemaContent) + obj := referencedObject{ Path: "message.test", - Context: tmpFile.Name(), + Context: ctxURL, Type: "TestType", Data: map[string]interface{}{ - "@context": tmpFile.Name(), + "@context": ctxURL, "@type": "TestType", "field1": "value1", }, } - - err = cache.validateReferencedObject(ctx, obj, 1*time.Hour, 30*time.Second, nil, false) + + err := cache.validateReferencedObject(ctx, obj, 1*time.Hour, 30*time.Second, nil, false) assert.NoError(t, err) } func TestValidateReferencedObject_Invalid(t *testing.T) { cache := newSchemaCache(10) ctx := context.Background() - - tmpFile, err := os.CreateTemp("", "test-schema-*.yaml") - assert.NoError(t, err) - defer os.Remove(tmpFile.Name()) - + schemaContent := `openapi: 3.1.0 info: title: Test Schema @@ -647,21 +657,20 @@ components: type: string required: - field1` - - tmpFile.Write([]byte(schemaContent)) - tmpFile.Close() - + + ctxURL := serveTempSchema(t, schemaContent) + obj := referencedObject{ Path: "message.test", - Context: tmpFile.Name(), + Context: ctxURL, Type: "TestType", Data: map[string]interface{}{ - "@context": tmpFile.Name(), + "@context": ctxURL, "@type": "TestType", }, } - - err = cache.validateReferencedObject(ctx, obj, 1*time.Hour, 30*time.Second, nil, false) + + err := cache.validateReferencedObject(ctx, obj, 1*time.Hour, 30*time.Second, nil, false) assert.Error(t, err) schemaErrors := []model.Error{} @@ -698,10 +707,6 @@ func TestValidateReferencedObject_EntityTypeNotFound(t *testing.T) { cache := newSchemaCache(10) ctx := context.Background() - tmpFile, err := os.CreateTemp("", "test-schema-*.yaml") - assert.NoError(t, err) - defer os.Remove(tmpFile.Name()) - schemaContent := `openapi: 3.1.0 info: title: Test Schema @@ -711,20 +716,19 @@ components: TestType: type: object` - tmpFile.Write([]byte(schemaContent)) - tmpFile.Close() + ctxURL := serveTempSchema(t, schemaContent) obj := referencedObject{ Path: "message.test", - Context: tmpFile.Name(), + Context: ctxURL, Type: "NonExistentType", Data: map[string]interface{}{ - "@context": tmpFile.Name(), + "@context": ctxURL, "@type": "NonExistentType", }, } - err = cache.validateReferencedObject(ctx, obj, 1*time.Hour, 30*time.Second, nil, false) + err := cache.validateReferencedObject(ctx, obj, 1*time.Hour, 30*time.Second, nil, false) assert.Error(t, err) becknErr, ok := err.(*model.Error) @@ -831,7 +835,7 @@ components: - field1` tests := []struct { - name string + name string allowedDomains []string }{ {name: "file scheme allowed when no allowlist (nil)", allowedDomains: nil}, @@ -850,6 +854,8 @@ components: ctx := context.Background() // Use file:// scheme — would be rejected by scheme check if allowlist were set. + // It is still refused, one layer down: the reader takes http and + // https only. What an empty allowlist skips is the HOST check. obj := referencedObject{ Path: "message.test", Context: "file://" + tmpFile.Name(), @@ -858,7 +864,8 @@ components: } err = cache.validateReferencedObject(ctx, obj, 1*time.Hour, 30*time.Second, tt.allowedDomains, false) - // No domain or scheme error — allowlist check was skipped entirely. + // So: no domain error and no @context scheme error, which is what + // an empty allowlist means. Not "anything is readable". if err != nil { assert.NotContains(t, err.Error(), "domain not allowed") assert.NotContains(t, err.Error(), "invalid scheme in @context") @@ -875,14 +882,14 @@ func TestValidateExtendedSchemas_NoObjects(t *testing.T) { }, schemaCache: newSchemaCache(10), } - + ctx := context.Background() body := map[string]interface{}{ "message": map[string]interface{}{ "field": "value", }, } - + err := v.validateExtendedSchemas(ctx, body) assert.NoError(t, err) } @@ -894,12 +901,12 @@ func TestValidateExtendedSchemas_MissingMessage(t *testing.T) { }, schemaCache: newSchemaCache(10), } - + ctx := context.Background() body := map[string]interface{}{ "context": map[string]interface{}{}, } - + err := v.validateExtendedSchemas(ctx, body) assert.Error(t, err) assert.Contains(t, err.Error(), "missing 'message' field") @@ -1055,9 +1062,9 @@ func TestIsSchemaVersionSegment(t *testing.T) { func TestExtractRelativeSchemaPath(t *testing.T) { tests := []struct { - name string + name string rawURL string - want string + want string }{ { name: "URL with /schema/ marker and version", @@ -1275,11 +1282,7 @@ func TestValidateReferencedObject_LocalMissFallsBackToContext(t *testing.T) { cache := newSchemaCache(10) ctx := context.Background() - tmpFile, err := os.CreateTemp("", "test-schema-*.yaml") - assert.NoError(t, err) - defer os.Remove(tmpFile.Name()) - - tmpFile.Write([]byte(`openapi: 3.1.0 + ctxURL := serveTempSchema(t, `openapi: 3.1.0 info: title: Test Schema version: 1.0.0 @@ -1289,22 +1292,21 @@ components: type: object properties: field1: - type: string`)) - tmpFile.Close() + type: string`) obj := referencedObject{ Path: "message.test", - Context: tmpFile.Name(), + Context: ctxURL, Type: "TestType", Data: map[string]interface{}{ - "@context": tmpFile.Name(), + "@context": ctxURL, "@type": "TestType", "field1": "value1", }, } - // rawSchemas empty, localSchema=true — local miss, falls back to @context file path - err = cache.validateReferencedObject(ctx, obj, 1*time.Hour, 30*time.Second, nil, true) + // rawSchemas empty, localSchema=true — local miss, falls back to fetching the @context + err := cache.validateReferencedObject(ctx, obj, 1*time.Hour, 30*time.Second, nil, true) assert.NoError(t, err) } @@ -1409,15 +1411,12 @@ components: type: string const: openagrinet:WeatherObservation` -func writeTempSchema(t *testing.T, content string) string { +// serveTempSchema serves content over http and returns a URL usable as an +// @context. Served rather than written to disk because a payload-directed load +// reads http and https only -- and because fetching is what production does. +func serveTempSchema(t *testing.T, content string) string { t.Helper() - f, err := os.CreateTemp("", "test-schema-*.yaml") - assert.NoError(t, err) - t.Cleanup(func() { os.Remove(f.Name()) }) - _, err = f.Write([]byte(content)) - assert.NoError(t, err) - assert.NoError(t, f.Close()) - return f.Name() + return serveSchema(t, content).URL + "/context.jsonld" } // A pack that requires @type must receive it. This is the case that could not @@ -1426,7 +1425,7 @@ func writeTempSchema(t *testing.T, content string) string { // "@type is required". func TestValidateReferencedObject_PackStyleKeepsAtType(t *testing.T) { cache := newSchemaCache(10) - path := writeTempSchema(t, packStyleSchema) + path := serveTempSchema(t, packStyleSchema) obj := referencedObject{ Path: "message.catalogs[0].resources[0].resourceAttributes", @@ -1448,7 +1447,7 @@ func TestValidateReferencedObject_PackStyleKeepsAtType(t *testing.T) { // stripping it silently skipped. func TestValidateReferencedObject_PackStyleWrongAtTypeRejected(t *testing.T) { cache := newSchemaCache(10) - path := writeTempSchema(t, packStyleSchema) + path := serveTempSchema(t, packStyleSchema) obj := referencedObject{ Path: "message.catalogs[0].resources[0].resourceAttributes", @@ -1570,3 +1569,165 @@ func TestSchemaDeclaresProperty(t *testing.T) { }) } } + +// A payload chooses the @context, so it chooses every document the loader then +// reads to resolve that document's $refs. The allowlist is consulted once, on +// the entry URL; these tests cover what happens after it. + +const entrySchemaRefTemplate = `openapi: 3.1.0 +info: + title: entry + version: "1" +paths: {} +components: + schemas: + TestType: + type: object + properties: + field1: + $ref: "REF_TARGET#/components/schemas/Borrowed" +` + +const borrowedSchema = `openapi: 3.1.0 +info: + title: borrowed + version: "1" +paths: {} +components: + schemas: + Borrowed: + type: string +` + +// serveSchema returns an https-less test server answering every path with body. +func serveSchema(t *testing.T, body string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/yaml") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + return srv +} + +func TestValidateReferencedObject_RefusesARefThatWouldReadTheDisk(t *testing.T) { + // A real file, so a successful read would be indistinguishable from a + // legitimate schema and the test could not tell the two apart. + onDisk := filepath.Join(t.TempDir(), "borrowed.yaml") + if err := os.WriteFile(onDisk, []byte(borrowedSchema), 0o600); err != nil { + t.Fatalf("failed to write the file under test: %v", err) + } + + for _, tt := range []struct { + name string + ref string + }{ + {name: "file scheme", ref: "file://" + onDisk}, + {name: "bare path, which parses with no scheme at all", ref: onDisk}, + } { + t.Run(tt.name, func(t *testing.T) { + entry := serveSchema(t, strings.Replace(entrySchemaRefTemplate, "REF_TARGET", tt.ref, 1)) + host, err := url.Parse(entry.URL) + if err != nil { + t.Fatalf("failed to parse the test server URL: %v", err) + } + + cache := newSchemaCache(10) + obj := referencedObject{ + Path: "message.test", + Context: entry.URL + "/context.jsonld", + Type: "TestType", + Data: map[string]interface{}{"field1": "value1"}, + } + + err = cache.validateReferencedObject(context.Background(), obj, + 1*time.Hour, 30*time.Second, []string{host.Host}, false) + + // The entry document is allowlisted and https, so nothing before + // the $ref refuses this. Only the reader can. + if err == nil { + t.Fatal("the $ref was read, so a payload can name any file on disk") + } + assert.Contains(t, err.Error(), "refusing to read schema from") + }) + } +} + +// The packs pull 15 documents across 3 hosts -- the one the allowlist names +// plus two external spec hosts the packs $ref into -- so a $ref to a host +// outside the allowlist is the normal case, not the attack. This pins that: +// applying isAllowedDomain to $refs as well would need all three hosts named +// in the allowlist first, and would otherwise stop every pack loading. +// Deliberate, not missed. +func TestValidateReferencedObject_AllowsARefToAHostOutsideTheAllowlist(t *testing.T) { + borrowed := serveSchema(t, borrowedSchema) + entry := serveSchema(t, strings.Replace(entrySchemaRefTemplate, "REF_TARGET", borrowed.URL+"/borrowed.yaml", 1)) + + entryHost, err := url.Parse(entry.URL) + if err != nil { + t.Fatalf("failed to parse the test server URL: %v", err) + } + borrowedHost, err := url.Parse(borrowed.URL) + if err != nil { + t.Fatalf("failed to parse the test server URL: %v", err) + } + if entryHost.Port() == borrowedHost.Port() { + t.Fatal("the two servers must differ, or this proves nothing") + } + + cache := newSchemaCache(10) + obj := referencedObject{ + Path: "message.test", + Context: entry.URL + "/context.jsonld", + Type: "TestType", + Data: map[string]interface{}{"field1": "value1"}, + } + + // Only the entry host is allowlisted; the $ref host is not. + if err := cache.validateReferencedObject(context.Background(), obj, + 1*time.Hour, 30*time.Second, []string{entryHost.Host}, false); err != nil { + t.Fatalf("a cross-host $ref must still resolve, or no pack can load: %v", err) + } +} + +func TestPayloadDirectedReader(t *testing.T) { + onDisk := filepath.Join(t.TempDir(), "schema.yaml") + if err := os.WriteFile(onDisk, []byte(borrowedSchema), 0o600); err != nil { + t.Fatalf("failed to write the file under test: %v", err) + } + + for _, tt := range []struct { + name string + raw string + refused bool + }{ + {name: "file scheme", raw: "file://" + onDisk, refused: true}, + {name: "bare path", raw: onDisk, refused: true}, + {name: "a scheme nobody serves schemas over", raw: "gopher://example.test/schema.yaml", refused: true}, + {name: "http is read", raw: "", refused: false}, + } { + t.Run(tt.name, func(t *testing.T) { + raw := tt.raw + if raw == "" { + raw = serveSchema(t, borrowedSchema).URL + "/schema.yaml" + } + u, err := url.Parse(raw) + if err != nil { + t.Fatalf("failed to parse %q: %v", raw, err) + } + + data, err := payloadDirectedReader(openapi3.NewLoader(), u) + if tt.refused { + if err == nil { + t.Fatalf("%q was read, and must not have been", raw) + } + assert.Contains(t, err.Error(), "only http and https are read") + // The point is that nothing was read, not merely that it errored. + assert.Empty(t, data) + return + } + assert.NoError(t, err) + assert.Contains(t, string(data), "Borrowed") + }) + } +} From 9a1594f32c1c45f99eeda9c7daf78332cabbfd28 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Tue, 8 Sep 2026 00:59:23 +0530 Subject: [PATCH 089/130] fix(schemav2validator): read the list form of @type, and reject what it cannot read [#16] findReferencedObjects required a string for both JSON-LD keys, so an object carrying the list form matched neither assertion, was never appended, and its resourceAttributes went to no schema at all -- while the extended layer reported a pass over an object it had not looked at. The list form is not exotic. The packs declare @type as a oneOf whose second branch is a list carrying the canonical OAN type alongside provider-defined ones, so a conforming payload could use it and be skipped. Which entry names the capability is not fixed either, so the document now decides: the first @type it declares a schema for wins, rather than the payload's ordering. @context takes the list form too, using the first string in it, because only a URL locates a schema and an inline object names no document to fetch. Skipping is replaced by rejection wherever the object claims a type this validator cannot read. @type ABSENT is left alone and still passes over: an object with a context and no type makes no claim about which schema applies, and there is nothing to validate it against. Also replaces the @type const test. It built obj.Type and Data["@type"] disagreeing, which the real path cannot produce -- both are read off one map -- so it demonstrated nothing about a payload. Tests now run through findReferencedObjects, and the const does have a payload-level case: the list branch forbids a second openagrinet: type, which is caught only because @type is kept in the data rather than stripped. --- .../schemav2validator/extended_schema.go | 149 +++++++++-- .../schemav2validator/extended_schema_test.go | 248 ++++++++++++++++-- 2 files changed, 365 insertions(+), 32 deletions(-) diff --git a/pkg/plugin/implementation/schemav2validator/extended_schema.go b/pkg/plugin/implementation/schemav2validator/extended_schema.go index 2fdcb614..8c71a5ea 100644 --- a/pkg/plugin/implementation/schemav2validator/extended_schema.go +++ b/pkg/plugin/implementation/schemav2validator/extended_schema.go @@ -79,7 +79,29 @@ type referencedObject struct { Path string Context string Type string - Data map[string]interface{} + // Types is every @type the object carries, in payload order. JSON-LD + // permits the array form and the packs allow it explicitly, so which + // entry names the capability is not known until a document is consulted. + Types []string + Data map[string]interface{} + // Unusable is set when the object claims a domain type but carries it in + // a shape this validator cannot resolve. It is a coded error rather than + // a bool because the object is then rejected, not skipped: a layer whose + // purpose is to turn a missing attribute into a rejection must not answer + // "valid" for an object it never looked at. + Unusable error +} + +// candidateTypes returns the @type values to try, tolerating an object built +// with Type alone -- which every caller outside findReferencedObjects does. +func (o referencedObject) candidateTypes() []string { + if len(o.Types) > 0 { + return o.Types + } + if o.Type == "" { + return nil + } + return []string{o.Type} } // schemaCache caches loaded domain schemas with LRU eviction. @@ -445,16 +467,29 @@ func findReferencedObjects(data interface{}, path string) []referencedObject { switch v := data.(type) { case map[string]interface{}: - // Check for @context and @type - if contextVal, hasContext := v["@context"].(string); hasContext { - if typeVal, hasType := v["@type"].(string); hasType { - results = append(results, referencedObject{ - Path: path, - Context: contextVal, - Type: typeVal, - Data: v, - }) + // @type ABSENT is not the same as @type unreadable. An object with a + // context and no type makes no claim about which schema applies, and + // there is nothing to validate it against, so it is passed over as + // before. An object that does claim a type is validated or rejected. + rawContext, hasContext := v["@context"] + rawType, hasType := v["@type"] + if hasContext && hasType { + obj := referencedObject{Path: path, Data: v} + contextVal, contextOK := jsonLDLocation(rawContext) + types, typesOK := jsonLDTypes(rawType) + switch { + case !contextOK: + obj.Unusable = model.NewCodedError("SCH_INVALID_JSONLD_CONTEXT", + "@context is not a URL this validator can resolve a schema from") + case !typesOK: + obj.Unusable = model.NewCodedError("SCH_INVALID_ENTITY_TYPE", + "@type is present but is not a type name or a list of them") + default: + obj.Context = contextVal + obj.Type = types[0] + obj.Types = types } + results = append(results, obj) } // Recurse into nested objects @@ -483,6 +518,70 @@ func transformContextToSchemaURL(contextURL string) string { return strings.Replace(contextURL, "context.jsonld", "attributes.yaml", 1) } +// jsonLDLocation returns the @context entry a schema can be located from. +// +// JSON-LD allows a string, an array mixing strings and inline objects, or a +// single inline object. Only a URL locates a schema, so the first string is +// taken and an inline object yields nothing -- there is no document to fetch. +func jsonLDLocation(raw interface{}) (string, bool) { + switch v := raw.(type) { + case string: + if v != "" { + return v, true + } + case []interface{}: + for _, entry := range v { + if s, ok := entry.(string); ok && s != "" { + return s, true + } + } + } + return "", false +} + +// jsonLDTypes returns every @type the object carries, in payload order. +// +// The array form is not exotic: the packs declare @type as a oneOf whose +// second branch is an array containing the canonical OAN type plus +// provider-defined ones. Reading only the string form left those objects +// matching nothing, so they were dropped before validation and the layer +// reported a pass over an object it had not looked at. +func jsonLDTypes(raw interface{}) ([]string, bool) { + switch v := raw.(type) { + case string: + if v != "" { + return []string{v}, true + } + case []interface{}: + types := make([]string, 0, len(v)) + for _, entry := range v { + if s, ok := entry.(string); ok && s != "" { + types = append(types, s) + } + } + if len(types) > 0 { + return types, true + } + } + return nil, false +} + +// findSchemaForAnyType resolves the first @type the document declares a schema +// for, and returns which one matched. With the array form the capability type +// sits among provider-defined ones and its position is not fixed, so the +// document decides rather than the payload's ordering. +func findSchemaForAnyType(ctx context.Context, doc *openapi3.T, types []string) (*openapi3.SchemaRef, string, error) { + var lastErr error + for _, typeName := range types { + schema, err := findSchemaByType(ctx, doc, typeName) + if err == nil { + return schema, typeName, nil + } + lastErr = err + } + return nil, "", lastErr +} + // findSchemaByType finds a schema in the document by @type value. func findSchemaByType(ctx context.Context, doc *openapi3.T, typeName string) (*openapi3.SchemaRef, error) { if doc.Components == nil || doc.Components.Schemas == nil { @@ -600,18 +699,31 @@ func (c *schemaCache) validateReferencedObject( allowedDomains []string, localSchema bool, ) error { + // An object that claims a domain type in a shape we cannot resolve is + // rejected here rather than dropped in findReferencedObjects. Dropping it + // meant the extended layer reported a pass over an object it never + // validated, which is the one outcome this layer exists to prevent. + if obj.Unusable != nil { + log.Warnf(ctx, "refusing an object at %s that carries @context in an unusable shape: %v", obj.Path, obj.Unusable) + return obj.Unusable + } + var doc *openapi3.T if localSchema { - typeName := obj.Type - if idx := strings.LastIndex(typeName, ":"); idx >= 0 { - typeName = typeName[idx+1:] - } - if typeName != "" && !strings.ContainsAny(typeName, "/\\") { + for _, candidate := range obj.candidateTypes() { + typeName := candidate + if idx := strings.LastIndex(typeName, ":"); idx >= 0 { + typeName = typeName[idx+1:] + } + if typeName == "" || strings.ContainsAny(typeName, "/\\") { + continue + } if localDoc, localErr := c.loadSchemaFromPath(ctx, typeName+"/attributes.yaml", ttl, timeout, localSchema); localErr != nil { - log.Debugf(ctx, "local @type lookup failed for %s: %v", obj.Type, localErr) + log.Debugf(ctx, "local @type lookup failed for %s: %v", candidate, localErr) } else { doc = localDoc + break } } } @@ -640,7 +752,10 @@ func (c *schemaCache) validateReferencedObject( } // Find schema by @type - schema, err := findSchemaByType(ctx, doc, obj.Type) + schema, matched, err := findSchemaForAnyType(ctx, doc, obj.candidateTypes()) + if err == nil && matched != obj.Type { + log.Debugf(ctx, "resolved @type %s from the array at %s", matched, obj.Path) + } if err != nil { log.Errorf(ctx, err, "Schema not found for @type: %s at path: %s", obj.Type, obj.Path) return model.NewCodedErrorWithCause("SCH_INVALID_ENTITY_TYPE", err.Error(), obj.Path, err) diff --git a/pkg/plugin/implementation/schemav2validator/extended_schema_test.go b/pkg/plugin/implementation/schemav2validator/extended_schema_test.go index 39beebff..8a6f5fae 100644 --- a/pkg/plugin/implementation/schemav2validator/extended_schema_test.go +++ b/pkg/plugin/implementation/schemav2validator/extended_schema_test.go @@ -1442,25 +1442,243 @@ func TestValidateReferencedObject_PackStyleKeepsAtType(t *testing.T) { assert.NoError(t, err) } -// The same pack must still reject a payload whose @type is not the one the -// capability declares -- keeping the key means its const is now checked, which -// stripping it silently skipped. -func TestValidateReferencedObject_PackStyleWrongAtTypeRejected(t *testing.T) { - cache := newSchemaCache(10) - path := serveTempSchema(t, packStyleSchema) +// packStyleTypeListSchema mirrors how the packs really declare @type: a oneOf +// whose first branch is the canonical string and whose second is a list +// carrying that type alongside provider-defined ones, which must not take the +// openagrinet: prefix. +const packStyleTypeListSchema = `openapi: 3.1.0 +info: + title: Pack Style With Type List + version: 1.0.0 +components: + schemas: + WeatherObservation: + type: object + x-jsonld: + "@context": https://schemas.example.org/schema/WeatherObservation/v0.1/context.jsonld + "@type": openagrinet:WeatherObservation + allOf: + - type: object + required: + - informationMode + properties: + informationMode: + type: string + enum: [OnDemand, Direct] + - type: object + required: + - "@type" + properties: + "@type": + oneOf: + - type: string + const: openagrinet:WeatherObservation + - type: array + minItems: 2 + uniqueItems: true + contains: + const: openagrinet:WeatherObservation + items: + oneOf: + - const: openagrinet:WeatherObservation + - type: string + minLength: 1 + not: + pattern: "^openagrinet:"` + +// resourceBody wraps resourceAttributes the way a payload carries them, so +// discovery runs over the same shape production sees. +func resourceBody(ctxURL string, atType interface{}, informationMode string) map[string]interface{} { + attrs := map[string]interface{}{"@context": ctxURL, "@type": atType} + if informationMode != "" { + attrs["informationMode"] = informationMode + } + return map[string]interface{}{ + "message": map[string]interface{}{ + "catalogs": []interface{}{ + map[string]interface{}{"resources": []interface{}{ + map[string]interface{}{"resourceAttributes": attrs}, + }}, + }, + }, + } +} - obj := referencedObject{ - Path: "message.catalogs[0].resources[0].resourceAttributes", - Context: path, - Type: "openagrinet:WeatherObservation", - Data: map[string]interface{}{ - "@type": "openagrinet:MandiPrice", - "informationMode": "OnDemand", +// theObjectIn runs the production discovery over a body and returns the single +// domain object in it. Tests go through this rather than building a +// referencedObject by hand: Context, Type and Data all come off one map there, +// so a hand-built object can assert a state the real path cannot produce. +func theObjectIn(t *testing.T, body map[string]interface{}) referencedObject { + t.Helper() + objects := findReferencedObjects(body["message"], "message") + if len(objects) != 1 { + t.Fatalf("expected exactly one domain object from discovery, got %d", len(objects)) + } + return objects[0] +} + +// The pack allows @type to be a list, and reading only the string form meant +// such an object matched nothing, was dropped before validation, and the layer +// reported a pass over a payload it had not looked at. +func TestValidateReferencedObject_AcceptsAndChecksATypeList(t *testing.T) { + ctxURL := serveTempSchema(t, packStyleTypeListSchema) + const canonical = "openagrinet:WeatherObservation" + + for _, tt := range []struct { + name string + atType interface{} + wantErr bool + wantErrHas string + }{ + { + name: "the canonical type alone, as a string", + atType: canonical, + }, + { + name: "the canonical type beside a provider type", + atType: []interface{}{canonical, "vendor:GriddedForecast"}, + }, + { + name: "provider type first -- the document decides which entry names the capability", + atType: []interface{}{"vendor:GriddedForecast", canonical}, + }, + { + // A real payload-level rejection, and one that only bites because + // @type is kept in the data rather than stripped: the list branch + // forbids a second openagrinet: type. + name: "a second openagrinet type, which the pack forbids", + atType: []interface{}{canonical, "openagrinet:MandiPrice"}, + wantErr: true, }, + { + // The string branch does not match a list and the list branch + // requires two entries, so neither is satisfied. + name: "a single-entry list, which satisfies neither branch", + atType: []interface{}{canonical}, + wantErr: true, + }, + { + name: "a list naming no type the document declares", + atType: []interface{}{"vendor:One", "vendor:Two"}, + wantErr: true, + wantErrHas: "no schema found", + }, + { + name: "@type present but not a type name", + atType: 42, + wantErr: true, + wantErrHas: "not a type name", + }, + { + name: "@type an empty list", + atType: []interface{}{}, + wantErr: true, + wantErrHas: "not a type name", + }, + } { + t.Run(tt.name, func(t *testing.T) { + obj := theObjectIn(t, resourceBody(ctxURL, tt.atType, "OnDemand")) + err := newSchemaCache(10).validateReferencedObject( + context.Background(), obj, 1*time.Hour, 30*time.Second, nil, false) + + if !tt.wantErr { + assert.NoError(t, err) + return + } + if err == nil { + t.Fatal("expected a rejection; a skipped object is reported as valid") + } + if tt.wantErrHas != "" { + assert.Contains(t, err.Error(), tt.wantErrHas) + } + }) } +} - err := cache.validateReferencedObject(context.Background(), obj, 1*time.Hour, 30*time.Second, nil, false) - assert.Error(t, err) +// An object claiming a type this validator cannot read must be rejected, not +// passed over. Skipping is what let unvalidated resourceAttributes through. +func TestFindReferencedObjects_TypeShapes(t *testing.T) { + const ctxURL = "https://schemas.example.org/schema/WeatherObservation/v0.1/context.jsonld" + + for _, tt := range []struct { + name string + attrs map[string]interface{} + wantFound bool + wantTypes []string + wantCode string + }{ + { + name: "string @type", + attrs: map[string]interface{}{"@context": ctxURL, "@type": "openagrinet:WeatherObservation"}, + wantFound: true, + wantTypes: []string{"openagrinet:WeatherObservation"}, + }, + { + name: "list @type keeps every entry, in payload order", + attrs: map[string]interface{}{"@context": ctxURL, "@type": []interface{}{"a", "b"}}, + wantFound: true, + wantTypes: []string{"a", "b"}, + }, + { + name: "list @context takes the first string, since only a URL locates a schema", + attrs: map[string]interface{}{"@context": []interface{}{ctxURL, map[string]interface{}{"inline": "term"}}, "@type": "T"}, + wantFound: true, + wantTypes: []string{"T"}, + }, + { + name: "inline-object @context names no document to fetch", + attrs: map[string]interface{}{"@context": map[string]interface{}{"inline": "term"}, "@type": "T"}, + wantFound: true, + wantCode: "SCH_INVALID_JSONLD_CONTEXT", + }, + { + name: "@type a number", + attrs: map[string]interface{}{"@context": ctxURL, "@type": 42}, + wantFound: true, + wantCode: "SCH_INVALID_ENTITY_TYPE", + }, + { + // No claim about which schema applies, so there is nothing to + // validate against. Passed over, as before. + name: "@context with no @type at all", + attrs: map[string]interface{}{"@context": ctxURL, "field": "value"}, + wantFound: false, + }, + } { + t.Run(tt.name, func(t *testing.T) { + objects := findReferencedObjects(map[string]interface{}{"resourceAttributes": tt.attrs}, "message") + if !tt.wantFound { + assert.Empty(t, objects) + return + } + if len(objects) != 1 { + t.Fatalf("expected one object, got %d", len(objects)) + } + obj := objects[0] + + if tt.wantCode != "" { + if obj.Unusable == nil { + t.Fatal("expected the object to be marked unusable, so it is rejected rather than skipped") + } + becknErr, ok := obj.Unusable.(*model.Error) + if !ok { + t.Fatalf("Unusable = %T, want *model.Error", obj.Unusable) + } + assert.Equal(t, tt.wantCode, becknErr.Code) + + // and it must actually reject when validated + err := newSchemaCache(10).validateReferencedObject( + context.Background(), obj, 1*time.Hour, 30*time.Second, nil, false) + assert.Error(t, err) + return + } + + assert.Nil(t, obj.Unusable) + assert.Equal(t, tt.wantTypes, obj.Types) + assert.Equal(t, tt.wantTypes[0], obj.Type) + assert.Equal(t, ctxURL, obj.Context) + }) + } } func TestStripUnaccountedJSONLDKeys(t *testing.T) { From 5132832bf89eea3ab11a38887898d0b84b8df40e Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Tue, 8 Sep 2026 17:55:22 +0530 Subject: [PATCH 090/130] fix(schemav2validator): enforce the host allowlist on every schema read [#16] The allowlist guarded the entry @context only. The document that came back was then trusted to name anything: its $refs resolved through the loader with no host check, so a payload could name an attacker's document on the allowed host and have this process fetch whatever that document pointed at -- an internal service, a cloud metadata endpoint. Server-side request forgery, driven by an unauthenticated field. The reach was the argument for fixing it rather than documenting it: loading one capability pack pulls 13-16 documents, of which exactly one is the entry. The refs were never the corner case, they were the traffic. The same allowlist is now checked on every read. Two refusals with different scopes, because they answer different questions: scheme http/https only, and only on the payload-directed path. An operator who set extendedSchema_localSchemaPath is asking for files to be read, so the refusal must not apply to them -- installing it on both branches broke exactly that, which TestLoadSchemaFromPath_LocalFile caught. host on BOTH paths. localSchema falls back to the network for a ref it does not hold locally, so it had the same exposure by a longer route. THE ALLOWLIST HAD TO GROW, and this is a code+config pair that cannot land half-applied. Measured, not assumed: every pack -- WeatherObservation, MandiPrice, KnowledgeResource -- touches raw.githubusercontent.com, schema.beckn.io and schema.nfh.global. With the previous single host and refs checked, the real packs fail with SCH_SCHEMA_ADAPTATION_FAILED on every payload; with the three named, they load and validate. Both directions are verified against the published packs. The test that pinned the old behaviour is inverted rather than deleted: it now asserts a cross-host $ref is refused, and a second test asserts the chain loads once both hosts are named -- which is the case the packs need. What this does NOT fix: raw.githubusercontent.com is world-writable, so the allowlist still trusts every GitHub account for schema content. Narrowing to a path prefix, or mirroring the packs on a host we control, is the real fix and is not a one-line change. Said so in the config. --- config/provider-adapter.yaml | 26 +++++- .../schemav2validator/extended_schema.go | 85 ++++++++++++------- .../schemav2validator/extended_schema_test.go | 84 +++++++++++++----- 3 files changed, 141 insertions(+), 54 deletions(-) diff --git a/config/provider-adapter.yaml b/config/provider-adapter.yaml index e892811b..b2329ab0 100644 --- a/config/provider-adapter.yaml +++ b/config/provider-adapter.yaml @@ -207,8 +207,30 @@ modules: # the first payload after a restart pays for it. A fetch that # FAILS rejects the payload -- it does not skip validation, which # is the right way round, but it does mean this adapter needs - # egress to the host allowed below. - extendedSchema_allowedDomains: "raw.githubusercontent.com" + # egress to every host allowed below. + # + # THIS IS THE WHOLE TRUST BOUNDARY, and it is checked on every + # read: the entry @context and every $ref under it. That matters + # because the document a payload names is NOT trusted -- it comes + # from a URL the payload chose, on a host anyone can publish to -- + # so without the check its $refs could send this process at an + # internal service or a cloud metadata endpoint. + # + # ALL THREE HOSTS ARE REQUIRED. Loading one capability pack pulls + # 13-16 documents across exactly these three (measured, not + # assumed): the pack itself and AgricultureResource from the raw + # CDN, then Descriptor/GeoJSONGeometry/Location/Address from + # schema.beckn.io, which in turn $ref schema.nfh.global. Remove + # any one and no pack loads at all -- the failure is + # SCH_SCHEMA_ADAPTATION_FAILED on every payload, not a partial + # validation. + # + # Keep it as tight as the packs allow. raw.githubusercontent.com + # is world-writable, so this trusts every GitHub account for + # schema content; narrowing it to a path prefix, or mirroring the + # packs on a host we control, is the real fix and is not a + # one-line change. + extendedSchema_allowedDomains: "raw.githubusercontent.com,schema.beckn.io,schema.nfh.global" extendedSchema_cacheTTL: "86400" # 24h extendedSchema_maxCacheSize: "100" diff --git a/pkg/plugin/implementation/schemav2validator/extended_schema.go b/pkg/plugin/implementation/schemav2validator/extended_schema.go index 8c71a5ea..1c3d7169 100644 --- a/pkg/plugin/implementation/schemav2validator/extended_schema.go +++ b/pkg/plugin/implementation/schemav2validator/extended_schema.go @@ -335,7 +335,7 @@ func (c *schemaCache) cleanupExpired() int { return len(expired) } -func (c *schemaCache) loadSchemaFromPath(ctx context.Context, schemaPath string, ttl, timeout time.Duration, localSchema bool) (*openapi3.T, error) { +func (c *schemaCache) loadSchemaFromPath(ctx context.Context, schemaPath string, ttl, timeout time.Duration, allowedDomains []string, localSchema bool) (*openapi3.T, error) { urlHash := hashURL(schemaPath) u, parseErr := url.Parse(schemaPath) @@ -345,15 +345,18 @@ func (c *schemaCache) loadSchemaFromPath(ctx context.Context, schemaPath string, loader := newFreshLoader() loader.Context = ctx - if !localSchema { - // The schema location on this path is derived from a payload's - // @context, so every read it causes is network-directed. Installed - // here rather than at the one @context check because that check runs - // once, on the entry document: the $refs inside whatever comes back - // are resolved by the loader and meet no check at all. One pack pulls - // 15 documents across 3 hosts, so this is the majority of the reads. - loader.ReadFromURIFunc = payloadDirectedReader - } + // Installed on BOTH branches, with the local-file allowance following + // localSchema. The check at the one @context runs once, on the entry + // document; the $refs inside whatever comes back are resolved by the + // loader, and a pack pulls 13-16 documents, so the refs are the great + // majority of the reads. + // + // localSchema is included because its rawSchemas path falls back to the + // NETWORK for a ref it does not hold -- so a local document could reach an + // arbitrary host, the same exposure by a longer route. What it keeps is + // the file read itself, which in that mode is the operator's stated + // intent rather than something a payload asked for. + loader.ReadFromURIFunc = payloadDirectedReader(allowedDomains, localSchema) var doc *openapi3.T var err error @@ -440,25 +443,49 @@ func (c *schemaCache) loadSchemaFromPath(ctx context.Context, schemaPath string, return doc, nil } -// payloadDirectedReader reads a schema document for a location that a payload -// chose, refusing any scheme but http and https. +// payloadDirectedReader returns a reader for schema documents whose location a +// payload chose, enforcing the allowlist on EVERY read -- the entry document +// and every $ref under it. +// +// Two separate refusals, for two separate reasons. +// +// SCHEME: freshReadFromURI falls through to os.ReadFile for every scheme but +// http and https, so a $ref of "file:///etc/passwd" -- or a bare path, which +// parses with no scheme at all -- is an instruction from the network to read +// this container's disk and parse it as a schema. The base spec loader keeps +// that fallthrough deliberately: its location is operator-configured, where a +// local file is the point. Here it never is. +// +// HOST: the entry @context is allowlisted, but the document it returns is not +// trusted -- it came from a payload-named URL on a host anyone can publish to. +// Its $refs used to reach any http host at all, so a payload could name an +// attacker's document and have this process fetch whatever that document +// pointed at: an internal service, a cloud metadata endpoint. Checking the +// same allowlist on every read closes that, and makes the allowlist mean what +// it says -- the hosts this deployment will read schemas from, not the hosts +// it will read the FIRST schema from. // -// freshReadFromURI falls through to os.ReadFile for every other scheme, so -// without this a $ref of "file:///etc/passwd" -- or a bare path, which parses -// with no scheme at all -- is an instruction from the network to read this -// container's disk and parse it as a schema. The base spec loader keeps that -// fallthrough deliberately: its location is operator-configured, where a local -// file is the point. Here it never is. +// This is why the allowlist cannot be a single host: loading one capability +// pack touches raw.githubusercontent.com, schema.beckn.io and +// schema.nfh.global (13-16 reads, measured), so all three have to be named or +// no pack loads at all. An empty allowlist still means "unset, do not check", +// as it does at the @context. // -// This does NOT restrict which hosts may be reached; isAllowedDomain still -// guards only the entry @context. Enforcing the allowlist here as well is the -// right shape, but the packs $ref two external spec hosts, so it needs those -// named in the allowlist or no pack loads at all. -func payloadDirectedReader(loader *openapi3.Loader, u *url.URL) ([]byte, error) { - if u.Scheme != "http" && u.Scheme != "https" { - return nil, fmt.Errorf("refusing to read schema from %q: only http and https are read for a location a payload chose", u.String()) - } - return freshReadFromURI(loader, u) +// allowLocal follows localSchema: an operator who configured +// extendedSchema_localSchemaPath is asking for files to be read, so the scheme +// refusal does not apply to them. The HOST check still does, because that +// mode falls back to the network for a ref it does not hold locally. +func payloadDirectedReader(allowedDomains []string, allowLocal bool) func(*openapi3.Loader, *url.URL) ([]byte, error) { + return func(loader *openapi3.Loader, u *url.URL) ([]byte, error) { + remote := u.Scheme == "http" || u.Scheme == "https" + if !remote && !allowLocal { + return nil, fmt.Errorf("refusing to read schema from %q: only http and https are read for a location a payload chose", u.String()) + } + if remote && len(allowedDomains) > 0 && !isAllowedDomain(u, allowedDomains) { + return nil, fmt.Errorf("refusing to read schema from %q: host is not in extendedSchema_allowedDomains", u.String()) + } + return freshReadFromURI(loader, u) + } } // findReferencedObjects recursively finds domain-specific objects with @context. @@ -719,7 +746,7 @@ func (c *schemaCache) validateReferencedObject( if typeName == "" || strings.ContainsAny(typeName, "/\\") { continue } - if localDoc, localErr := c.loadSchemaFromPath(ctx, typeName+"/attributes.yaml", ttl, timeout, localSchema); localErr != nil { + if localDoc, localErr := c.loadSchemaFromPath(ctx, typeName+"/attributes.yaml", ttl, timeout, allowedDomains, localSchema); localErr != nil { log.Debugf(ctx, "local @type lookup failed for %s: %v", candidate, localErr) } else { doc = localDoc @@ -745,7 +772,7 @@ func (c *schemaCache) validateReferencedObject( schemaPath := transformContextToSchemaURL(obj.Context) log.Debugf(ctx, "Transformed %s -> %s (localSchema=%v)", obj.Context, schemaPath, localSchema) var err error - doc, err = c.loadSchemaFromPath(ctx, schemaPath, ttl, timeout, false) + doc, err = c.loadSchemaFromPath(ctx, schemaPath, ttl, timeout, allowedDomains, false) if err != nil { return model.NewCodedErrorWithCause("SCH_SCHEMA_ADAPTATION_FAILED", err.Error(), obj.Path, err) } diff --git a/pkg/plugin/implementation/schemav2validator/extended_schema_test.go b/pkg/plugin/implementation/schemav2validator/extended_schema_test.go index 8a6f5fae..9facca44 100644 --- a/pkg/plugin/implementation/schemav2validator/extended_schema_test.go +++ b/pkg/plugin/implementation/schemav2validator/extended_schema_test.go @@ -484,7 +484,7 @@ components: // localSchema=false means the location came from a payload's @context -- // the only way the production caller passes it. A local file is not // something the network may ask this process to open, so it is refused. - doc, err := cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, false) + doc, err := cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, nil, false) if err == nil { t.Fatal("a payload-directed load opened a local file") } @@ -493,7 +493,7 @@ components: // localSchema=true is an operator naming a path in the adapter's own // config, which is the one case where opening a file is the intent. - doc, err = cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, true) + doc, err = cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, nil, true) assert.NoError(t, err) assert.NotNil(t, doc) assert.Equal(t, "3.1.0", doc.OpenAPI) @@ -517,10 +517,10 @@ info: tmpFile.Write([]byte(schemaContent)) tmpFile.Close() - doc1, err := cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, true) + doc1, err := cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, nil, true) assert.NoError(t, err) - doc2, err := cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, true) + doc2, err := cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, nil, true) assert.NoError(t, err) assert.Equal(t, doc1, doc2) @@ -530,7 +530,7 @@ func TestLoadSchemaFromPath_InvalidPath(t *testing.T) { cache := newSchemaCache(10) ctx := context.Background() - _, err := cache.loadSchemaFromPath(ctx, "/nonexistent/schema.yaml", 1*time.Hour, 30*time.Second, false) + _, err := cache.loadSchemaFromPath(ctx, "/nonexistent/schema.yaml", 1*time.Hour, 30*time.Second, nil, false) assert.Error(t, err) } @@ -559,7 +559,7 @@ components: tmpFile.Write([]byte(schemaContent)) tmpFile.Close() - doc, err := cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, true) + doc, err := cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, nil, true) assert.NoError(t, err) schema, err := findSchemaByType(ctx, doc, "TestType") @@ -589,7 +589,7 @@ components: tmpFile.Write([]byte(schemaContent)) tmpFile.Close() - doc, err := cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, true) + doc, err := cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, nil, true) assert.NoError(t, err) _, err = findSchemaByType(ctx, doc, "NonExistentType") @@ -1209,7 +1209,7 @@ components: cache.rawSchemas["TestType/attributes.yaml"] = []byte(schemaContent) - doc, err := cache.loadSchemaFromPath(ctx, "TestType/attributes.yaml", 1*time.Hour, 30*time.Second, true) + doc, err := cache.loadSchemaFromPath(ctx, "TestType/attributes.yaml", 1*time.Hour, 30*time.Second, nil, true) assert.NoError(t, err) assert.NotNil(t, doc) assert.Equal(t, "3.1.0", doc.OpenAPI) @@ -1223,7 +1223,7 @@ func TestLoadSchemaFromPath_LRUHit(t *testing.T) { cache.set(hashURL("TestType/attributes.yaml"), expected, 1*time.Hour) // localSchema=false skips rawSchemas step, goes straight to LRU - doc, err := cache.loadSchemaFromPath(ctx, "TestType/attributes.yaml", 1*time.Hour, 30*time.Second, false) + doc, err := cache.loadSchemaFromPath(ctx, "TestType/attributes.yaml", 1*time.Hour, 30*time.Second, nil, false) assert.NoError(t, err) assert.Equal(t, expected, doc) } @@ -1243,7 +1243,7 @@ info: tmpFile.Close() // rawSchemas empty, localSchema=true — local miss, falls through to file load - doc, err := cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, true) + doc, err := cache.loadSchemaFromPath(ctx, tmpFile.Name(), 1*time.Hour, 30*time.Second, nil, true) assert.NoError(t, err) assert.NotNil(t, doc) } @@ -1367,7 +1367,7 @@ func TestLoadSchemaFromPath_TTLExpiry_FetchesFresh(t *testing.T) { ctx := context.Background() // Load v1 with a 1ms TTL so the LRU entry expires almost immediately. - doc1, err := cache.loadSchemaFromPath(ctx, server.URL, 1*time.Millisecond, 30*time.Second, false) + doc1, err := cache.loadSchemaFromPath(ctx, server.URL, 1*time.Millisecond, 30*time.Second, nil, false) assert.NoError(t, err) assert.Equal(t, "Schema v1", doc1.Info.Title) @@ -1376,7 +1376,7 @@ func TestLoadSchemaFromPath_TTLExpiry_FetchesFresh(t *testing.T) { serveV2.Store(true) // Re-load — LRU miss (expired), freshReadFromURI fetches from the server and gets v2. - doc2, err := cache.loadSchemaFromPath(ctx, server.URL, 1*time.Hour, 30*time.Second, false) + doc2, err := cache.loadSchemaFromPath(ctx, server.URL, 1*time.Hour, 30*time.Second, nil, false) assert.NoError(t, err) assert.Equal(t, "Schema v2", doc2.Info.Title, "expected v2 after TTL expiry — global URIMapCache not bypassed") } @@ -1871,13 +1871,17 @@ func TestValidateReferencedObject_RefusesARefThatWouldReadTheDisk(t *testing.T) } } -// The packs pull 15 documents across 3 hosts -- the one the allowlist names -// plus two external spec hosts the packs $ref into -- so a $ref to a host -// outside the allowlist is the normal case, not the attack. This pins that: -// applying isAllowedDomain to $refs as well would need all three hosts named -// in the allowlist first, and would otherwise stop every pack loading. -// Deliberate, not missed. -func TestValidateReferencedObject_AllowsARefToAHostOutsideTheAllowlist(t *testing.T) { +// A $ref may not reach a host the allowlist does not name. +// +// The entry @context being allowlisted is not enough. The document it returns +// is NOT trusted -- it came from a URL the payload chose, on a host anyone can +// publish to -- so its $refs used to reach any http host at all. That let a +// payload name an attacker's document and have this process fetch whatever +// that document pointed at: an internal service, a cloud metadata endpoint. +// +// This is the case the allowlist has to cover to mean anything, because the +// refs are the great majority of the reads: one pack pulls 13-16 documents. +func TestValidateReferencedObject_RefusesARefToAHostOutsideTheAllowlist(t *testing.T) { borrowed := serveSchema(t, borrowedSchema) entry := serveSchema(t, strings.Replace(entrySchemaRefTemplate, "REF_TARGET", borrowed.URL+"/borrowed.yaml", 1)) @@ -1898,13 +1902,47 @@ func TestValidateReferencedObject_AllowsARefToAHostOutsideTheAllowlist(t *testin Path: "message.test", Context: entry.URL + "/context.jsonld", Type: "TestType", + Types: []string{"TestType"}, + Data: map[string]interface{}{"field1": "value1"}, + } + + // Only the entry host is allowlisted. The $ref host is not. + err = cache.validateReferencedObject(context.Background(), obj, + 1*time.Hour, 30*time.Second, []string{entryHost.Host}, false) + if err == nil { + t.Fatal("the cross-host $ref was fetched; a payload can point this process at any http host") + } + assert.Contains(t, err.Error(), "not in extendedSchema_allowedDomains") +} + +// And naming both hosts loads it, which is the case the packs need: a +// capability pack $refs schema.beckn.io, which $refs schema.nfh.global, so the +// allowlist has to carry every host in the chain or nothing loads. +func TestValidateReferencedObject_AllowsARefWhenBothHostsAreAllowlisted(t *testing.T) { + borrowed := serveSchema(t, borrowedSchema) + entry := serveSchema(t, strings.Replace(entrySchemaRefTemplate, "REF_TARGET", borrowed.URL+"/borrowed.yaml", 1)) + + entryHost, err := url.Parse(entry.URL) + if err != nil { + t.Fatalf("failed to parse the test server URL: %v", err) + } + borrowedHost, err := url.Parse(borrowed.URL) + if err != nil { + t.Fatalf("failed to parse the test server URL: %v", err) + } + + cache := newSchemaCache(10) + obj := referencedObject{ + Path: "message.test", + Context: entry.URL + "/context.jsonld", + Type: "TestType", + Types: []string{"TestType"}, Data: map[string]interface{}{"field1": "value1"}, } - // Only the entry host is allowlisted; the $ref host is not. if err := cache.validateReferencedObject(context.Background(), obj, - 1*time.Hour, 30*time.Second, []string{entryHost.Host}, false); err != nil { - t.Fatalf("a cross-host $ref must still resolve, or no pack can load: %v", err) + 1*time.Hour, 30*time.Second, []string{entryHost.Host, borrowedHost.Host}, false); err != nil { + t.Fatalf("both hosts allowlisted, so the chain must load: %v", err) } } @@ -1934,7 +1972,7 @@ func TestPayloadDirectedReader(t *testing.T) { t.Fatalf("failed to parse %q: %v", raw, err) } - data, err := payloadDirectedReader(openapi3.NewLoader(), u) + data, err := payloadDirectedReader(nil, false)(openapi3.NewLoader(), u) if tt.refused { if err == nil { t.Fatalf("%q was read, and must not have been", raw) From cca7ff2b739f0ea73a9c5cff66549c2cb4504112 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Tue, 8 Sep 2026 17:55:22 +0530 Subject: [PATCH 091/130] docs(MandiPrice): follow the rename in a cross-reference [#16] The prerequisites note pointed at weather/prerequisites.go, which this branch renamed to WeatherObservation/. It was the last stale plugin path in a Go comment. --- pkg/plugin/implementation/MandiPrice/prerequisites.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugin/implementation/MandiPrice/prerequisites.go b/pkg/plugin/implementation/MandiPrice/prerequisites.go index d583092c..d7c1f8aa 100644 --- a/pkg/plugin/implementation/MandiPrice/prerequisites.go +++ b/pkg/plugin/implementation/MandiPrice/prerequisites.go @@ -15,6 +15,6 @@ import "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstr // code, a token to exchange, a point to turn into a market. Each of those is a // different upstream than the one this was written against, and each would // bring the question of where the provider-to-function binding belongs -- see -// the note in weather/prerequisites.go and prefer keeping the payload explicit +// the note in WeatherObservation/prerequisites.go and prefer keeping the payload explicit // over adding an entry here. var prerequisites = upstream.Prerequisites{} From a20dac26108390aa568df53b936ac72cbb79aa8c Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 00:32:58 +0530 Subject: [PATCH 092/130] feat(upstream): accept an oauth2 client-credentials auth scheme [#7] First half of a scheme for an upstream behind an OAuth2 token endpoint: the config surface and the one piece of arithmetic worth isolating. Fetching and sending the token comes next. A static bearer token is not an option for the provider this is for. Its Keycloak service account issues tokens that live ten hours, so a value pasted into an environment variable is wrong twice a day, and the failure is a 401 on every request with nothing in the config to explain it. Only client_credentials. That grant has no user to redirect and issues no refresh token, so the other flows would be dead code. tokenLifetime is pure and separate because its edge cases are the whole difficulty and they need no clock: expires_in absent or <= 0 refused. "We were not told" is not a lifetime, and the alternatives are caching for a guessed period or re-fetching on every request. expires_in <= skew halved. now + 30s - 60s is in the PAST, so the naive subtraction makes every request fetch a fresh token. otherwise expires_in - skew, so a request that passes the check cannot reach the provider after expiry. The lifetime is read from the response and never from config: the issuer owns it, and a configured copy is silently wrong the moment a realm is retuned. --- .../internal/upstream/upstream.go | 55 ++++++++++++++++- .../internal/upstream/upstream_test.go | 60 +++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index 2d70ea0d..a82d70d4 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -89,6 +89,16 @@ const ( // transport error -- so the value is redacted from anything this package // logs or returns. See redact. AuthSchemeQuery = "query" + + // AuthSchemeOAuth2 exchanges a client id and secret for a short-lived + // bearer token, and sends that. For an upstream behind an OAuth2 token + // endpoint -- a Keycloak service account, say -- where a static token is not + // an option: the live one this was written against lives ten hours, so a + // value pasted into an environment variable is wrong twice a day. + // + // Only the client_credentials grant. There is no user to redirect and no + // refresh token in that grant, so the other flows would be dead code. + AuthSchemeOAuth2 = "oauth2" ) // codeUpstreamUnavailable reports a provider that could not be reached or @@ -156,6 +166,15 @@ type Config struct { QueryName string `yaml:"queryName" json:"queryName"` QueryValueEnv string `yaml:"queryValueEnv" json:"queryValueEnv"` + // TokenURL is the OAuth2 token endpoint. Not a credential, so it is named + // here rather than through an environment variable -- but it IS + // deployment-specific, so the reference config carries a placeholder. + TokenURL string `yaml:"tokenUrl" json:"tokenUrl"` + // ClientIDEnv and ClientSecretEnv name the variables holding the client + // credentials. The values never appear in config, the registry, or a log. + ClientIDEnv string `yaml:"clientIdEnv" json:"clientIdEnv"` + ClientSecretEnv string `yaml:"clientSecretEnv" json:"clientSecretEnv"` + // MaxResponseBytes caps what is read from the provider. MaxResponseBytes int64 `yaml:"maxResponseBytes" json:"maxResponseBytes"` } @@ -269,9 +288,14 @@ func applyDefaults(cfg *Config) error { if cfg.QueryName == "" || cfg.QueryValueEnv == "" { return errors.New("upstream: authScheme query requires queryName and queryValueEnv") } + case AuthSchemeOAuth2: + if cfg.TokenURL == "" || cfg.ClientIDEnv == "" || cfg.ClientSecretEnv == "" { + return errors.New( + "upstream: authScheme oauth2 requires tokenUrl, clientIdEnv and clientSecretEnv") + } default: return fmt.Errorf( - "upstream: unknown authScheme %q: must be none, basic, header or query", cfg.AuthScheme) + "upstream: unknown authScheme %q: must be none, basic, header, query or oauth2", cfg.AuthScheme) } return nil } @@ -789,6 +813,35 @@ type redactedErr struct { func (e redactedErr) Error() string { return e.text } func (e redactedErr) Unwrap() error { return e.err } +// tokenLifetime turns a token response's expires_in into how long we may hold +// that token, or reports that we may not hold it at all. +// +// The issuer owns this number, so it is read from the response and never from +// our config: a configured copy is a second version of the same fact, and it +// is wrong the moment a realm's token lifetime is retuned -- silently, with +// every request 401ing until someone edits a file. +// +// skew is subtracted so a request that passes the check cannot arrive at the +// provider after expiry. It has to exceed the round trip, and costs one extra +// refresh per token lifetime. +// +// Two answers rather than one duration, because "we were not told" is not a +// lifetime. A response with no expires_in is refused rather than cached for a +// guessed period or re-fetched on every single request. +func tokenLifetime(expiresIn int, skew time.Duration) (time.Duration, bool) { + if expiresIn <= 0 { + return 0, false + } + lifetime := time.Duration(expiresIn) * time.Second + // A lifetime at or under the skew would put the expiry in the past, and + // then every request fetches a fresh token. Half is still early enough to + // refresh before the real expiry. + if lifetime <= skew { + return lifetime / 2, true + } + return lifetime - skew, true +} + // redactString removes the configured credential from any text about to be // logged or returned -- an error, a provider's response body, or the URL that // was requested. diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go index 2e871dea..5adc4eb3 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -152,6 +152,14 @@ func TestNewValidatesTheAuthScheme(t *testing.T) { {"header with both settings", &Config{AuthScheme: AuthSchemeHeader, HeaderName: "X-Key", HeaderValueEnv: "V"}, true}, {"header missing the value variable", &Config{AuthScheme: AuthSchemeHeader, HeaderName: "X-Key"}, false}, {"an unknown scheme", &Config{AuthScheme: "oauth"}, false}, + {"oauth2 with all three settings", &Config{AuthScheme: AuthSchemeOAuth2, + TokenURL: "https://issuer.invalid/token", ClientIDEnv: "ID", ClientSecretEnv: "SECRET"}, true}, + {"oauth2 missing the token url", &Config{AuthScheme: AuthSchemeOAuth2, + ClientIDEnv: "ID", ClientSecretEnv: "SECRET"}, false}, + {"oauth2 missing the client id variable", &Config{AuthScheme: AuthSchemeOAuth2, + TokenURL: "https://issuer.invalid/token", ClientSecretEnv: "SECRET"}, false}, + {"oauth2 missing the client secret variable", &Config{AuthScheme: AuthSchemeOAuth2, + TokenURL: "https://issuer.invalid/token", ClientIDEnv: "ID"}, false}, } for _, tc := range testCases { @@ -2138,3 +2146,55 @@ func TestRunPassesAnEmptyLocalWhenThereAreNoPrerequisites(t *testing.T) { t.Errorf("_local = %v, want it empty", local) } } + +// A token's lifetime comes from the token response, never from our config: the +// issuer owns it, and a configured copy drifts the moment a realm is retuned. +// +// Pure so the edge cases need no clock and no sleeping -- the same shape as +// budget() above. +func TestTokenLifetime(t *testing.T) { + t.Parallel() + + const skew = 60 * time.Second + + testCases := []struct { + name string + expiresIn int + want time.Duration + usable bool + }{ + { + // The live Keycloak realm: 10 hours, so we stop trusting it a + // minute early and refresh on the next request through. + name: "the ordinary case", expiresIn: 36000, + want: 36000*time.Second - skew, usable: true, + }, + { + // SHORTER THAN THE SKEW. now + 30s - 60s is in the past, so the + // naive arithmetic makes every single request fetch a new token. + // Half the lifetime is still ahead of expiry and still refreshes. + name: "shorter than the skew", expiresIn: 30, + want: 15 * time.Second, usable: true, + }, + {name: "exactly the skew", expiresIn: 60, want: 30 * time.Second, usable: true}, + {name: "one second", expiresIn: 1, want: 500 * time.Millisecond, usable: true}, + // A response that does not say how long its token lives is refused + // rather than cached for a guessed duration or re-fetched forever. + {name: "absent", expiresIn: 0, usable: false}, + {name: "negative", expiresIn: -1, usable: false}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got, ok := tokenLifetime(tc.expiresIn, skew) + if ok != tc.usable { + t.Fatalf("tokenLifetime(%d) usable = %v, want %v", tc.expiresIn, ok, tc.usable) + } + if ok && got != tc.want { + t.Errorf("tokenLifetime(%d) = %v, want %v", tc.expiresIn, got, tc.want) + } + }) + } +} From c1c4378fe3aa45b3ffc1faf7d6f4b02d2b020eb3 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 01:04:28 +0530 Subject: [PATCH 093/130] feat(upstream): exchange and cache the oauth2 bearer token [#7] Completes the scheme: the client_credentials exchange, an expiry-aware cache, and redaction of both secrets. TWO MECHANISMS FOR THE TOKEN, and the reason is a deadlock this had on the first attempt. tokenMu serialises the exchange, so a cold start sends one request to the issuer rather than one per concurrent caller. The token itself is atomic, so readers never take that mutex -- which matters because secretForms is a reader and it is reached from inside the exchange, through redactString on the non-2xx path. Guarding the value with tokenMu instead self-deadlocked on the first failing exchange. NOTHING IS CACHED ON FAILURE. A transient outage at the token endpoint must not leave the step holding a failure for a lifetime it never obtained. Three failing requests make three exchanges; the fourth can succeed. Both secrets are redacted: the client secret we send, and the token we get back. The token is the one that reaches the provider, so it is the one an echoing 401 body quotes. The client id is deliberately not redacted -- it identifies rather than authenticates, and it is what makes the log line useful. Verified by removing each guard in turn: no caching fails the reuse test, caching failures fails the retry test, dropping the oauth2 case in secretForms fails redaction, and an unclassified error fails the classification test. That last one only bites after a fix to the test itself. Asserting on the 502 alone was vacuous: the retry loop wraps its terminal failure in a 502 too, so an unclassified token error still surfaced as one. The test now asserts the message identifies the exchange as where it broke. --- .../internal/upstream/upstream.go | 157 +++++++++++ .../internal/upstream/upstream_test.go | 252 ++++++++++++++++++ 2 files changed, 409 insertions(+) diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index a82d70d4..2478aca1 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -28,6 +28,8 @@ import ( "sort" "strconv" "strings" + "sync" + "sync/atomic" "time" "github.com/beckn-one/beckn-onix/pkg/log" @@ -105,6 +107,12 @@ const ( // answered with a failure. It is not this adapter's fault and not the caller's. const codeUpstreamUnavailable = "NET_DOWNSTREAM_UNAVAILABLE" +// tokenRefreshSkew is how early an oauth2 token stops being trusted. It has to +// exceed the round trip to the provider, so that a request which passes the +// expiry check cannot arrive after the token has actually died. One extra +// exchange per token lifetime is the whole cost. +const tokenRefreshSkew = 60 * time.Second + // Prerequisites are the values a capability needs that its payload does not // carry, keyed by binding key. // @@ -188,6 +196,17 @@ type Step struct { registry definition.ProviderRecordLookup mapper definition.Mapper httpClient *http.Client + + // The oauth2 token this step holds. + // + // Two mechanisms because there are two jobs. tokenMu serialises the + // EXCHANGE, so a cold start sends one request to the issuer rather than one + // per concurrent caller. token is atomic so READERS never take that mutex, + // which matters because secretForms is one of them and it is reached from + // inside the exchange -- guarding the value with tokenMu instead deadlocked + // on the first failing exchange, which is how this was found. + tokenMu sync.Mutex + token atomic.Pointer[cachedToken] } // New creates the step. @@ -763,6 +782,12 @@ func (s *Step) authenticate(req *http.Request) error { query := req.URL.Query() query.Set(s.config.QueryName, value) req.URL.RawQuery = query.Encode() + case AuthSchemeOAuth2: + token, err := s.bearerToken(req.Context()) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+token) } return nil } @@ -813,6 +838,122 @@ type redactedErr struct { func (e redactedErr) Error() string { return e.text } func (e redactedErr) Unwrap() error { return e.err } +// bearerToken returns a token to send, exchanging one if what we hold has +// expired or if we hold none. +// +// The lock spans the fetch on purpose. Without it a cold start sends every +// concurrent request to the token endpoint, and the provider's issuer sees a +// burst of identical exchanges. Serialising them costs one wait per token +// lifetime and nothing after that, which is a better trade than a second +// caching layer. +// +// NOTHING IS CACHED ON FAILURE. A transient outage at the token endpoint must +// not leave this step holding a failure for the lifetime it never obtained -- +// the next request tries again. +func (s *Step) bearerToken(ctx context.Context) (string, error) { + if held := s.token.Load(); held != nil && time.Now().Before(held.expiry) { + return held.value, nil + } + + s.tokenMu.Lock() + defer s.tokenMu.Unlock() + + // Re-checked after acquiring: while this caller waited, whoever held the + // mutex may already have exchanged a fresh token. + if held := s.token.Load(); held != nil && time.Now().Before(held.expiry) { + return held.value, nil + } + + token, lifetime, err := s.exchangeToken(ctx) + if err != nil { + return "", err + } + s.token.Store(&cachedToken{value: token, expiry: time.Now().Add(lifetime)}) + return token, nil +} + +// cachedToken is a token and the moment it stops being trusted. +type cachedToken struct { + value string + expiry time.Time +} + +// tokenResponse is the part of an OAuth2 token response this step reads. +type tokenResponse struct { + AccessToken string `json:"access_token"` + ExpiresIn int `json:"expires_in"` +} + +// exchangeToken performs the client_credentials grant. +// +// Every failure here is the UPSTREAM exchange failing, so all of them carry +// 502. Unclassified they would surface as a 500, which tells a network peer +// this adapter broke when in fact the provider's issuer did. +func (s *Step) exchangeToken(ctx context.Context) (string, time.Duration, error) { + clientID, clientSecret := os.Getenv(s.config.ClientIDEnv), os.Getenv(s.config.ClientSecretEnv) + if clientID == "" || clientSecret == "" { + return "", 0, s.missingCredential(ctx, "oauth2", + s.config.ClientIDEnv+" and "+s.config.ClientSecretEnv) + } + + form := url.Values{ + "grant_type": {"client_credentials"}, + "client_id": {clientID}, + "client_secret": {clientSecret}, + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.config.TokenURL, + strings.NewReader(form.Encode())) + if err != nil { + return "", 0, s.tokenErr(fmt.Errorf("token request could not be built: %w", err)) + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + resp, err := s.httpClient.Do(req) + if err != nil { + return "", 0, s.tokenErr(fmt.Errorf("token endpoint %s could not be reached: %w", + s.config.TokenURL, err)) + } + defer resp.Body.Close() + + // Bounded like any other upstream read: a token response is small, and an + // unbounded read here would be a hole in the same wall. + body, err := io.ReadAll(io.LimitReader(resp.Body, s.config.MaxResponseBytes+1)) + if err != nil { + return "", 0, s.tokenErr(fmt.Errorf("token response could not be read: %w", err)) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + // The status, not the body: what an issuer puts in a failure body is its + // own business, and it routinely quotes the request back. + log.Warnf(ctx, "upstream: token endpoint %s returned %s: %s", + s.config.TokenURL, resp.Status, s.redactString(explain(body))) + return "", 0, s.tokenErr(fmt.Errorf("token endpoint returned %s", resp.Status)) + } + + var parsed tokenResponse + if err := json.Unmarshal(body, &parsed); err != nil { + return "", 0, s.tokenErr(fmt.Errorf("token response from %s is not JSON: %w", + s.config.TokenURL, err)) + } + if parsed.AccessToken == "" { + return "", 0, s.tokenErr(fmt.Errorf("token response from %s carries no access_token", + s.config.TokenURL)) + } + lifetime, ok := tokenLifetime(parsed.ExpiresIn, tokenRefreshSkew) + if !ok { + return "", 0, s.tokenErr(fmt.Errorf( + "token response from %s carries no usable expires_in, so its lifetime is unknown", + s.config.TokenURL)) + } + return parsed.AccessToken, lifetime, nil +} + +// tokenErr classifies a failed exchange. Always 502: the exchange is with the +// provider's issuer, so its failure is upstream's, never the caller's. +func (s *Step) tokenErr(err error) error { + return model.NewCodedErr(http.StatusBadGateway, codeUpstreamUnavailable, + fmt.Errorf("upstream: oauth2 token exchange failed: %w", err)) +} + // tokenLifetime turns a token response's expires_in into how long we may hold // that token, or reports that we may not hold it at all. // @@ -907,6 +1048,22 @@ func (s *Step) secretForms() []string { return nil } return []string{value} + case AuthSchemeOAuth2: + // Both halves: the client secret we send to the issuer, and the token + // it gave back. The token is the one that reaches the provider, so it + // is the one an echoing 401 body quotes. + var forms []string + if secret := os.Getenv(s.config.ClientSecretEnv); secret != "" { + forms = append(forms, secret) + } + // Read without tokenMu: this is reached from inside the exchange, which + // holds it. + if held := s.token.Load(); held != nil && held.value != "" { + forms = append(forms, held.value) + } + // The client id is deliberately NOT redacted: it identifies, it does + // not authenticate, and it is what makes a log line useful. + return longestFirst(forms) case AuthSchemeQuery: value := os.Getenv(s.config.QueryValueEnv) if value == "" { diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go index 5adc4eb3..3031c554 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -2198,3 +2198,255 @@ func TestTokenLifetime(t *testing.T) { }) } } + +// tokenServer stands in for an OAuth2 token endpoint. It records what it was +// asked and answers with whatever the test needs. +type tokenServer struct { + calls atomic.Int32 + expiresIn int // 0 omits the field entirely + token string // rotates per call when empty + status int + body string // overrides the JSON when set + gotGrant string + gotID string + gotSecret string +} + +func (ts *tokenServer) start(t *testing.T) string { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := ts.calls.Add(1) + _ = r.ParseForm() + ts.gotGrant, ts.gotID, ts.gotSecret = + r.PostFormValue("grant_type"), r.PostFormValue("client_id"), r.PostFormValue("client_secret") + if ts.status != 0 { + w.WriteHeader(ts.status) + } + if ts.body != "" { + fmt.Fprint(w, ts.body) + return + } + token := ts.token + if token == "" { + token = fmt.Sprintf("tok-%d", n) + } + if ts.expiresIn == 0 { + fmt.Fprintf(w, `{"access_token":%q,"token_type":"Bearer"}`, token) + return + } + fmt.Fprintf(w, `{"access_token":%q,"token_type":"Bearer","expires_in":%d}`, token, ts.expiresIn) + })) + t.Cleanup(srv.Close) + return srv.URL + "/token" +} + +// oauth2Step wires a step whose provider records the Authorization it received. +func oauth2Step(t *testing.T, tokenURL string, seen *[]string) *Step { + t.Helper() + provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + *seen = append(*seen, r.Header.Get("Authorization")) + fmt.Fprint(w, `{"ok":true}`) + })) + t.Cleanup(provider.Close) + + return newStep(t, &stubRegistry{plan: testPlan(provider.URL, http.MethodGet)}, + &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{"a":1}`)}, + func(c *Config) { + c.AuthScheme = AuthSchemeOAuth2 + c.TokenURL = tokenURL + c.ClientIDEnv, c.ClientSecretEnv = "TEST_OAUTH_ID", "TEST_OAUTH_SECRET" + }) +} + +// A3: the exchanged token has to reach the provider, and the client +// credentials have to reach the token endpoint the way the grant specifies. +func TestOAuth2SendsTheExchangedTokenAsABearer(t *testing.T) { + t.Setenv("TEST_OAUTH_ID", "svc-client") + t.Setenv("TEST_OAUTH_SECRET", "svc-secret") + + ts := &tokenServer{expiresIn: 36000, token: "the-token"} + var seen []string + step := oauth2Step(t, ts.start(t), &seen) + + if _, err := runStep(t, step, selectBody); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + if len(seen) != 1 || seen[0] != "Bearer the-token" { + t.Errorf("provider saw Authorization %q, want %q", seen, "Bearer the-token") + } + if ts.gotGrant != "client_credentials" { + t.Errorf("grant_type = %q, want client_credentials", ts.gotGrant) + } + if ts.gotID != "svc-client" || ts.gotSecret != "svc-secret" { + t.Errorf("token endpoint saw id/secret %q/%q, want the configured pair", ts.gotID, ts.gotSecret) + } +} + +// A4: a token good for ten hours must be exchanged ONCE, not per request. +// Without this the provider's issuer takes one extra round trip per call. +func TestOAuth2ReusesTheTokenWithinItsLifetime(t *testing.T) { + t.Setenv("TEST_OAUTH_ID", "svc-client") + t.Setenv("TEST_OAUTH_SECRET", "svc-secret") + + ts := &tokenServer{expiresIn: 36000} + var seen []string + step := oauth2Step(t, ts.start(t), &seen) + + for i := 0; i < 5; i++ { + if _, err := runStep(t, step, selectBody); err != nil { + t.Fatalf("request %d: %v", i, err) + } + } + if got := ts.calls.Load(); got != 1 { + t.Errorf("token endpoint called %d times for 5 requests, want 1", got) + } + for i, h := range seen { + if h != "Bearer tok-1" { + t.Errorf("request %d sent %q, want the cached token", i, h) + } + } +} + +// A5: and once it expires, the next request exchanges a fresh one. expires_in +// of 1s halves to 500ms under the skew, so this needs no long sleep. +func TestOAuth2RefreshesAnExpiredToken(t *testing.T) { + t.Setenv("TEST_OAUTH_ID", "svc-client") + t.Setenv("TEST_OAUTH_SECRET", "svc-secret") + + ts := &tokenServer{expiresIn: 1} + var seen []string + step := oauth2Step(t, ts.start(t), &seen) + + if _, err := runStep(t, step, selectBody); err != nil { + t.Fatalf("first request: %v", err) + } + time.Sleep(700 * time.Millisecond) + if _, err := runStep(t, step, selectBody); err != nil { + t.Fatalf("second request: %v", err) + } + if got := ts.calls.Load(); got != 2 { + t.Errorf("token endpoint called %d times, want 2 -- the second token was not fetched", got) + } + if len(seen) == 2 && seen[0] == seen[1] { + t.Errorf("both requests sent %q, want a rotated token", seen[0]) + } +} + +// A6: the token endpoint failing is the UPSTREAM exchange failing, so 502. +// Unclassified it becomes a 500, which tells a peer this adapter broke. +func TestOAuth2ClassifiesATokenFailureAsUpstream(t *testing.T) { + for _, tc := range []struct { + name string + ts *tokenServer + }{ + {"the endpoint refuses", &tokenServer{status: http.StatusInternalServerError, body: `{"error":"boom"}`}}, + {"the endpoint says unauthorized", &tokenServer{status: http.StatusUnauthorized, body: `{"error":"invalid_client"}`}}, + {"the response is not JSON", &tokenServer{body: `nope`}}, + {"the response carries no token", &tokenServer{body: `{"token_type":"Bearer","expires_in":36000}`}}, + {"the response omits expires_in", &tokenServer{expiresIn: 0}}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("TEST_OAUTH_ID", "svc-client") + t.Setenv("TEST_OAUTH_SECRET", "svc-secret") + + var seen []string + step := oauth2Step(t, tc.ts.start(t), &seen) + + _, err := runStep(t, step, selectBody) + if err == nil { + t.Fatal("expected the request to fail when no token could be obtained") + } + var coded *model.CodedErr + if !errors.As(err, &coded) { + t.Fatalf("error %v (%T) is unclassified, so it becomes a 500", err, err) + } + if coded.HTTPStatus() != http.StatusBadGateway { + t.Errorf("status = %d, want 502: the token exchange is upstream's failure", coded.HTTPStatus()) + } + // The status alone does not prove this classification: the retry + // loop wraps its terminal failure in a 502 too, so an unclassified + // token error would still surface as one. The message is what says + // the exchange is where it broke, rather than the provider call. + if !strings.Contains(err.Error(), "oauth2 token exchange failed") { + t.Errorf("error %v does not identify the token exchange as the failure", err) + } + if len(seen) != 0 { + t.Errorf("the provider was called %d times without a token", len(seen)) + } + }) + } +} + +// A6b: a failed exchange must not be cached. One transient failure would +// otherwise poison the step for the whole lifetime it never obtained. +func TestOAuth2DoesNotCacheAFailedExchange(t *testing.T) { + t.Setenv("TEST_OAUTH_ID", "svc-client") + t.Setenv("TEST_OAUTH_SECRET", "svc-secret") + + ts := &tokenServer{status: http.StatusInternalServerError, body: `{"error":"boom"}`} + var seen []string + step := oauth2Step(t, ts.start(t), &seen) + + for i := 0; i < 3; i++ { + if _, err := runStep(t, step, selectBody); err == nil { + t.Fatalf("request %d unexpectedly succeeded", i) + } + } + if got := ts.calls.Load(); got != 3 { + t.Errorf("token endpoint called %d times for 3 failing requests, want 3 -- a failure was cached", got) + } +} + +// A7: neither the client secret nor the token may reach a log or an error, +// in any form. A provider echoing the request it rejected is the ordinary +// shape of a 401 body. +func TestOAuth2RedactsTheSecretAndTheToken(t *testing.T) { + t.Setenv("TEST_OAUTH_ID", "svc-client") + t.Setenv("TEST_OAUTH_SECRET", `s3cr3t"quoted`) + + ts := &tokenServer{expiresIn: 36000, token: "the-token"} + var seen []string + step := oauth2Step(t, ts.start(t), &seen) + + // warm the cache so the step holds a token to redact + if _, err := runStep(t, step, selectBody); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + + for _, secret := range []string{`s3cr3t"quoted`, "the-token"} { + body := fmt.Sprintf(`{"error":"rejected %s"}`, secret) + got := step.redactString(explain([]byte(body))) + if strings.Contains(got, secret) { + t.Errorf("%q survived redaction: %s", secret, got) + } + if !strings.Contains(got, redactedMarker) { + t.Errorf("nothing was redacted from %q", got) + } + } + // the client id identifies rather than authenticates, so it stays + if got := step.redactString("client svc-client failed"); !strings.Contains(got, "svc-client") { + t.Errorf("the client id was redacted (%q); it identifies, it does not authenticate", got) + } +} + +// A8: an unset credential names the VARIABLE in our log and only the scheme on +// the wire -- a peer has no business learning which variables we read. +func TestOAuth2ReportsAMissingCredential(t *testing.T) { + t.Setenv("TEST_OAUTH_ID", "") + t.Setenv("TEST_OAUTH_SECRET", "") + + ts := &tokenServer{expiresIn: 36000} + var seen []string + step := oauth2Step(t, ts.start(t), &seen) + + _, err := runStep(t, step, selectBody) + if err == nil { + t.Fatal("expected a missing credential to fail the request") + } + if strings.Contains(err.Error(), "TEST_OAUTH_ID") || strings.Contains(err.Error(), "TEST_OAUTH_SECRET") { + t.Errorf("the error names the environment variables, which goes to the peer: %v", err) + } + if ts.calls.Load() != 0 { + t.Error("the token endpoint was called with no credentials") + } +} From 569435597663b41eec29e354cc1dd6b5b3324551 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 01:12:54 +0530 Subject: [PATCH 094/130] feat: add the KnowledgeAdvisory provider plugin [#7] A third provider capability, openagrinet:KnowledgeAdvisory, served against a retrieval service that answers a query with ranked passages from a document corpus. internal/upstream needed one addition for it -- authScheme oauth2, in the two commits before this -- and nothing else. The domain package is a name and an empty prerequisites map, as the other two are. THE FIRST CAPABILITY TO SEND A REQUEST BODY. The others are GETs whose mapping output becomes a query string; this one POSTs JSON. Same machinery, a path that had never been exercised by a shipped plugin. The mapping is the whole of the work, and every non-obvious line in it came from running it against the live corpus rather than from reading the API spec: topics -> query the caller's topics, joined. Only `query` is required upstream, so nothing else but the corpus is sent. ONE recommendation per DOCUMENT, not per hit. Four hits arrive over two documents: the engine returns several chunks per document AND this corpus indexes one chunk twice, once as type "document" and once as "scheme". topics, languages echoed from the REQUEST. category_tags and doc_language come back BLANK on every live hit, and recommendations[].language must be at least two characters -- so reading them from upstream would emit an invalid advisory. source the participant. NOT hits[].source, which reads "docs-pipeline": the ingestion pipeline, not the authority behind the knowledge. supportingResourceIds $distinct(doc_id). uniqueItems, and per-hit would repeat an id. no results resources: []. Commitment.resources is required but carries no minItems, so an empty array says "nothing matched". Building an advisory anyway produced an empty resource id and no recommendations. Two JSONata traps worth naming, both found by running it: $map over a one-element sequence returns the OBJECT, not a one-element array, so `resources` was malformed exactly when the answer narrowed to a single document -- the common case. Hence the brackets. `and` does not short-circuit in this engine, and comparing an undefined operand throws rather than yielding false. mappings_test.go serves the shipped file over HTTP through the real mapper and the real step, so the mapping is provable with go test. Verified by breaking it four ways: per-hit recommendations, the language read from upstream, the source read from hits[].source, and an advisory built for an empty result -- each fails its own test and nothing else. --- .../knowledge/knowledge-advisory.select.yaml | 154 +++++++ config/provider-adapter.yaml | 49 +++ install/build-plugins.sh | 1 + .../KnowledgeAdvisory/KnowledgeAdvisory.go | 37 ++ .../KnowledgeAdvisory/cmd/plugin.go | 112 +++++ .../KnowledgeAdvisory/cmd/plugin_test.go | 279 +++++++++++++ .../KnowledgeAdvisory/mappings_test.go | 386 ++++++++++++++++++ .../KnowledgeAdvisory/prerequisites.go | 22 + 8 files changed, 1040 insertions(+) create mode 100644 config/mappings/knowledge/knowledge-advisory.select.yaml create mode 100644 pkg/plugin/implementation/KnowledgeAdvisory/KnowledgeAdvisory.go create mode 100644 pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin.go create mode 100644 pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin_test.go create mode 100644 pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go create mode 100644 pkg/plugin/implementation/KnowledgeAdvisory/prerequisites.go diff --git a/config/mappings/knowledge/knowledge-advisory.select.yaml b/config/mappings/knowledge/knowledge-advisory.select.yaml new file mode 100644 index 00000000..e409071b --- /dev/null +++ b/config/mappings/knowledge/knowledge-advisory.select.yaml @@ -0,0 +1,154 @@ +# Knowledge advisory: a Beckn /select against the network's knowledge +# capability, answered from a provider's retrieval service. +# +# ONE FILE, BOTH DIRECTIONS, one Beckn action. The registry row for this +# capability names this URL, and the adapter fetches it verbatim -- so a +# deployment pointing at a different corpus points at its own copy of this +# file, and nothing about the provider is compiled in. +# +# Both halves read: +# beckn the inbound Beckn payload -- the context to echo, the topics to +# search for, the resource the caller selected +# and the response half additionally reads: +# response the provider's answer, in its own shape +# +# The base URL, the path and the credentials are NOT here. The first two are +# the registry's ProviderSchema row; the credentials are environment variables +# named in the adapter config. No host appears in this file. + +request: | + /* Request half: an OnDemand KnowledgeAdvisory select -> POST /search. + + Only `query` is required upstream; every other SearchRequest field has a + provider-side default, so nothing else is sent. + + The base URL and path are NOT here -- they live in the provider's registry + ProviderSchema row, so moving the provider is a registry write and no file + in this repo names a host. */ + ( + $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; + + /* DEPLOYMENT CONSTANT. Which corpus this deployment searches. The registry + row names this mapping file per deployment, so a different environment + points at a mapping carrying its own index. Omit the key entirely to let + the provider fall back to its persisted SearchSettings.indexName. */ + $indexName := "test_doc_pipeline"; + + { + "query": $trim($join($ra.topics, " ")), + "index_name": $indexName + } + ) + +response: | + ( + $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; + $ctx := $ra.`@context`; + $sel := beckn.message.contract.commitments[0].resources[0].id; + + /* Every hit the provider returned, unfiltered. It ran the query and chose + what to send back; second-guessing that with a score threshold here would + be this adapter deciding relevance on the provider's behalf. + + Worth knowing what that means in practice: hybrid RRF scores are not + calibrated, so a query can return hits within 0.005 of each other whose + subjects are unrelated. They will all be cited. Reinstating a floor is + one $filter if a deployment wants it. */ + $usable := $filter(response.hits, function($h) { $h.is_reference != true }); + + /* ONE recommendation per DOCUMENT, not per hit. doc_id repeats: the engine + returns up to max_chunks_per_doc chunks per document, and this corpus also + indexes one chunk twice, once as type "document" and once as "scheme". + Per-hit would say the same thing two or four times. */ + $docIds := $distinct($usable.doc_id); + $forDoc := function($id) { $filter($usable, function($h) { $h.doc_id = $id }) }; + $textOf := function($id) { $join($distinct($forDoc($id).text), "\n\n") }; + + /* Language comes from the REQUEST, not the hit. doc_language is "" on every + hit in this corpus (verified live), and recommendations[].language is + required with minLength 2, so reading it from upstream would emit an + invalid advisory. OnDemand requires languages, so the caller has sent it. */ + $lang := $ra.languages[0]; + + /* Scheme hits carry a governed code; document hits do not. Only "scheme" + maps -- "document" names a container format, not a subject. */ + $cats := $distinct($map($usable, function($h) { + $h.type = "scheme" ? "Scheme" + })); + + /* RATIONALE cites aggregate provenance only. Naming the index, the ranking + method or its parameters would leak engine internals to the caller. */ + $rationale := "Retrieved " & $string($count($usable)) & " of " + & $string(response.candidate_count) & " candidate passages from the " + & $distinct($usable.instance_name)[0] & " corpus."; + + { + "context": $merge([ beckn.context, { "action": "on_select" } ]), + "message": { + "contract": { + "status": { "code": "DRAFT" }, + "commitments": [{ + "status": { "descriptor": { "code": "DRAFT", "name": "Draft" } }, + "offer": { + "id": "offer:knowledge:" & $sel, + /* BOTH ids. The OnDemand id the caller selected, plus the Direct + id minted here -- OnDemand and Direct require different fields, + so the answer cannot restate the selected resource. Without the + old id the caller cannot correlate the callback. */ + /* Only mint an id when there IS a document. With no hits the + old form emitted "res:knowledge-advisory:" -- a dangling id + pointing at nothing. */ + "resourceIds": $exists($docIds) + ? [$sel, "res:knowledge-advisory:" & $docIds[0]] + : [$sel] + }, + /* ONE advisory, every contributing document cited beneath it. + + NO RESULTS IS AN ANSWER, NOT A FAILURE. With nothing above the + floor this emits resources: [] -- Commitment.resources is + required but carries no minItems, so an empty array is valid and + says "nothing matched" without inventing an advisory. Building + one anyway produced an empty resource id and no recommendations, + which is a malformed advisory the response leg never validates. */ + "resources": $exists($docIds) ? [{ + "id": "res:knowledge-advisory:" & $docIds[0], + "quantity": 1, + "descriptor": $merge([ + { "code": "KNOWLEDGE_ADVISORY" }, + $exists($distinct($usable.scheme_name)[0]) + ? { "name": $distinct($usable.scheme_name)[0] } : {} + ]), + "resourceAttributes": { + "@context": $ctx, + "@type": "openagrinet:KnowledgeAdvisory", + "informationMode": "Direct", + /* Echoed: category_tags is "" on every hit in this corpus. */ + "topics": $ra.topics, + "languages": $ra.languages, + "issuedAt": $now(), + "recommendations": [$map($docIds, function($id) { + { "message": $textOf($id), "language": $lang } + })], + /* Resource IDENTIFIERS, not raw doc hashes: the pack calls + these "Beckn Resource identifiers of supporting + KnowledgeResource information". Caveat worth knowing: they + only resolve if those documents are also published as + KnowledgeResource resources on the network. Today they are + not, so a caller can correlate but not dereference them. */ + "supportingResourceIds": [$map($docIds, function($id) { + "res:knowledge:" & $id + })], + "rationale": $rationale, + /* NOT hits[].source -- that is "docs-pipeline", the ingestion + pipeline name, not the authority behind the knowledge. */ + "source": { + "sourceId": "participant:bharat-vistaar", + "sourceName": $distinct($usable.instance_name)[0] + } + } + }] : [] + }] + } + } + } + ) diff --git a/config/provider-adapter.yaml b/config/provider-adapter.yaml index b2329ab0..e336a3c1 100644 --- a/config/provider-adapter.yaml +++ b/config/provider-adapter.yaml @@ -377,6 +377,54 @@ modules: # learning which variables this deployment reads. The variable name # is in the adapter's own log at error level. + # ------------------------------------------------------------------ + # KnowledgeAdvisory -- the network's knowledge capability, answered + # from a provider's retrieval service. + # + # Same shape as the two above and for the same reason: the package + # supplies a name, and recognising the capability, resolving the call + # plan, authenticating and translating are all internal/upstream's. + # ------------------------------------------------------------------ + - id: KnowledgeAdvisory + config: + bindingKeys: "knowledge-provider|openagrinet:KnowledgeAdvisory" + + # THE ONLY CAPABILITY HERE THAT CANNOT USE A STATIC CREDENTIAL. + # Its provider sits behind an OAuth2 token endpoint whose tokens + # live ten hours, so a value pasted into an environment variable + # is wrong twice a day and every request 401s until someone + # pastes another. authScheme oauth2 exchanges the client + # credentials for a token, holds it until just before it expires, + # and exchanges a fresh one on the next request after that. + authScheme: oauth2 + + # A PLACEHOLDER, because this is per-deployment and no host + # belongs in this file. Whatever goes here is the provider's + # token endpoint -- for a Keycloak realm, the openid-connect + # token path: + # + # https:///auth/realms//protocol/openid-connect/token + tokenUrl: <> + + # The VARIABLES holding the client credentials, never the values. + # Both must be set in the adapter's environment; with either + # missing the step refuses to call the upstream unauthenticated + # and the error names the scheme, not the variable -- a network + # peer has no business learning which variables we read. The + # variable names are in the adapter's own log at error level. + # + # docker: -e KNOWLEDGE_CLIENT_ID=... -e KNOWLEDGE_CLIENT_SECRET=... + # kubernetes: a Secret, mounted as environment variables + # local: KNOWLEDGE_CLIENT_ID=... KNOWLEDGE_CLIENT_SECRET=... ./server + clientIdEnv: KNOWLEDGE_CLIENT_ID + clientSecretEnv: KNOWLEDGE_CLIENT_SECRET + + # A retrieval answer is larger than a price row: four passages + # measured 15.7 KB against the live corpus, and top_k defaults to + # 12. The package default would truncate a full answer into + # something that no longer parses. + maxResponseBytes: 1048576 + # Declaring a step above is not enough: THIS list is what runs. A step # that appears under providerSteps but not here never executes, and the # request falls through to the 404 above -- which looks like a registry @@ -386,6 +434,7 @@ modules: - validateSchema # the pinned Beckn v2 spec - WeatherObservation # its binding key, or pass through - MandiPrice # its binding key, or pass through + - KnowledgeAdvisory # its binding key, or pass through - signAck # signs whatever the step answered with # ---------------------------------------------------------------------------- diff --git a/install/build-plugins.sh b/install/build-plugins.sh index b7e64256..59e5b817 100755 --- a/install/build-plugins.sh +++ b/install/build-plugins.sh @@ -34,6 +34,7 @@ plugins=( "jsonmapper" "WeatherObservation" "MandiPrice" + "KnowledgeAdvisory" "manifestloader" "reqpreprocessor" "otelsetup" diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/KnowledgeAdvisory.go b/pkg/plugin/implementation/KnowledgeAdvisory/KnowledgeAdvisory.go new file mode 100644 index 00000000..1f27849d --- /dev/null +++ b/pkg/plugin/implementation/KnowledgeAdvisory/KnowledgeAdvisory.go @@ -0,0 +1,37 @@ +// Package KnowledgeAdvisory serves the network's knowledge advisory capability. +// +// One package per schema pack family, so which plugin owns a capability is +// readable from its binding key: openagrinet:KnowledgeAdvisory is this one's, +// openagrinet:MandiPrice is MandiPrice's. +// +// Almost nothing lives here, and that is the point. Recognising a capability, +// resolving the call plan, authenticating, calling with the registry's budget +// and translating in both directions are all internal/upstream's, because none +// of them differ by domain. What this package owns is its name. +// +// The upstream this was written against is a retrieval service: it takes a +// query and answers with ranked passages from a document corpus. Turning the +// caller's topics into that query, and those passages into an advisory, is +// entirely the mapping's work -- so this package is a name and nothing else. +// See prerequisites.go for why that is worth stating rather than assuming. +package KnowledgeAdvisory + +import ( + "context" + + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" +) + +// Config is upstream's, unchanged. Aliased here so a domain plugin's cmd +// package need not know where the machinery lives. +type Config = upstream.Config + +// New creates the knowledge advisory step. +// +// Which capabilities it answers to is configuration, with no default: a package +// serving a family cannot guess which of them a deployment has providers for. +func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper definition.Mapper, + cfg *Config) (definition.Step, func() error, error) { + return upstream.New(ctx, registry, mapper, prerequisites, cfg) +} diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin.go b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin.go new file mode 100644 index 00000000..5bcefed5 --- /dev/null +++ b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin.go @@ -0,0 +1,112 @@ +// Command plugin builds the knowledge advisory provider step as a loadable +// plugin. +// +// The filename of the built .so is the id a deployment names in providerSteps, +// so this package is KnowledgeAdvisory's whole public surface: a config map in, +// a step out. +package main + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/beckn-one/beckn-onix/pkg/log" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/KnowledgeAdvisory" +) + +// knowledgeAdvisoryProvider implements definition.ProviderStepProvider. +type knowledgeAdvisoryProvider struct{} + +// newStepFunc creates a new step. Indirected for tests. +var newStepFunc = KnowledgeAdvisory.New + +// parseConfig turns the plugin config map into a typed Config. Anything absent +// is left zero: KnowledgeAdvisory.New applies the defaults and validates the +// auth scheme, so those rules live in one place. +func (p knowledgeAdvisoryProvider) parseConfig(config map[string]string) (*KnowledgeAdvisory.Config, error) { + cfg := &KnowledgeAdvisory.Config{ + BindingKeys: splitList(config["bindingKeys"]), + // Absent means the Beckn v2 convention. See upstream.Config for why + // this is a default rather than something to set. + ProviderIDAt: config["providerIdAt"], + CapabilityCodeAt: config["capabilityCodeAt"], + AuthScheme: config["authScheme"], + UsernameEnv: config["usernameEnv"], + PasswordEnv: config["passwordEnv"], + HeaderName: config["headerName"], + HeaderValueEnv: config["headerValueEnv"], + QueryName: config["queryName"], + QueryValueEnv: config["queryValueEnv"], + // authScheme oauth2. tokenUrl is not a credential and is named here; + // the client id and secret are named only as the variables holding + // them, so neither value appears in a config file or the registry. + TokenURL: config["tokenUrl"], + ClientIDEnv: config["clientIdEnv"], + ClientSecretEnv: config["clientSecretEnv"], + } + + if raw, exists := config["maxResponseBytes"]; exists && raw != "" { + value, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid maxResponseBytes value '%s': %w", raw, err) + } + if value <= 0 { + return nil, fmt.Errorf("maxResponseBytes must be positive, got %d", value) + } + cfg.MaxResponseBytes = value + } + + return cfg, nil +} + +// New creates a new knowledge advisory provider step instance. +func (p knowledgeAdvisoryProvider) New(ctx context.Context, registry definition.ProviderRecordLookup, mapper definition.Mapper, config map[string]string) (definition.Step, func() error, error) { + if ctx == nil { + return nil, nil, errors.New("context cannot be nil") + } + + cfg, err := p.parseConfig(config) + if err != nil { + log.Errorf(ctx, err, "Failed to parse knowledge advisory configuration") + return nil, nil, fmt.Errorf("failed to parse knowledge advisory configuration: %w", err) + } + + step, closer, err := newStepFunc(ctx, registry, mapper, cfg) + if err != nil { + log.Errorf(ctx, err, "Failed to create knowledge advisory step") + return nil, nil, err + } + + log.Infof(ctx, "Knowledge advisory step created successfully") + return step, closer, nil +} + +// splitList reads a comma-separated config value, which is how a list reaches a +// plugin -- the config is map[string]string. Blanks are dropped and spaces +// trimmed, so a trailing comma or a wrapped line is not a config error. +// +// A comma is unambiguous here: a binding key separates its own halves with a +// pipe. +func splitList(raw string) []string { + if strings.TrimSpace(raw) == "" { + return nil + } + var out []string + for _, part := range strings.Split(raw, ",") { + if trimmed := strings.TrimSpace(part); trimmed != "" { + out = append(out, trimmed) + } + } + return out +} + +// Provider is the exported plugin instance. +var Provider = knowledgeAdvisoryProvider{} + +// Compile-time proof the provider satisfies the interface the manager asserts +// against. A mismatch is otherwise a runtime cast failure at startup. +var _ definition.ProviderStepProvider = Provider diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin_test.go new file mode 100644 index 00000000..70e101a8 --- /dev/null +++ b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin_test.go @@ -0,0 +1,279 @@ +package main + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/KnowledgeAdvisory" +) + +type stubRegistry struct{} + +func (stubRegistry) ProviderRecord(context.Context, string) (*model.ProviderRecord, error) { + return nil, nil +} + +type stubMapper struct{} + +func (stubMapper) Verify(context.Context, string, any) error { return nil } + +func (stubMapper) Transform(context.Context, string, definition.Direction, any) ([]byte, error) { + return nil, nil +} + +func TestParseConfig(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + config map[string]string + expected *KnowledgeAdvisory.Config + expectedErr string + }{ + { + // Everything absent is left zero: KnowledgeAdvisory.New defaults + // it, so the rules are defined in exactly one place. + name: "leaves everything unset for New to default", + config: map[string]string{}, + expected: &KnowledgeAdvisory.Config{}, + }, + { + // The scheme this capability exists to use. Its provider sits + // behind an OAuth2 token endpoint issuing ten-hour tokens, so a + // static value in an environment variable is wrong twice a day. + name: "reads the oauth2 settings this capability needs", + config: map[string]string{ + "bindingKeys": "knowledge-provider|openagrinet:KnowledgeAdvisory", + "authScheme": "oauth2", + "tokenUrl": "https://issuer.invalid/token", + "clientIdEnv": "KNOWLEDGE_CLIENT_ID", + "clientSecretEnv": "KNOWLEDGE_CLIENT_SECRET", + }, + expected: &KnowledgeAdvisory.Config{ + BindingKeys: []string{"knowledge-provider|openagrinet:KnowledgeAdvisory"}, + AuthScheme: "oauth2", + TokenURL: "https://issuer.invalid/token", + ClientIDEnv: "KNOWLEDGE_CLIENT_ID", + ClientSecretEnv: "KNOWLEDGE_CLIENT_SECRET", + }, + }, + { + name: "reads every supported setting", + config: map[string]string{ + "bindingKeys": "other|capability", + "authScheme": "basic", + "usernameEnv": "U", + "passwordEnv": "P", + "headerName": "X-Key", + "headerValueEnv": "V", + "queryName": "q", + "queryValueEnv": "Q", + "tokenUrl": "https://issuer.invalid/token", + "clientIdEnv": "ID", + "clientSecretEnv": "SECRET", + "maxResponseBytes": "2048", + }, + expected: &KnowledgeAdvisory.Config{ + BindingKeys: []string{"other|capability"}, + AuthScheme: "basic", + UsernameEnv: "U", + PasswordEnv: "P", + HeaderName: "X-Key", + HeaderValueEnv: "V", + QueryName: "q", + QueryValueEnv: "Q", + TokenURL: "https://issuer.invalid/token", + ClientIDEnv: "ID", + ClientSecretEnv: "SECRET", + MaxResponseBytes: 2048, + }, + }, + { + name: "rejects a malformed response cap", + config: map[string]string{"maxResponseBytes": "lots"}, + expectedErr: "invalid maxResponseBytes value 'lots'", + }, + { + name: "rejects a non-positive response cap", + config: map[string]string{"maxResponseBytes": "0"}, + expectedErr: "maxResponseBytes must be positive", + }, + { + // Present but empty is not malformed. A rendered config with an + // unset variable produces this, and it should read as "unset". + name: "treats an empty response cap as unset", + config: map[string]string{"maxResponseBytes": ""}, + expected: &KnowledgeAdvisory.Config{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got, err := knowledgeAdvisoryProvider{}.parseConfig(tc.config) + + if tc.expectedErr != "" { + if err == nil { + t.Fatalf("expected error %q but got none", tc.expectedErr) + } + if !strings.Contains(err.Error(), tc.expectedErr) { + t.Errorf("expected error containing %q, got %q", tc.expectedErr, err.Error()) + } + return + } + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !reflect.DeepEqual(got, tc.expected) { + t.Errorf("expected config %+v, got %+v", tc.expected, got) + } + }) + } +} + +// A plugin config is map[string]string, so a list arrives comma-separated. +// Binding keys separate their own halves with a pipe, so a comma is +// unambiguous. +func TestSplitList(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + raw string + want []string + }{ + {name: "empty is nil, not a one-element list of nothing", raw: "", want: nil}, + {name: "whitespace only is nil", raw: " ", want: nil}, + {name: "one value", raw: "a|openagrinet:One", want: []string{"a|openagrinet:One"}}, + { + name: "several, with the blanks and spacing a wrapped config line produces", + raw: "a|openagrinet:One, b|openagrinet:Two ,, ", + want: []string{"a|openagrinet:One", "b|openagrinet:Two"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + if got := splitList(tc.raw); !reflect.DeepEqual(got, tc.want) { + t.Errorf("splitList(%q) = %v, want %v", tc.raw, got, tc.want) + } + }) + } +} + +// The override is two keys, both or neither. Absent leaves the step on the +// Beckn v2 convention, which is what every deployment should be running. +func TestParseConfigReadsTheBindingKeyOverride(t *testing.T) { + t.Parallel() + + cfg, err := knowledgeAdvisoryProvider{}.parseConfig(map[string]string{ + "bindingKeys": "a|openagrinet:One", + "providerIdAt": "who.provider", + "capabilityCodeAt": "what[].type", + }) + if err != nil { + t.Fatalf("parseConfig() returned an unexpected error: %v", err) + } + if cfg.ProviderIDAt != "who.provider" || cfg.CapabilityCodeAt != "what[].type" { + t.Errorf("override = %q / %q, want the configured paths", cfg.ProviderIDAt, cfg.CapabilityCodeAt) + } +} + +func TestNew(t *testing.T) { + t.Parallel() + + t.Run("rejects a nil context", func(t *testing.T) { + t.Parallel() + + //nolint:staticcheck // deliberately passing a nil context to assert the guard. + _, _, err := knowledgeAdvisoryProvider{}.New(nil, stubRegistry{}, stubMapper{}, map[string]string{}) + if err == nil { + t.Fatal("expected an error for a nil context, got none") + } + }) + + t.Run("rejects an unparseable config", func(t *testing.T) { + t.Parallel() + + _, _, err := knowledgeAdvisoryProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, + map[string]string{"maxResponseBytes": "lots"}) + if err == nil { + t.Fatal("expected an error for an invalid cap, got none") + } + }) + + // oauth2 without its three settings cannot authenticate, so it is refused + // at startup rather than on the first request. + t.Run("refuses oauth2 with no token url", func(t *testing.T) { + t.Parallel() + + _, _, err := knowledgeAdvisoryProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, + map[string]string{ + "bindingKeys": "a|openagrinet:One", + "authScheme": "oauth2", + "clientIdEnv": "ID", + "clientSecretEnv": "SECRET", + }) + if err == nil { + t.Fatal("expected oauth2 without tokenUrl to be refused") + } + }) + + // A domain plugin serves a family of capabilities, so it cannot guess which + // of them a deployment has providers for. + t.Run("refuses a config naming no capability", func(t *testing.T) { + t.Parallel() + + if _, _, err := (knowledgeAdvisoryProvider{}).New(context.Background(), stubRegistry{}, stubMapper{}, + map[string]string{}); err == nil { + t.Fatal("expected a config with no bindingKeys to be refused") + } + }) + + t.Run("returns the step and its closer on a config that loads", func(t *testing.T) { + closed := false + original := newStepFunc + newStepFunc = func(context.Context, definition.ProviderRecordLookup, definition.Mapper, + *KnowledgeAdvisory.Config) (definition.Step, func() error, error) { + return nil, func() error { closed = true; return nil }, nil + } + defer func() { newStepFunc = original }() + + _, closer, err := knowledgeAdvisoryProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, + map[string]string{"bindingKeys": "knowledge-provider|openagrinet:KnowledgeAdvisory"}) + if err != nil { + t.Fatalf("New() returned an unexpected error: %v", err) + } + if closer == nil { + t.Fatal("New() returned no closer, so nothing can release the step") + } + if err := closer(); err != nil { + t.Errorf("closer() returned %v, want nil", err) + } + if !closed { + t.Error("closer() did not reach the step's own closer") + } + }) + + t.Run("propagates a failure from the step constructor", func(t *testing.T) { + original := newStepFunc + wanted := errors.New("upstream refused the config") + newStepFunc = func(context.Context, definition.ProviderRecordLookup, definition.Mapper, + *KnowledgeAdvisory.Config) (definition.Step, func() error, error) { + return nil, nil, wanted + } + defer func() { newStepFunc = original }() + + _, _, err := knowledgeAdvisoryProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, + map[string]string{"bindingKeys": "knowledge-provider|openagrinet:KnowledgeAdvisory"}) + if !errors.Is(err, wanted) { + t.Errorf("New() error = %v, want it to wrap %v", err, wanted) + } + }) +} diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go new file mode 100644 index 00000000..4a678521 --- /dev/null +++ b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go @@ -0,0 +1,386 @@ +package KnowledgeAdvisory_test + +// mappings_test.go runs the shipped knowledge mapping through the real mapper +// and the real provider step. It is the only test that proves the three pieces +// fit: a mapping is JSONata inside YAML fetched over HTTP, and nothing but +// running it establishes that what is published actually produces valid Beckn. +// +// An external test package on purpose -- it uses the plugins exactly as the +// adapter does, through their exported surface and nothing else. + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/KnowledgeAdvisory" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" +) + +// mappingsDir is where the shipped mappings live, relative to this package. +const mappingsDir = "../../../../config/mappings/knowledge" + +// shippedMapping is the file this binding-action publishes: one file, both +// directions. The action segment must match the action the registry entry +// declares -- a mismatch would apply a correct mapping to the wrong call. +const shippedMapping = "knowledge-advisory.select.yaml" + +const shippedCapability = "openagrinet:KnowledgeAdvisory" + +const shippedBindingKey = "knowledge-provider|" + shippedCapability + +// selectRequest is a KnowledgeAdvisory select in OnDemand mode: it carries the +// topics to search for and the languages it can read, and no advisory of its +// own -- the pack's OnDemand branch requires exactly those two. +const selectRequest = `{ + "context": { "version": "2.0.0", "action": "select", + "networkId": "oan-dev", + "transactionId": "9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44", + "messageId": "7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22", + "timestamp": "2026-09-09T06:12:01.330Z" }, + "message": { "contract": { "commitments": [{ + "status": { "descriptor": { "code": "DRAFT", "name": "Draft" } }, + "resources": [{ + "id": "res:knowledge-advisory:schemes:ondemand", + "resourceAttributes": { + "@context": "https://openagrinet.github.io/network-specs/schema/KnowledgeAdvisory/v0.1/context.jsonld", + "@type": "openagrinet:KnowledgeAdvisory", + "informationMode": "OnDemand", + "topics": ["gruha jyoti scheme eligibility"], + "languages": ["en"] + } + }], + "offer": { + "id": "offer:knowledge-advisory:schemes", + "provider": { "id": "knowledge-provider" }, + "resourceIds": ["res:knowledge-advisory:schemes:ondemand"] + } + }] } } +}` + +// providerResponse is captured from the live retrieval service, trimmed to the +// fields the mapping reads. Four hits over TWO documents: the engine returns +// several chunks per document, and this corpus indexes one chunk twice, once +// as type "document" and once as "scheme". Note doc_language and +// category_tags come back BLANK -- which is why the mapping takes languages +// and topics from the request instead. +const providerResponse = `{ + "candidate_count": 51, + "final_count": 4, + "hits": [ + { + "_id": "26860172-2762-4b0b-cabf-3c947846c2e1", + "_score": 0.77918214, + "doc_id": "74b89b5f69dcd56f6566d4b1a9392ffd", + "text": "![House icon](a2bc7bb7c74e387ff445bcc6cf9b2a5a_1_img.webp)\n# Gruha Jyoti Scheme:\n1. 200 units of free electricity per family per month\n1.1. Your monthly household electricity consumption:\n0-100 Units\n100-200 Units\nAbove 200 Units\n1.2. Household consumption electricity meter connection number _____\n![Hand icon](a2bc7bb7c74e387ff445bcc6cf9b2a5a_7_img.webp)\n# Chayata Scheme:\nTo receive Rs.4000 per month under Chayata Scheme and Rs.6000 Disability Pension, provide the following details.\n( Those currently receiving pension do not need to apply )\n1. Disability Certificate Number: _____\n2. Others:\nOld Age\nWidow\nWeavers\nCotton Workers\nDiabetics\nAIDS Affected\nBeedi Workers' Livelihood Pension\nFilariasis Affected\nSingle Woman Livelihood Pension\nBeedi Contractor Livelihood Pension\nPath of Progress.. Welfare of All.. Our People's Government\n![Government of Telangana logo](a2bc7bb7c74e387ff445bcc6cf9b2a5a_16_img.webp)\nGovernment of Telangana\n![Circular graphic with Nirmala Sitharaman's portrait and icons for various schemes: Mahalaxmi, Rashtra Janjane, Grahajyoti, Chayata, Yuvavikasan, and Indiramandir](a2bc7bb7c74e387ff445bcc6cf9b2a5a_18_img.webp)![Portrait of Nirmala Sitharaman and N. Chandrababu Naidu](a2bc7bb7c74e387ff445bcc6cf9b2a5a_19_img.webp)\nPeople's Governance\nFrom 28.12.2023 to 06.01.2024", + "type": "document", + "instance_name": "Bharat Vistaar", + "is_reference": false, + "doc_language": "", + "category_tags": "" + }, + { + "_id": "0e753c2b-040a-4af9-3dc4-c9fca3b01e4d", + "_score": 0.7783708, + "doc_id": "74b89b5f69dcd56f6566d4b1a9392ffd", + "text": "![House icon](71c6c13ee057c1204dd8980597d79d5b_1_img.webp)\n# Gruha Jyoti Scheme:\n1. 200 units of free electricity per family per month\n1.1. Your monthly household electricity consumption:\n0-100 Units\n100-200 Units\nAbove 200 Units\n1.2. Household consumption electricity meter connection number _____\n![Hand icon](71c6c13ee057c1204dd8980597d79d5b_7_img.webp)\n# Chayatha Scheme:\nTo receive Rs.4000 per month under Chayatha Scheme and Rs.6000 pension for disabled persons, provide the following details.\n(Those currently receiving pension do not need to apply)\n1. Disabled person certificate number: _____\n2. Others:\nOld Age\nWidow\nGita Workers\nAgricultural Laborers\nDiabetes Affected\nAIDS Affected\nBeedi Workers' Livelihood Pension\nFilaria Affected\nBantali Women Livelihood Pension\nBeedi Contractors' Livelihood Pension\nPath of Progress.. Welfare of All.. Our People's Government\n![Government of Telangana logo](71c6c13ee057c1204dd8980597d79d5b_16_img.webp)\nGovernment of Telangana\n![Circular graphic with Nirmala Sitharaman's portrait and icons for various schemes: Mahalaxmi, Rashtra Janjane, Grahajyoti, Chayath, Yuvavikasan, and Indiramandir](71c6c13ee057c1204dd8980597d79d5b_18_img.webp)![Portrait of Nirmala Sitharaman and N. Chandrababu Naidu](71c6c13ee057c1204dd8980597d79d5b_19_img.webp)\nPeople's Governance\nFrom 28.12.2023 to 06.01.2024", + "type": "scheme", + "scheme_code": "tgj", + "scheme_name": "Telangana Gruha Jyoti", + "instance_name": "Bharat Vistaar", + "is_reference": false, + "doc_language": "", + "category_tags": "" + }, + { + "_id": "b9ec5c75-7331-0424-f741-4b3294c6b75e", + "_score": 0.77453613, + "doc_id": "2cc58f8a1c39a64fc132338be3547f52", + "text": "- (d) Under the Anna Bhagya Scheme, from which state is the rice being brought; what is the rate of this rice; what is the transportation and other cost? (Details to be provided)\n- (e) Following what standard has the rice brought from outside the state been entrusted for transportation; and which company/agency is currently transporting the rice?\n- (f) At what rate is the levy quantity of rice being collected in the state for the Anna Bhagya Scheme being purchased; for how many months is this collected rice sufficient? (Detailed information to be provided)\n**Smart City Scheme**\n**Shri Kavatagi Matha Mahantesh Mallikarjuna (Local Institutions Department):-**\n1053 (1274) Will the Hon'ble Minister of Urban Development kindly inform the following matters :-\n- (a) Has the fact that the ambitious Smart City Scheme of the Central Government is not being implemented properly in the state come to the government's notice;\n- (b) If it has come to notice, what steps has the government taken to implement the Smart City Scheme in the state and complete it at a rapid pace; for the delay", + "type": "document", + "instance_name": "Bharat Vistaar", + "is_reference": false, + "doc_language": "", + "category_tags": "" + }, + { + "_id": "4c6466a5-6975-a3be-4a62-637ae1a611d3", + "_score": 0.7743588, + "doc_id": "2cc58f8a1c39a64fc132338be3547f52", + "text": "**Regarding the 'Jalashree' Scheme being implemented by KUIDFC in the State, Shri N. Appajigaoud (Local Institutions Department) :-**\n1031 (1244) Hon'ble Urban Development Minister, please inform on these matters :-\n- (a) What are the urban areas selected under the 24 x 7 drinking water supply scheme under the 'Jalashree' scheme being implemented by KUIDFC in the State, and what is the financial assistance allocated for their implementation; (Provide complete information)\n- (b) On what criteria were the urban areas included in this scheme selected; (Provide details)\n- (c) At what stage is the implementation of works in the urban areas selected under this scheme; (Provide complete information)\n- (d) Many urban areas selected under this scheme have been taken up under the drinking water development program under the Central Government's 'AMRUT' scheme, is it correct to include these urban areas again under the 'Jalashree' scheme; (Provide complete information)\n\n(d) Has the Government considered including the taluks in the districts where drought occurs frequently and the groundwater level is depleting in the state under the earlier 'Jalashree' scheme; if so, provide complete information?\n**Regarding Industrial Clusters in the state and industries established therein, Shri N. Appajigaoud (Local Bodies Department) :-**\n1032 (1248) Hon'ble Minister of Large and Medium Industries, please inform on these matters :-\n- (a) How many Industrial Clusters are there in the state; (Provide district-wise information including area)\n- (b) Out of these Industrial Clusters, in how many clusters has the Automobile industry been established; Provide complete information for each cluster including factories/enterprises;\n- (c) In several Small and Medium Industrial Estates located in the state for several decades such as Peethya Industrial Estate, Bidadi Industrial Estate, Mysore-Hebbal Industrial Estate, Belur-Hubballi Industrial Estate, etc., due to recent economic recession", + "type": "document", + "instance_name": "Bharat Vistaar", + "is_reference": false, + "doc_language": "", + "category_tags": "" + } + ] +}` + +// serveMappings publishes the shipped mapping over HTTP, which is how the +// mapper fetches it in production. +func serveMappings(t *testing.T) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := os.ReadFile(filepath.Join(mappingsDir, filepath.Base(r.URL.Path))) + if err != nil { + t.Errorf("could not read the mapping %q: %v", r.URL.Path, err) + w.WriteHeader(http.StatusNotFound) + return + } + fmt.Fprint(w, string(body)) + })) +} + +type stubRegistry struct{ plan *model.ProviderRecord } + +func (s *stubRegistry) ProviderRecord(context.Context, string) (*model.ProviderRecord, error) { + return s.plan, nil +} + +// runShipped drives the real step over the real mapping and returns the body +// the provider was sent and the answer produced. +func runShipped(t *testing.T, request string) (map[string]any, map[string]any) { + return runShippedWith(t, request, providerResponse) +} + +func runShippedWith(t *testing.T, request, providerBody string) (map[string]any, map[string]any) { + t.Helper() + + mappings := serveMappings(t) + defer mappings.Close() + + var sent map[string]any + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(raw, &sent) + fmt.Fprint(w, providerBody) + })) + defer upstream.Close() + + mapper, closeMapper, err := jsonmapper.New(context.Background(), &jsonmapper.Config{}) + if err != nil { + t.Fatalf("failed to build the mapper: %v", err) + } + defer closeMapper() + + registry := &stubRegistry{plan: &model.ProviderRecord{ + BindingKey: shippedBindingKey, + ParticipantID: "knowledge-provider", + CapabilityCode: shippedCapability, + BaseURL: upstream.URL, + Actions: map[string]model.ActionPlan{ + // POST with a body -- the first capability to send one. The others + // are GETs whose mapping output becomes a query string. + "select": {Method: http.MethodPost, Path: "/docs-pipeline-api/search", + Mappings: mappings.URL + "/" + shippedMapping, TimeoutMs: 30000, RetryMax: 1}, + }, + }} + + step, closeStep, err := KnowledgeAdvisory.New(context.Background(), registry, mapper, + &KnowledgeAdvisory.Config{BindingKeys: []string{shippedBindingKey}}) + if err != nil { + t.Fatalf("failed to build the step: %v", err) + } + defer closeStep() + + stepCtx := &model.StepContext{Context: t.Context(), Body: []byte(request)} + if err := step.Run(stepCtx); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + if len(stepCtx.ResponseBody) == 0 { + t.Fatal("the step produced no answer") + } + var answer map[string]any + if err := json.Unmarshal(stepCtx.ResponseBody, &answer); err != nil { + t.Fatalf("the answer is not JSON: %v\n%s", err, stepCtx.ResponseBody) + } + return sent, answer +} + +// attributesOf returns the single advisory the answer carries. +func attributesOf(t *testing.T, answer map[string]any) map[string]any { + t.Helper() + res := resourcesOf(t, answer) + if len(res) != 1 { + t.Fatalf("expected one advisory resource, got %d", len(res)) + } + attrs, ok := res[0].(map[string]any)["resourceAttributes"].(map[string]any) + if !ok { + t.Fatal("the resource carries no resourceAttributes") + } + return attrs +} + +func resourcesOf(t *testing.T, answer map[string]any) []any { + t.Helper() + msg, ok := answer["message"].(map[string]any) + if !ok { + t.Fatalf("the answer carries no message: %v", answer) + } + contract := msg["contract"].(map[string]any) + commitments := contract["commitments"].([]any) + if len(commitments) != 1 { + t.Fatalf("expected one commitment, got %d", len(commitments)) + } + res, _ := commitments[0].(map[string]any)["resources"].([]any) + return res +} + +// The request half's whole job: the caller's topics become the query. Only +// `query` is required upstream, so nothing else but the corpus is sent. +func TestShippedMappingTurnsTopicsIntoTheQuery(t *testing.T) { + sent, _ := runShipped(t, selectRequest) + + if got := sent["query"]; got != "gruha jyoti scheme eligibility" { + t.Errorf("query = %v, want the caller's topics joined", got) + } + if _, ok := sent["index_name"]; !ok { + t.Error("the request names no index_name") + } +} + +// The answer is a Direct advisory: the caller asked a question and this +// payload carries the response, so it must not claim OnDemand -- that would +// tell the caller to invoke a provider it just invoked. +func TestShippedMappingAnswersWithADirectAdvisory(t *testing.T) { + _, answer := runShipped(t, selectRequest) + attrs := attributesOf(t, answer) + + if got := attrs["informationMode"]; got != "Direct" { + t.Errorf("informationMode = %v, want Direct", got) + } + if got := attrs["@type"]; got != shippedCapability { + t.Errorf("@type = %v, want %s", got, shippedCapability) + } + // Every field the pack's Direct branch requires. + for _, required := range []string{"topics", "issuedAt", "recommendations", "source"} { + if _, ok := attrs[required]; !ok { + t.Errorf("the advisory omits %q, which Direct requires", required) + } + } +} + +// ONE recommendation per DOCUMENT, not per hit. Four hits arrive over two +// documents, because the engine returns several chunks per document and this +// corpus indexes one chunk twice. Per-hit would say the same thing twice. +func TestShippedMappingCollapsesHitsToDocuments(t *testing.T) { + _, answer := runShipped(t, selectRequest) + attrs := attributesOf(t, answer) + + recs, _ := attrs["recommendations"].([]any) + if len(recs) != 2 { + t.Errorf("recommendations = %d, want 2 -- one per document, from 4 hits", len(recs)) + } + // supportingResourceIds declares uniqueItems, so a per-hit mapping would + // emit the same id twice and be refused by the pack. + sup, _ := attrs["supportingResourceIds"].([]any) + if len(sup) != 2 { + t.Errorf("supportingResourceIds = %d, want 2 distinct documents", len(sup)) + } + seen := map[any]bool{} + for _, id := range sup { + if seen[id] { + t.Errorf("supportingResourceIds repeats %v, which uniqueItems forbids", id) + } + seen[id] = true + } +} + +// topics and the recommendation language come from the REQUEST. This corpus +// returns category_tags and doc_language BLANK on every hit, so reading them +// from upstream would emit an advisory with an empty language -- which the +// pack requires to be at least two characters. +func TestShippedMappingTakesTopicsAndLanguageFromTheRequest(t *testing.T) { + _, answer := runShipped(t, selectRequest) + attrs := attributesOf(t, answer) + + topics, _ := attrs["topics"].([]any) + if len(topics) != 1 || topics[0] != "gruha jyoti scheme eligibility" { + t.Errorf("topics = %v, want the caller's own", topics) + } + recs, _ := attrs["recommendations"].([]any) + for i, r := range recs { + lang := r.(map[string]any)["language"] + if lang != "en" { + t.Errorf("recommendation %d language = %v, want the request's en", i, lang) + } + if msg, _ := r.(map[string]any)["message"].(string); strings.TrimSpace(msg) == "" { + t.Errorf("recommendation %d carries no message", i) + } + } +} + +// source names the PARTICIPANT, never the provider's own `source` field -- +// that one reads "docs-pipeline", the ingestion pipeline, not the authority +// behind the knowledge. +func TestShippedMappingNamesTheParticipantAsTheSource(t *testing.T) { + _, answer := runShipped(t, selectRequest) + attrs := attributesOf(t, answer) + + source, ok := attrs["source"].(map[string]any) + if !ok { + t.Fatal("the advisory carries no source") + } + id, _ := source["sourceId"].(string) + if id == "" { + t.Error("source carries no sourceId") + } + if strings.Contains(id, "docs-pipeline") { + t.Errorf("sourceId = %q, which is the ingestion pipeline, not the source", id) + } +} + +// NO RESULTS IS AN ANSWER, not a failure. Commitment.resources is required but +// carries no minItems, so an empty array says "nothing matched" without +// inventing an advisory -- and without the dangling resource id that building +// one anyway produced. +func TestShippedMappingAnswersAnEmptyResultWithNoResources(t *testing.T) { + _, answer := runShippedWith(t, selectRequest, + `{"candidate_count":0,"final_count":0,"hits":[]}`) + + if res := resourcesOf(t, answer); len(res) != 0 { + t.Errorf("resources = %d, want 0 for a query nothing matched", len(res)) + } + commitments := answer["message"].(map[string]any)["contract"].(map[string]any)["commitments"].([]any) + offer := commitments[0].(map[string]any)["offer"].(map[string]any) + ids, _ := offer["resourceIds"].([]any) + if len(ids) != 1 { + t.Fatalf("offer.resourceIds = %v, want only the selected id", ids) + } + if !strings.Contains(fmt.Sprint(ids[0]), "ondemand") { + t.Errorf("offer.resourceIds = %v, want the id the caller selected", ids) + } +} + +// The offer keeps BOTH ids: the OnDemand one the caller selected and the +// Direct one minted here. OnDemand and Direct require different fields, so the +// answer cannot restate the selected resource -- and without the old id the +// caller has nothing tying the callback to what it asked for. +func TestShippedMappingKeepsBothResourceIds(t *testing.T) { + _, answer := runShipped(t, selectRequest) + + commitments := answer["message"].(map[string]any)["contract"].(map[string]any)["commitments"].([]any) + offer := commitments[0].(map[string]any)["offer"].(map[string]any) + ids, _ := offer["resourceIds"].([]any) + if len(ids) != 2 { + t.Fatalf("offer.resourceIds = %v, want the selected id and the minted one", ids) + } + if !strings.Contains(fmt.Sprint(ids[0]), "ondemand") { + t.Errorf("the selected id is missing from %v", ids) + } + minted := attributesOf(t, answer) + _ = minted + res := resourcesOf(t, answer) + if got := res[0].(map[string]any)["id"]; fmt.Sprint(ids[1]) != fmt.Sprint(got) { + t.Errorf("offer.resourceIds[1] = %v but the resource id is %v", ids[1], got) + } +} diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/prerequisites.go b/pkg/plugin/implementation/KnowledgeAdvisory/prerequisites.go new file mode 100644 index 00000000..553f921e --- /dev/null +++ b/pkg/plugin/implementation/KnowledgeAdvisory/prerequisites.go @@ -0,0 +1,22 @@ +package KnowledgeAdvisory + +import "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" + +// prerequisites is what a knowledge advisory capability needs that its payload +// does not carry, keyed by binding key. +// +// Empty, and the reason is worth writing down because a token looks like a +// counter-example. This upstream sits behind an OAuth2 token endpoint, and +// "exchange a token" is exactly the kind of real I/O an entry here exists for +// -- but a token belongs in a request HEADER, and whatever a prerequisite +// returns reaches the mapping as _local and from there the request body or +// query string. There is no path from here to an Authorization header, and a +// credential handed to a published mapping file is a credential one expression +// away from being echoed into a request. So it is upstream's authScheme oauth2 +// instead, where the value is read, used, and never passed on. +// +// An entry would be needed for something the payload cannot carry and the +// mapping cannot obtain: a free-text topic resolved to a governed taxonomy id, +// say. The corpus index is not one -- it is a deployment constant, and the +// mapping file is itself per-deployment, named by the registry row. +var prerequisites = upstream.Prerequisites{} From 3037504a1c0124ecf6fc4d4ce76883e7678fa738 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 01:30:11 +0530 Subject: [PATCH 095/130] test(KnowledgeAdvisory): verify the whole path against the live provider [#7] Three things only the real endpoints can establish, and all three now are: the token endpoint answers the shape the code parses -- access_token plus expires_in, which came back 36000, exactly the ten hours that made a static credential unworkable; the token it issues is accepted by the provider; a second request REUSES it. Measured rather than timed: the exchange goes through a counting proxy, and two requests produce one exchange. The answer it produced was then validated against the published KnowledgeAdvisory pack: informationMode Direct, two recommendations over two documents from four hits, two distinct supporting ids, 2651 characters of real passage text. SKIPPED BY DEFAULT, so CI and a plain `go test ./...` never reach it. It runs only when the endpoints and credentials are in the environment, named by variable so no value is in this repo -- and no host is either, for the same reason none appears in a mapping or a config. --- .../KnowledgeAdvisory/live_test.go | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 pkg/plugin/implementation/KnowledgeAdvisory/live_test.go diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/live_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/live_test.go new file mode 100644 index 00000000..ddce11a5 --- /dev/null +++ b/pkg/plugin/implementation/KnowledgeAdvisory/live_test.go @@ -0,0 +1,153 @@ +package KnowledgeAdvisory_test + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/KnowledgeAdvisory" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" +) + +// TestLiveEndToEnd drives the real step against the LIVE provider: a real +// oauth2 exchange, a real POST /search, and the shipped mapping over the +// answer that comes back. +// +// SKIPPED BY DEFAULT. It runs only when the credentials and endpoints are in +// the environment, so CI and an ordinary `go test ./...` never reach it: +// +// KNOWLEDGE_TOKEN_URL the provider's OAuth2 token endpoint +// KNOWLEDGE_BASE_URL the provider's base URL +// KNOWLEDGE_CLIENT_ID } the client credentials, by variable name -- +// KNOWLEDGE_CLIENT_SECRET } the values are never in this repo +// +// No host appears in this file for the same reason it appears in no mapping or +// config: the endpoints are deployment facts, so they arrive from the +// environment. +// +// It is worth having despite needing credentials, because three things can +// only be established against the real thing: that the token endpoint answers +// the shape the code parses, that the token it issues is accepted by the +// provider, and that a second request reuses it. The last is measured, not +// timed -- the exchange goes through a counting proxy. +func TestLiveEndToEnd(t *testing.T) { + tokenURL, base := os.Getenv("KNOWLEDGE_TOKEN_URL"), os.Getenv("KNOWLEDGE_BASE_URL") + if tokenURL == "" || base == "" || os.Getenv("KNOWLEDGE_CLIENT_ID") == "" { + t.Skip("live credentials not in the environment") + } + + mappings := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := os.ReadFile(filepath.Join("../../../../config/mappings/knowledge", + filepath.Base(r.URL.Path))) + if err != nil { + t.Errorf("mapping read: %v", err) + w.WriteHeader(http.StatusNotFound) + return + } + fmt.Fprint(w, string(body)) + })) + defer mappings.Close() + + // A counting proxy in front of the real issuer, so token REUSE is measured + // against the live endpoint rather than inferred from timings. + var exchanges atomic.Int32 + issuer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + exchanges.Add(1) + body, _ := io.ReadAll(r.Body) + fwd, err := http.NewRequestWithContext(r.Context(), http.MethodPost, tokenURL, + strings.NewReader(string(body))) + if err != nil { + w.WriteHeader(http.StatusBadGateway) + return + } + fwd.Header.Set("Content-Type", r.Header.Get("Content-Type")) + resp, err := http.DefaultClient.Do(fwd) + if err != nil { + w.WriteHeader(http.StatusBadGateway) + return + } + defer resp.Body.Close() + out, _ := io.ReadAll(resp.Body) + w.WriteHeader(resp.StatusCode) + _, _ = w.Write(out) + })) + defer issuer.Close() + + mapper, closeMapper, err := jsonmapper.New(context.Background(), &jsonmapper.Config{}) + if err != nil { + t.Fatalf("mapper: %v", err) + } + defer closeMapper() + + const key = "knowledge-provider|openagrinet:KnowledgeAdvisory" + registry := &stubRegistry{plan: &model.ProviderRecord{ + BindingKey: key, ParticipantID: "knowledge-provider", + CapabilityCode: "openagrinet:KnowledgeAdvisory", BaseURL: base, + Actions: map[string]model.ActionPlan{ + "select": {Method: http.MethodPost, Path: "/docs-pipeline-api/search", + Mappings: mappings.URL + "/knowledge-advisory.select.yaml", + TimeoutMs: 60000, RetryMax: 1}, + }, + }} + + step, closeStep, err := KnowledgeAdvisory.New(context.Background(), registry, mapper, + &KnowledgeAdvisory.Config{ + BindingKeys: []string{key}, + AuthScheme: "oauth2", + TokenURL: issuer.URL + "/token", + ClientIDEnv: "KNOWLEDGE_CLIENT_ID", + ClientSecretEnv: "KNOWLEDGE_CLIENT_SECRET", + MaxResponseBytes: 1048576, + }) + if err != nil { + t.Fatalf("step: %v", err) + } + defer closeStep() + + // Two requests: the second must reuse the token rather than exchange again. + for i := 1; i <= 2; i++ { + ctx := &model.StepContext{Context: t.Context(), Body: []byte(selectRequest)} + if err := step.Run(ctx); err != nil { + t.Fatalf("request %d: %v", i, err) + } + if len(ctx.ResponseBody) == 0 { + t.Fatalf("request %d produced no answer", i) + } + var answer map[string]any + if err := json.Unmarshal(ctx.ResponseBody, &answer); err != nil { + t.Fatalf("request %d: answer is not JSON: %v", i, err) + } + res := resourcesOf(t, answer) + t.Logf(" request %d: %d resource(s), %d bytes", i, len(res), len(ctx.ResponseBody)) + if i == 2 && len(res) > 0 { + a := res[0].(map[string]any)["resourceAttributes"].(map[string]any) + recs, _ := a["recommendations"].([]any) + sup, _ := a["supportingResourceIds"].([]any) + msg, _ := recs[0].(map[string]any)["message"].(string) + t.Logf(" informationMode=%v topics=%v", a["informationMode"], a["topics"]) + t.Logf(" recommendations=%d supportingIds=%d first message=%d chars", + len(recs), len(sup), len(msg)) + t.Logf(" rationale=%v", a["rationale"]) + if strings.TrimSpace(msg) == "" { + t.Error("the advisory carries an empty message") + } + out, _ := json.Marshal(a) + _ = os.WriteFile(os.Getenv("CLAUDE_JOB_DIR")+"/tmp/live_attrs.json", out, 0o644) + } + } + + // The whole point of caching: two requests, ONE exchange. + if got := exchanges.Load(); got != 1 { + t.Errorf("the issuer was called %d times for 2 requests, want 1", got) + } + t.Logf(" token exchanges for 2 requests: %d", exchanges.Load()) +} From 5db84963f05b9e898bb51bf76f17cd8ac4e33188 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 09:33:00 +0530 Subject: [PATCH 096/130] fix(mappings): one advisory per document, so none borrows another's name [#7] The answer carried a single resource holding every document's text, and it could not label itself honestly. It took its id from one doc_id and its descriptor from whichever hit happened to carry a scheme_name -- so against the live corpus it came back named "Telangana Gruha Jyoti" while its second recommendation was a Karnataka legislative question about something else entirely. One resource per document fixes that by construction: each advisory covers exactly what its id names, cites exactly its own document, and can only take a name from that document. It also matches how the other capabilities answer -- mandi emits one resource per market row rather than one holding all of them. Nothing is dropped. Four hits still become two advisories over two documents, because the engine returns up to max_chunks_per_doc passages and this corpus indexes one chunk twice under two type labels; each document's passages are joined into its own recommendation. offer.resourceIds now lists the selected OnDemand id plus every minted Direct id, so a caller can resolve all of them. The tests caught the shape change rather than being edited to match it, and a new one pins the actual defect: exactly one advisory may carry a name here, and only the document that has a scheme_name. Borrowing across documents fails it -- verified by reverting to the old expression. --- .../knowledge/knowledge-advisory.select.yaml | 113 ++++++++++-------- .../KnowledgeAdvisory/mappings_test.go | 106 +++++++++++----- 2 files changed, 143 insertions(+), 76 deletions(-) diff --git a/config/mappings/knowledge/knowledge-advisory.select.yaml b/config/mappings/knowledge/knowledge-advisory.select.yaml index e409071b..f3fd7886 100644 --- a/config/mappings/knowledge/knowledge-advisory.select.yaml +++ b/config/mappings/knowledge/knowledge-advisory.select.yaml @@ -98,55 +98,74 @@ response: | /* Only mint an id when there IS a document. With no hits the old form emitted "res:knowledge-advisory:" -- a dangling id pointing at nothing. */ - "resourceIds": $exists($docIds) - ? [$sel, "res:knowledge-advisory:" & $docIds[0]] - : [$sel] + "resourceIds": $append([$sel], + [$map($docIds, function($id) { "res:knowledge-advisory:" & $id })]) }, - /* ONE advisory, every contributing document cited beneath it. - - NO RESULTS IS AN ANSWER, NOT A FAILURE. With nothing above the - floor this emits resources: [] -- Commitment.resources is - required but carries no minItems, so an empty array is valid and - says "nothing matched" without inventing an advisory. Building - one anyway produced an empty resource id and no recommendations, - which is a malformed advisory the response leg never validates. */ - "resources": $exists($docIds) ? [{ - "id": "res:knowledge-advisory:" & $docIds[0], - "quantity": 1, - "descriptor": $merge([ - { "code": "KNOWLEDGE_ADVISORY" }, - $exists($distinct($usable.scheme_name)[0]) - ? { "name": $distinct($usable.scheme_name)[0] } : {} - ]), - "resourceAttributes": { - "@context": $ctx, - "@type": "openagrinet:KnowledgeAdvisory", - "informationMode": "Direct", - /* Echoed: category_tags is "" on every hit in this corpus. */ - "topics": $ra.topics, - "languages": $ra.languages, - "issuedAt": $now(), - "recommendations": [$map($docIds, function($id) { - { "message": $textOf($id), "language": $lang } - })], - /* Resource IDENTIFIERS, not raw doc hashes: the pack calls - these "Beckn Resource identifiers of supporting - KnowledgeResource information". Caveat worth knowing: they - only resolve if those documents are also published as - KnowledgeResource resources on the network. Today they are - not, so a caller can correlate but not dereference them. */ - "supportingResourceIds": [$map($docIds, function($id) { - "res:knowledge:" & $id - })], - "rationale": $rationale, - /* NOT hits[].source -- that is "docs-pipeline", the ingestion - pipeline name, not the authority behind the knowledge. */ - "source": { - "sourceId": "participant:bharat-vistaar", - "sourceName": $distinct($usable.instance_name)[0] + /* ONE RESOURCE PER DOCUMENT. + + A single resource carrying every document's text was the first + shape here, and it could not label itself honestly: it took + its id from one doc_id and its descriptor from whichever hit + happened to carry a scheme_name, then held recommendations + from documents on entirely different subjects. On this corpus + an advisory came back named "Telangana Gruha Jyoti" whose + second recommendation was a Karnataka legislative question. + + Per document, each advisory covers exactly what its id names. + It also matches how the other capabilities answer -- mandi + emits one resource per market row, not one holding all of + them. + + NO RESULTS IS AN ANSWER: with nothing to map this is [], + which Commitment.resources allows -- required, but with no + minItems. */ + "resources": [$map($docIds, function($id) { + { + "id": "res:knowledge-advisory:" & $id, + "quantity": 1, + "descriptor": $merge([ + { "code": "KNOWLEDGE_ADVISORY" }, + /* Only THIS document's scheme_name, and only when it has + one. document-typed hits carry none, and borrowing + another document's name is what made the old shape + misleading. */ + $exists($distinct($forDoc($id).scheme_name)[0]) + ? { "name": $distinct($forDoc($id).scheme_name)[0] } : {} + ]), + "resourceAttributes": { + "@context": $ctx, + "@type": "openagrinet:KnowledgeAdvisory", + "informationMode": "Direct", + /* Echoed: category_tags is "" on every hit here. */ + "topics": $ra.topics, + "languages": $ra.languages, + "issuedAt": $now(), + /* This document's passages, joined: the engine returns up + to max_chunks_per_doc of them, and the same chunk can + arrive twice under two type labels. */ + "recommendations": [{ + "message": $textOf($id), + "language": $lang + }], + /* The document this advisory is drawn from, as a resource + IDENTIFIER rather than the raw hash -- the pack calls + these "Beckn Resource identifiers of supporting + KnowledgeResource information". They resolve only if + those documents are themselves published as + KnowledgeResource resources, which today they are not, + so a caller can correlate but not dereference them. */ + "supportingResourceIds": ["res:knowledge:" & $id], + "rationale": $rationale, + /* NOT hits[].source -- that is "docs-pipeline", the + ingestion pipeline, not the authority behind the + knowledge. */ + "source": { + "sourceId": "participant:bharat-vistaar", + "sourceName": $distinct($usable.instance_name)[0] + } + } } - } - }] : [] + })] }] } } diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go index 4a678521..f7ab5566 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go @@ -206,14 +206,15 @@ func runShippedWith(t *testing.T, request, providerBody string) (map[string]any, return sent, answer } -// attributesOf returns the single advisory the answer carries. -func attributesOf(t *testing.T, answer map[string]any) map[string]any { +// attributesOf returns the nth advisory's attributes. There is one resource per +// DOCUMENT, so the fixture's four hits over two documents produce two. +func attributesOf(t *testing.T, answer map[string]any, n int) map[string]any { t.Helper() res := resourcesOf(t, answer) - if len(res) != 1 { - t.Fatalf("expected one advisory resource, got %d", len(res)) + if len(res) <= n { + t.Fatalf("wanted advisory %d, but the answer carries %d resources", n, len(res)) } - attrs, ok := res[0].(map[string]any)["resourceAttributes"].(map[string]any) + attrs, ok := res[n].(map[string]any)["resourceAttributes"].(map[string]any) if !ok { t.Fatal("the resource carries no resourceAttributes") } @@ -253,7 +254,7 @@ func TestShippedMappingTurnsTopicsIntoTheQuery(t *testing.T) { // tell the caller to invoke a provider it just invoked. func TestShippedMappingAnswersWithADirectAdvisory(t *testing.T) { _, answer := runShipped(t, selectRequest) - attrs := attributesOf(t, answer) + attrs := attributesOf(t, answer, 0) if got := attrs["informationMode"]; got != "Direct" { t.Errorf("informationMode = %v, want Direct", got) @@ -274,24 +275,63 @@ func TestShippedMappingAnswersWithADirectAdvisory(t *testing.T) { // corpus indexes one chunk twice. Per-hit would say the same thing twice. func TestShippedMappingCollapsesHitsToDocuments(t *testing.T) { _, answer := runShipped(t, selectRequest) - attrs := attributesOf(t, answer) - recs, _ := attrs["recommendations"].([]any) - if len(recs) != 2 { - t.Errorf("recommendations = %d, want 2 -- one per document, from 4 hits", len(recs)) + res := resourcesOf(t, answer) + if len(res) != 2 { + t.Fatalf("resources = %d, want 2 -- one per document, from 4 hits", len(res)) } - // supportingResourceIds declares uniqueItems, so a per-hit mapping would - // emit the same id twice and be refused by the pack. - sup, _ := attrs["supportingResourceIds"].([]any) - if len(sup) != 2 { - t.Errorf("supportingResourceIds = %d, want 2 distinct documents", len(sup)) + ids := map[string]bool{} + for i := range res { + attrs := attributesOf(t, answer, i) + recs, _ := attrs["recommendations"].([]any) + if len(recs) != 1 { + t.Errorf("advisory %d carries %d recommendations, want 1 -- its own document's", i, len(recs)) + } + // supportingResourceIds declares uniqueItems. One document per + // advisory means one id, and it must be that advisory's own. + sup, _ := attrs["supportingResourceIds"].([]any) + if len(sup) != 1 { + t.Errorf("advisory %d cites %d documents, want 1", i, len(sup)) + } + id := fmt.Sprint(res[i].(map[string]any)["id"]) + if ids[id] { + t.Errorf("two advisories share the id %q", id) + } + ids[id] = true + // The resource id and the document it cites name the same document. + if len(sup) == 1 && !strings.HasSuffix(id, strings.TrimPrefix(fmt.Sprint(sup[0]), "res:knowledge:")) { + t.Errorf("advisory %q cites %v, which is a different document", id, sup[0]) + } } - seen := map[any]bool{} - for _, id := range sup { - if seen[id] { - t.Errorf("supportingResourceIds repeats %v, which uniqueItems forbids", id) +} + +// AN ADVISORY MUST NOT BORROW ANOTHER DOCUMENT'S NAME. The first shape here +// was one resource holding every document's text, which took its descriptor +// from whichever hit happened to carry a scheme_name -- so on this fixture it +// came back named "Telangana Gruha Jyoti" while carrying a recommendation from +// an unrelated Karnataka legislative document. Per document, a name can only +// come from the document it names. +func TestShippedMappingNeverBorrowsAnotherDocumentsName(t *testing.T) { + _, answer := runShipped(t, selectRequest) + res := resourcesOf(t, answer) + + named := 0 + for i := range res { + d, _ := res[i].(map[string]any)["descriptor"].(map[string]any) + name, _ := d["name"].(string) + if name == "" { + continue } - seen[id] = true + named++ + // Only the scheme-typed hit carries a scheme_name, and it belongs to + // the first document. Any other advisory claiming a name is borrowing. + id := fmt.Sprint(res[i].(map[string]any)["id"]) + if !strings.Contains(id, "74b89b5f69dcd56f6566d4b1a9392ffd") { + t.Errorf("advisory %q is named %q, but only the other document has a scheme name", id, name) + } + } + if named != 1 { + t.Errorf("%d advisories carry a name, want 1 -- exactly one document has a scheme_name", named) } } @@ -301,7 +341,7 @@ func TestShippedMappingCollapsesHitsToDocuments(t *testing.T) { // pack requires to be at least two characters. func TestShippedMappingTakesTopicsAndLanguageFromTheRequest(t *testing.T) { _, answer := runShipped(t, selectRequest) - attrs := attributesOf(t, answer) + attrs := attributesOf(t, answer, 0) topics, _ := attrs["topics"].([]any) if len(topics) != 1 || topics[0] != "gruha jyoti scheme eligibility" { @@ -324,7 +364,7 @@ func TestShippedMappingTakesTopicsAndLanguageFromTheRequest(t *testing.T) { // behind the knowledge. func TestShippedMappingNamesTheParticipantAsTheSource(t *testing.T) { _, answer := runShipped(t, selectRequest) - attrs := attributesOf(t, answer) + attrs := attributesOf(t, answer, 0) source, ok := attrs["source"].(map[string]any) if !ok { @@ -371,16 +411,24 @@ func TestShippedMappingKeepsBothResourceIds(t *testing.T) { commitments := answer["message"].(map[string]any)["contract"].(map[string]any)["commitments"].([]any) offer := commitments[0].(map[string]any)["offer"].(map[string]any) ids, _ := offer["resourceIds"].([]any) - if len(ids) != 2 { - t.Fatalf("offer.resourceIds = %v, want the selected id and the minted one", ids) + res := resourcesOf(t, answer) + if len(ids) != len(res)+1 { + t.Fatalf("offer.resourceIds = %v, want the selected id plus one per advisory (%d)", ids, len(res)) } if !strings.Contains(fmt.Sprint(ids[0]), "ondemand") { t.Errorf("the selected id is missing from %v", ids) } - minted := attributesOf(t, answer) - _ = minted - res := resourcesOf(t, answer) - if got := res[0].(map[string]any)["id"]; fmt.Sprint(ids[1]) != fmt.Sprint(got) { - t.Errorf("offer.resourceIds[1] = %v but the resource id is %v", ids[1], got) + // Every minted advisory has to appear, or a caller cannot resolve it. + for i := range res { + want := fmt.Sprint(res[i].(map[string]any)["id"]) + found := false + for _, got := range ids[1:] { + if fmt.Sprint(got) == want { + found = true + } + } + if !found { + t.Errorf("resource %q is missing from offer.resourceIds %v", want, ids) + } } } From a890913c4e50b2871eb385f42c6c7d4e6793544f Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 09:45:01 +0530 Subject: [PATCH 097/130] fix(mappings): echo the request offer instead of minting a dangling one [#7] The on_select offer listed the OnDemand resource id the caller selected alongside the minted advisory ids, and built its own offer id by gluing a prefix to that resource id -- so the answer carried offer:knowledge:res:knowledge-advisory:schemes:ondemand for an offer the request had already named offer:knowledge-advisory:schemes. The selected id is a reference to a resource appearing nowhere in this answer. resourceIds are "references (IDs) to resources covered by this offer", and OnDemand and Direct require different fields, so the answer cannot restate the selected resource. It was retained to let a caller correlate the callback, which the context already does with transactionId and messageId. On an empty result the offer referenced the selected id while resources[] was empty -- a reference to nothing. Echo the request's offer and rebuild only resourceIds from the documents answered, which is what MandiPrice already does and what its own comment prescribes. With no hits resourceIds is now [] rather than a dangling id, matching resources[]; Offer.resourceIds declares no minItems. --- .../knowledge/knowledge-advisory.select.yaml | 32 +++++++----- .../KnowledgeAdvisory/mappings_test.go | 49 ++++++++++++------- 2 files changed, 50 insertions(+), 31 deletions(-) diff --git a/config/mappings/knowledge/knowledge-advisory.select.yaml b/config/mappings/knowledge/knowledge-advisory.select.yaml index f3fd7886..62d901aa 100644 --- a/config/mappings/knowledge/knowledge-advisory.select.yaml +++ b/config/mappings/knowledge/knowledge-advisory.select.yaml @@ -44,7 +44,7 @@ response: | ( $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; $ctx := $ra.`@context`; - $sel := beckn.message.contract.commitments[0].resources[0].id; + $offer := beckn.message.contract.commitments[0].offer; /* Every hit the provider returned, unfiltered. It ran the query and chose what to send back; second-guessing that with a score threshold here would @@ -89,18 +89,24 @@ response: | "status": { "code": "DRAFT" }, "commitments": [{ "status": { "descriptor": { "code": "DRAFT", "name": "Draft" } }, - "offer": { - "id": "offer:knowledge:" & $sel, - /* BOTH ids. The OnDemand id the caller selected, plus the Direct - id minted here -- OnDemand and Direct require different fields, - so the answer cannot restate the selected resource. Without the - old id the caller cannot correlate the callback. */ - /* Only mint an id when there IS a document. With no hits the - old form emitted "res:knowledge-advisory:" -- a dangling id - pointing at nothing. */ - "resourceIds": $append([$sel], - [$map($docIds, function($id) { "res:knowledge-advisory:" & $id })]) - }, + /* The offer is echoed, but its references are not. The request + named the OnDemand resource -- the question -- while the answer + returns the advisories, so resourceIds is rebuilt from those. + + The selected id is deliberately NOT retained. resourceIds are + "references to resources covered by this offer", and the OnDemand + resource appears nowhere in resources[] here: OnDemand and Direct + require different fields, so the answer cannot restate it. Listing + it would leave the offer pointing at an id a caller cannot resolve. + Correlating the callback needs no help from it -- context carries + transactionId and messageId. Mandi echoes its offer the same way. + + With no hits $docIds is absent, so this is [] rather than a + dangling "res:knowledge-advisory:" -- matching resources[]. */ + "offer": $merge([ + $offer, + { "resourceIds": [$map($docIds, function($id) { "res:knowledge-advisory:" & $id })] } + ]), /* ONE RESOURCE PER DOCUMENT. A single resource carrying every document's text was the first diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go index f7ab5566..4550f5f6 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go @@ -66,6 +66,9 @@ const selectRequest = `{ }] } } }` +// The offer id selectRequest carries. The answer echoes it: one offer, one id. +const requestOfferID = "offer:knowledge-advisory:schemes" + // providerResponse is captured from the live retrieval service, trimmed to the // fields the mapping reads. Four hits over TWO documents: the engine returns // several chunks per document, and this corpus indexes one chunk twice, once @@ -393,36 +396,46 @@ func TestShippedMappingAnswersAnEmptyResultWithNoResources(t *testing.T) { commitments := answer["message"].(map[string]any)["contract"].(map[string]any)["commitments"].([]any) offer := commitments[0].(map[string]any)["offer"].(map[string]any) ids, _ := offer["resourceIds"].([]any) - if len(ids) != 1 { - t.Fatalf("offer.resourceIds = %v, want only the selected id", ids) - } - if !strings.Contains(fmt.Sprint(ids[0]), "ondemand") { - t.Errorf("offer.resourceIds = %v, want the id the caller selected", ids) + if len(ids) != 0 { + t.Fatalf("offer.resourceIds = %v, want none: the offer covers no resource", ids) } } -// The offer keeps BOTH ids: the OnDemand one the caller selected and the -// Direct one minted here. OnDemand and Direct require different fields, so the -// answer cannot restate the selected resource -- and without the old id the -// caller has nothing tying the callback to what it asked for. -func TestShippedMappingKeepsBothResourceIds(t *testing.T) { +// The offer is echoed and its references are rebuilt. resourceIds are +// "references to resources covered by this offer", so every id has to resolve +// inside resources[] -- and the OnDemand id the caller selected must NOT +// appear, because the answer cannot restate it: OnDemand and Direct require +// different fields. Correlation is context.transactionId's job, not this +// field's. +func TestShippedMappingReferencesOnlyTheResourcesItReturns(t *testing.T) { _, answer := runShipped(t, selectRequest) commitments := answer["message"].(map[string]any)["contract"].(map[string]any)["commitments"].([]any) offer := commitments[0].(map[string]any)["offer"].(map[string]any) + + // The request's own offer id, carried through rather than minted. + if got := fmt.Sprint(offer["id"]); got != requestOfferID { + t.Errorf("offer.id = %q, want the request's %q echoed", got, requestOfferID) + } + ids, _ := offer["resourceIds"].([]any) res := resourcesOf(t, answer) - if len(ids) != len(res)+1 { - t.Fatalf("offer.resourceIds = %v, want the selected id plus one per advisory (%d)", ids, len(res)) + if len(ids) != len(res) { + t.Fatalf("offer.resourceIds = %v, want exactly one per advisory (%d)", ids, len(res)) } - if !strings.Contains(fmt.Sprint(ids[0]), "ondemand") { - t.Errorf("the selected id is missing from %v", ids) + + returned := map[string]bool{} + for _, r := range res { + returned[fmt.Sprint(r.(map[string]any)["id"])] = true } - // Every minted advisory has to appear, or a caller cannot resolve it. - for i := range res { - want := fmt.Sprint(res[i].(map[string]any)["id"]) + for _, id := range ids { + if !returned[fmt.Sprint(id)] { + t.Errorf("offer references %q, which is not in resources[] -- a caller cannot resolve it", id) + } + } + for want := range returned { found := false - for _, got := range ids[1:] { + for _, got := range ids { if fmt.Sprint(got) == want { found = true } From b5e36a4dd87afb6d3be4b3ca1c51c1c4b9773f5d Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 12:02:39 +0530 Subject: [PATCH 098/130] fix(mappings): derive the advisory from the response, not the request [#7] Ports the derivation rules from the OpenAgriNet non-normative implementation guide for this capability onto the shape this adapter actually supplies. The guide reads a hand-built { search, pn } object; the response half is handed { beckn, _local, response }, so its rules are followed and its input shape is not. Four fields were taken from the REQUEST and read as provider data once they were in the answer: topics <- category_tags, else effective_config.query, the engine reporting the query it ran languages <- doc_language, else the guide's "en" recommendations[].language same source.sourceName already came from the response but read instance_name across EVERY hit inside a per-document resource -- the same borrowing bug already fixed one field over, latent only because this corpus has a single instance Adopted from the guide: the ""-is-absent helper, hits[].type -> subjectCategories via lookup ("document" maps to nothing, being a container format rather than a subject), scheme_code -> an upper-cased descriptor code, and the descriptor.name preference order that ranks name_en/name_gu LAST because this corpus fills them with the ingestion filename slug. source.sourceId stays a marked deployment constant beside index_name: SourceReference requires it and retrieval cannot supply it. The two upstream fields that look like candidates are traps, and the guide says so -- hits[].instance is a provider-internal code and hits[].source names the ingestion pipeline. agricultureSubjects and coverageAreas remain absent although scheme_code and scheme_name are real data: AgricultureSubjectReference requires a subjectId taxonomy URI nothing here supplies, and coverageAreas has no counterpart at all. Emitting either would mean inventing one. Deliberate divergences, each marked at its site: one Resource per document rather than one for the top-scoring hit; hits[].text IS the recommendation message, since nothing in this path synthesizes; no $assert, because an empty result is an answer; Contract.consideration omitted, because the guide's own caveat is that beckn:PriceSpecification is never defined in v2.0.0. Both produced advisories validate against the published KnowledgeAdvisory pack under a validator that does evaluate if/then, so the Direct branch is really checked -- verified with five negative controls that each fail as intended. --- .../knowledge/knowledge-advisory.select.yaml | 283 +++++++++++------- .../KnowledgeAdvisory/mappings_test.go | 53 +++- 2 files changed, 218 insertions(+), 118 deletions(-) diff --git a/config/mappings/knowledge/knowledge-advisory.select.yaml b/config/mappings/knowledge/knowledge-advisory.select.yaml index 62d901aa..ddc06846 100644 --- a/config/mappings/knowledge/knowledge-advisory.select.yaml +++ b/config/mappings/knowledge/knowledge-advisory.select.yaml @@ -40,138 +40,215 @@ request: | } ) + +# Response half derived from the OpenAgriNet non-normative implementation guide +# for this capability (knowledge-mapper/on_select.jsonata). Its derivation +# rules are followed; its INPUT SHAPE is not, because it reads a hand-built +# { search, pn } object and this adapter supplies { beckn, _local, response }. +# +# Divergences from the guide, each deliberate and marked at its site: +# - one Resource PER DOCUMENT, not one for the top-scoring hit +# - hits[].text IS the recommendation message; nothing here synthesizes +# - no $assert: an empty result is an answer, not an error +# - senderId/receiverId, not bapId/bppId, per the runtime Beckn spec +# - `res:` id prefix, matching the other OAN capabilities +# - Contract.consideration omitted; see the note at the emit block response: | ( - $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; - $ctx := $ra.`@context`; + $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; + $ctx := $ra.`@context`; $offer := beckn.message.contract.commitments[0].offer; - /* Every hit the provider returned, unfiltered. It ran the query and chose - what to send back; second-guessing that with a score threshold here would - be this adapter deciding relevance on the provider's behalf. + /* DEPLOYMENT CONSTANT, like index_name in the request half: the + network-stable participant id of the authority behind this corpus. + SourceReference.sourceId is required and retrieval cannot supply it. + Two upstream fields look like candidates and are traps: + hits[].instance "bv" provider-internal, collides across PNs + hits[].source "docs-pipeline" the ingestion pipeline, not the authority */ + $sourceId := "participant:bharat-vistaar"; + + /* This API returns "" for absent strings -- doc_language, category_tags, + title_en, doc_short_description and description are all "" on every hit + in the live corpus. Treat "" and absent identically, or a blank ships as + a present-but-empty value and fails minLength. */ + $text := function($s) { $exists($s) ? ($trim($s) != "" ? $trim($s)) : undefined }; + + /* hits[].type -> subjectCategories enum. Only these have an unambiguous + counterpart; "document" maps to NOTHING on purpose, because it names a + container format rather than a subject. */ + $categoryOf := function($t) { + $lookup({ + "scheme": "Scheme", "crop": "Crop", "livestock": "Livestock", + "weather": "Weather", "market": "Market", "practice": "Practice" + }, $lowercase($t)) + }; - Worth knowing what that means in practice: hybrid RRF scores are not - calibrated, so a query can return hits within 0.005 of each other whose - subjects are unrelated. They will all be cited. Reinstating a floor is - one $filter if a deployment wants it. */ + /* Governed codes are upper-case; the corpus emits scheme_code as "tgj". */ + $toCode := function($s) { $uppercase($replace($s, /[^A-Za-z0-9]+/, "_")) }; + + /* Every hit the provider returned, less its own reference chunks. It ran + the query and chose what to send; applying a score floor here would be + this adapter overriding the provider's relevance judgement. The guide + argues for a floor (unrelated documents can land within 0.005 of the top + hit) -- that is one $filter to reinstate if a deployment wants it. */ $usable := $filter(response.hits, function($h) { $h.is_reference != true }); - /* ONE recommendation per DOCUMENT, not per hit. doc_id repeats: the engine - returns up to max_chunks_per_doc chunks per document, and this corpus also - indexes one chunk twice, once as type "document" and once as "scheme". - Per-hit would say the same thing two or four times. */ + /* ONE ADVISORY PER DOCUMENT. doc_id repeats: the engine returns up to + max_chunks_per_doc chunks per document, and this corpus also indexes one + chunk twice, as type "document" and again as type "scheme". */ $docIds := $distinct($usable.doc_id); - $forDoc := function($id) { $filter($usable, function($h) { $h.doc_id = $id }) }; - $textOf := function($id) { $join($distinct($forDoc($id).text), "\n\n") }; - - /* Language comes from the REQUEST, not the hit. doc_language is "" on every - hit in this corpus (verified live), and recommendations[].language is - required with minLength 2, so reading it from upstream would emit an - invalid advisory. OnDemand requires languages, so the caller has sent it. */ - $lang := $ra.languages[0]; - - /* Scheme hits carry a governed code; document hits do not. Only "scheme" - maps -- "document" names a container format, not a subject. */ - $cats := $distinct($map($usable, function($h) { - $h.type = "scheme" ? "Scheme" - })); - - /* RATIONALE cites aggregate provenance only. Naming the index, the ranking - method or its parameters would leak engine internals to the caller. */ - $rationale := "Retrieved " & $string($count($usable)) & " of " - & $string(response.candidate_count) & " candidate passages from the " - & $distinct($usable.instance_name)[0] & " corpus."; + $hitsOf := function($id) { $filter($usable, function($h) { $h.doc_id = $id }) }; + + /* Counts are whole-response facts, so the rationale reads the same on every + advisory. It cites aggregate provenance only -- naming the index, the + ranking method or its parameters would leak engine internals. */ + $counts := "Retrieved " & $string($count($usable)) & " of " + & $string(response.candidate_count) & " candidate passages"; { "context": $merge([ beckn.context, { "action": "on_select" } ]), "message": { "contract": { "status": { "code": "DRAFT" }, + /* Contract.consideration is NOT emitted. The guide recommends a + priced quote, but its own caveat is the reason to leave it out: + Beckn v2.0.0 never DEFINES beckn:PriceSpecification, so any + considerationAttributes shape here is unverifiable. Contract + requires only `commitments`, so omitting it is valid, and an + advisory served at no charge is better said by saying nothing than + by inventing a schema. */ "commitments": [{ "status": { "descriptor": { "code": "DRAFT", "name": "Draft" } }, - /* The offer is echoed, but its references are not. The request + /* The offer is echoed, but its references are not: the request named the OnDemand resource -- the question -- while the answer - returns the advisories, so resourceIds is rebuilt from those. - - The selected id is deliberately NOT retained. resourceIds are - "references to resources covered by this offer", and the OnDemand - resource appears nowhere in resources[] here: OnDemand and Direct - require different fields, so the answer cannot restate it. Listing - it would leave the offer pointing at an id a caller cannot resolve. - Correlating the callback needs no help from it -- context carries - transactionId and messageId. Mandi echoes its offer the same way. - - With no hits $docIds is absent, so this is [] rather than a - dangling "res:knowledge-advisory:" -- matching resources[]. */ + returns the advisories. The selected id is NOT retained; it + resolves to nothing in resources[], and context.transactionId + already correlates the callback. With no hits this is []. */ "offer": $merge([ $offer, { "resourceIds": [$map($docIds, function($id) { "res:knowledge-advisory:" & $id })] } ]), - /* ONE RESOURCE PER DOCUMENT. - - A single resource carrying every document's text was the first - shape here, and it could not label itself honestly: it took - its id from one doc_id and its descriptor from whichever hit - happened to carry a scheme_name, then held recommendations - from documents on entirely different subjects. On this corpus - an advisory came back named "Telangana Gruha Jyoti" whose - second recommendation was a Karnataka legislative question. - - Per document, each advisory covers exactly what its id names. - It also matches how the other capabilities answer -- mandi - emits one resource per market row, not one holding all of - them. - - NO RESULTS IS AN ANSWER: with nothing to map this is [], - which Commitment.resources allows -- required, but with no - minItems. */ - "resources": [$map($docIds, function($id) { - { - "id": "res:knowledge-advisory:" & $id, - "quantity": 1, - "descriptor": $merge([ - { "code": "KNOWLEDGE_ADVISORY" }, - /* Only THIS document's scheme_name, and only when it has - one. document-typed hits carry none, and borrowing - another document's name is what made the old shape - misleading. */ - $exists($distinct($forDoc($id).scheme_name)[0]) - ? { "name": $distinct($forDoc($id).scheme_name)[0] } : {} - ]), - "resourceAttributes": { + "resources": [$map($docIds, function($id) { ( + $h := $hitsOf($id); + + /* Per document, the metadata-bearing hit: the scheme-typed one if + this document has it, else the highest scoring. _score orders + only and is never emitted. Without the first clause a document + indexed both ways loses its scheme_name to an untyped chunk. */ + $sch := $h[$exists(scheme_code)]; + $top := $exists($sch) ? $sch[0] : ($h^(>`_score`))[0]; + + /* doc_language, then the guide's "en" default. recommendations[] + requires language with minLength 2, so there is no honest empty + to fall back to -- the corpus leaves it blank on every hit. */ + $lang := $exists($text($top.doc_language)) ? $text($top.doc_language) : "en"; + + /* category_tags is a COMMA-JOINED STRING, not an array, and is "" + throughout this corpus. Falling back to the query the engine + reports having run keeps topics sourced from the response + rather than echoed from the request. topics is required in + Direct mode with minItems 1. */ + $tags := $distinct($h.category_tags[$ != ""].$split($, ",").$trim($)[$ != ""]); + $query := $text(response.effective_config.query); + $topics := $exists($tags) ? $tags : ($exists($query) ? [ $query ] : undefined); + + $cats := $distinct($h.$categoryOf(type)); + + /* scheme_name, then the human titles, then name_en/name_gu LAST: + this corpus fills those with the ingestion filename slug + ("Praja-Palana-Application-Form"), which reads badly to a + farmer. Only this document's hits are consulted -- borrowing + another document's name is what made an earlier shape lie. */ + $label := $exists($text($top.scheme_name)) ? $text($top.scheme_name) + : $exists($text($top.title_en)) ? $text($top.title_en) + : $exists($text($top.title_gu)) ? $text($top.title_gu) + : $exists($text($top.name_en)) ? $text($top.name_en) + : $exists($text($top.name_gu)) ? $text($top.name_gu) + : undefined; + + { + "id": "res:knowledge-advisory:" & $id, + /* Required by Commitment.resources, which declares no quantity + property and carries no Quantity schema at all -- a defect + upstream. One resource is one advisory. */ + "quantity": 1, + "descriptor": $merge([ + { + "code": $exists($text($top.scheme_code)) + ? $toCode($text($top.scheme_code)) & "_ADVISORY" + : "KNOWLEDGE_ADVISORY" + }, + $exists($label) ? { "name": $label & " advisory" } : {}, + /* Blank on every hit here, so $text drops them both today. */ + $exists($text($top.doc_short_description)) + ? { "shortDesc": $text($top.doc_short_description) } : {}, + $exists($text($top.doc_llm_description)) + ? { "longDesc": $text($top.doc_llm_description) } + : ($exists($text($top.description)) + ? { "longDesc": $text($top.description) } : {}) + ]), + "resourceAttributes": $merge([ + { "@context": $ctx, "@type": "openagrinet:KnowledgeAdvisory", + /* The discriminator. Its Direct branch is what makes + topics, issuedAt, recommendations and source required. */ "informationMode": "Direct", - /* Echoed: category_tags is "" on every hit here. */ - "topics": $ra.topics, - "languages": $ra.languages, + "languages": [ $lang ], + /* Not in the search response: the advisory is issued by + this answer, at this moment. */ "issuedAt": $now(), - /* This document's passages, joined: the engine returns up - to max_chunks_per_doc of them, and the same chunk can - arrive twice under two type labels. */ + /* This document's passages, joined. The engine returns up + to max_chunks_per_doc of them and the same chunk can + arrive twice under two type labels, hence $distinct. + + The guide holds that hits[].text is evidence and the + message must be synthesized upstream. Nothing in this + path synthesizes, and /search returns passages, so the + retrieved text IS the message here -- a deliberate + divergence, recorded rather than hidden. */ "recommendations": [{ - "message": $textOf($id), + "message": $join($distinct($h.text), "\n\n"), "language": $lang }], - /* The document this advisory is drawn from, as a resource - IDENTIFIER rather than the raw hash -- the pack calls - these "Beckn Resource identifiers of supporting - KnowledgeResource information". They resolve only if - those documents are themselves published as - KnowledgeResource resources, which today they are not, - so a caller can correlate but not dereference them. */ + /* This document only, as a resource IDENTIFIER rather than + the raw hash. doc_id never chunk_id: uniqueItems is true + and one document contributes several chunks. They resolve + only once those documents are themselves published as + KnowledgeResource resources, which today they are not. */ "supportingResourceIds": ["res:knowledge:" & $id], - "rationale": $rationale, - /* NOT hits[].source -- that is "docs-pipeline", the - ingestion pipeline, not the authority behind the - knowledge. */ - "source": { - "sourceId": "participant:bharat-vistaar", - "sourceName": $distinct($usable.instance_name)[0] - } - } - } - })] + "rationale": $exists($text($top.instance_name)) + ? $counts & " from the " & $text($top.instance_name) & " corpus." + : $counts & ".", + "source": $merge([ + { "sourceId": $sourceId }, + /* This document's own instance, not the first across all + of them. */ + $exists($text($top.instance_name)) + ? { "sourceName": $text($top.instance_name) } : {} + ]) + }, + /* Optional, so emitted only when the corpus yields one. */ + $exists($cats) ? { "subjectCategories": [ $cats ] } : {}, + /* REQUIRED in Direct mode, yet category_tags is blank across + this corpus and effective_config.query is the only other + source. When neither speaks, the key is OMITTED rather than + emitted as [] -- an empty array asserts "this advisory has + no topics", where absence says nothing was available to + say. Both fail the pack; only one of them is honest, and + the absence is what makes the gap visible. */ + $exists($topics) ? { "topics": [ $topics ] } : {} + /* agricultureSubjects is deliberately absent even though + scheme_code and scheme_name are real data here: + AgricultureSubjectReference requires subjectId, a taxonomy + URI retrieval does not supply. Emitting the array without + it would be invalid; inventing a base URI would be a guess. + coverageAreas is absent for the same reason -- it has no + counterpart in the response, and deriving it from a + scheme's name ("Telangana" -> IN-TG) is guesswork. */ + ]) + } + ) })] }] } } diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go index 4550f5f6..4e5a6e6b 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go @@ -73,11 +73,17 @@ const requestOfferID = "offer:knowledge-advisory:schemes" // fields the mapping reads. Four hits over TWO documents: the engine returns // several chunks per document, and this corpus indexes one chunk twice, once // as type "document" and once as "scheme". Note doc_language and -// category_tags come back BLANK -- which is why the mapping takes languages -// and topics from the request instead. +// category_tags come back BLANK on every hit, so the mapping derives topics +// from effective_config.query and falls back to the guide's "en" for language. const providerResponse = `{ "candidate_count": 51, "final_count": 4, + "effective_config": { + "index_name": "test_doc_pipeline", + "query": "gruha jyoti scheme eligibility", + "max_chunks_per_doc": 2, + "exclude_reference": true + }, "hits": [ { "_id": "26860172-2762-4b0b-cabf-3c947846c2e1", @@ -338,23 +344,30 @@ func TestShippedMappingNeverBorrowsAnotherDocumentsName(t *testing.T) { } } -// topics and the recommendation language come from the REQUEST. This corpus -// returns category_tags and doc_language BLANK on every hit, so reading them -// from upstream would emit an advisory with an empty language -- which the -// pack requires to be at least two characters. -func TestShippedMappingTakesTopicsAndLanguageFromTheRequest(t *testing.T) { +// topics and language are DERIVED FROM THE RESPONSE, never echoed from the +// request. This corpus returns category_tags and doc_language blank on every +// hit, so topics falls back to effective_config.query -- the engine reporting +// the query it actually ran -- and language to the implementation guide's "en". +// A value copied from the request would read as provider data in the answer, +// and a consumer could not tell the difference. +func TestShippedMappingDerivesTopicsAndLanguageFromTheResponse(t *testing.T) { _, answer := runShipped(t, selectRequest) attrs := attributesOf(t, answer, 0) topics, _ := attrs["topics"].([]any) if len(topics) != 1 || topics[0] != "gruha jyoti scheme eligibility" { - t.Errorf("topics = %v, want the caller's own", topics) + t.Errorf("topics = %v, want effective_config.query", topics) + } + if langs, _ := attrs["languages"].([]any); len(langs) != 1 || langs[0] != "en" { + t.Errorf("languages = %v, want the derived [en]", langs) } recs, _ := attrs["recommendations"].([]any) + if len(recs) == 0 { + t.Fatal("no recommendations") + } for i, r := range recs { - lang := r.(map[string]any)["language"] - if lang != "en" { - t.Errorf("recommendation %d language = %v, want the request's en", i, lang) + if lang := r.(map[string]any)["language"]; lang != "en" { + t.Errorf("recommendation %d language = %v, want en", i, lang) } if msg, _ := r.(map[string]any)["message"].(string); strings.TrimSpace(msg) == "" { t.Errorf("recommendation %d carries no message", i) @@ -362,9 +375,19 @@ func TestShippedMappingTakesTopicsAndLanguageFromTheRequest(t *testing.T) { } } -// source names the PARTICIPANT, never the provider's own `source` field -- -// that one reads "docs-pipeline", the ingestion pipeline, not the authority -// behind the knowledge. +// With no source for topics at all -- category_tags blank AND no +// effective_config.query -- the key is omitted rather than emitted as an empty +// array. [] would assert "this advisory covers no topics"; absence reports that +// nothing was available to say. +func TestShippedMappingOmitsTopicsWhenNothingSuppliesThem(t *testing.T) { + _, answer := runShippedWith(t, selectRequest, + `{"candidate_count":1,"final_count":1,"hits":[ + {"doc_id":"d1","_score":0.5,"text":"some passage","instance_name":"Bharat Vistaar"}]}`) + attrs := attributesOf(t, answer, 0) + if v, present := attrs["topics"]; present { + t.Errorf("topics = %v, want the key absent when no source supplies it", v) + } +} func TestShippedMappingNamesTheParticipantAsTheSource(t *testing.T) { _, answer := runShipped(t, selectRequest) attrs := attributesOf(t, answer, 0) @@ -388,7 +411,7 @@ func TestShippedMappingNamesTheParticipantAsTheSource(t *testing.T) { // one anyway produced. func TestShippedMappingAnswersAnEmptyResultWithNoResources(t *testing.T) { _, answer := runShippedWith(t, selectRequest, - `{"candidate_count":0,"final_count":0,"hits":[]}`) + `{"candidate_count":0,"final_count":0,"effective_config":{"query":"gruha jyoti scheme eligibility"},"hits":[]}`) if res := resourcesOf(t, answer); len(res) != 0 { t.Errorf("resources = %d, want 0 for a query nothing matched", len(res)) From faa93895446616cc8e9dbbc7e46fa2959402fca5 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 12:10:46 +0530 Subject: [PATCH 099/130] fix(mappings): retain the selected resource id in the offer [#7] The implementation guide keeps the OnDemand id the caller selected in offer.resourceIds alongside the Direct id minted here, so the caller can correlate the callback. Commit a890913 removed it on the grounds that it references a resource appearing nowhere in resources[]; that is true and deliberate -- OnDemand and Direct require different fields, so the answer cannot restate the selected resource, and the caller resolves that id against its own request rather than against this payload. $distinct, as in the guide, so a re-selected advisory cannot appear twice. On an empty result the offer now names just the question it was asked, with resources[] empty, rather than referencing nothing at all. --- .../knowledge/knowledge-advisory.select.yaml | 19 ++++--- .../KnowledgeAdvisory/mappings_test.go | 52 +++++++++++-------- 2 files changed, 43 insertions(+), 28 deletions(-) diff --git a/config/mappings/knowledge/knowledge-advisory.select.yaml b/config/mappings/knowledge/knowledge-advisory.select.yaml index ddc06846..01df49d1 100644 --- a/config/mappings/knowledge/knowledge-advisory.select.yaml +++ b/config/mappings/knowledge/knowledge-advisory.select.yaml @@ -58,6 +58,7 @@ response: | $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; $ctx := $ra.`@context`; $offer := beckn.message.contract.commitments[0].offer; + $sel := beckn.message.contract.commitments[0].resources[0].id; /* DEPLOYMENT CONSTANT, like index_name in the request half: the network-stable participant id of the authority behind this corpus. @@ -119,14 +120,20 @@ response: | by inventing a schema. */ "commitments": [{ "status": { "descriptor": { "code": "DRAFT", "name": "Draft" } }, - /* The offer is echoed, but its references are not: the request - named the OnDemand resource -- the question -- while the answer - returns the advisories. The selected id is NOT retained; it - resolves to nothing in resources[], and context.transactionId - already correlates the callback. With no hits this is []. */ + /* The offer is echoed; its references are rebuilt as the guide + prescribes -- the OnDemand id the caller selected, RETAINED so it + can correlate the callback, plus the Direct id minted per + document. OnDemand and Direct require different fields, so the + answer cannot restate the selected resource; the caller resolves + that id against its own request, not against resources[] here. + + $distinct because a re-selected advisory would otherwise appear + twice. With no hits this is just the selected id, and resources[] + is [] -- the offer still names the question it was asked. */ "offer": $merge([ $offer, - { "resourceIds": [$map($docIds, function($id) { "res:knowledge-advisory:" & $id })] } + { "resourceIds": $distinct($append([$sel], + [$map($docIds, function($id) { "res:knowledge-advisory:" & $id })])) } ]), "resources": [$map($docIds, function($id) { ( $h := $hitsOf($id); diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go index 4e5a6e6b..9ca0dd48 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go @@ -69,6 +69,10 @@ const selectRequest = `{ // The offer id selectRequest carries. The answer echoes it: one offer, one id. const requestOfferID = "offer:knowledge-advisory:schemes" +// The OnDemand resource id selectRequest selects. The answer retains it in +// offer.resourceIds so the caller can correlate the callback. +const selectedResourceID = "res:knowledge-advisory:schemes:ondemand" + // providerResponse is captured from the live retrieval service, trimmed to the // fields the mapping reads. Four hits over TWO documents: the engine returns // several chunks per document, and this corpus indexes one chunk twice, once @@ -419,18 +423,19 @@ func TestShippedMappingAnswersAnEmptyResultWithNoResources(t *testing.T) { commitments := answer["message"].(map[string]any)["contract"].(map[string]any)["commitments"].([]any) offer := commitments[0].(map[string]any)["offer"].(map[string]any) ids, _ := offer["resourceIds"].([]any) - if len(ids) != 0 { - t.Fatalf("offer.resourceIds = %v, want none: the offer covers no resource", ids) + if len(ids) != 1 || fmt.Sprint(ids[0]) != selectedResourceID { + t.Fatalf("offer.resourceIds = %v, want just the selected id %q", ids, selectedResourceID) } } -// The offer is echoed and its references are rebuilt. resourceIds are -// "references to resources covered by this offer", so every id has to resolve -// inside resources[] -- and the OnDemand id the caller selected must NOT -// appear, because the answer cannot restate it: OnDemand and Direct require -// different fields. Correlation is context.transactionId's job, not this -// field's. -func TestShippedMappingReferencesOnlyTheResourcesItReturns(t *testing.T) { +// The offer is echoed and its references are rebuilt, per the implementation +// guide: the OnDemand id the caller selected is RETAINED so it can correlate +// the callback, alongside the Direct id minted for each document. The selected +// id deliberately does not resolve inside resources[] -- OnDemand and Direct +// require different fields, so the answer cannot restate it, and the caller +// matches it against its own request. $distinct guards a re-selected advisory +// appearing twice. +func TestShippedMappingRetainsTheSelectedIdAlongsideTheMintedOnes(t *testing.T) { _, answer := runShipped(t, selectRequest) commitments := answer["message"].(map[string]any)["contract"].(map[string]any)["commitments"].([]any) @@ -443,22 +448,17 @@ func TestShippedMappingReferencesOnlyTheResourcesItReturns(t *testing.T) { ids, _ := offer["resourceIds"].([]any) res := resourcesOf(t, answer) - if len(ids) != len(res) { - t.Fatalf("offer.resourceIds = %v, want exactly one per advisory (%d)", ids, len(res)) - } - - returned := map[string]bool{} - for _, r := range res { - returned[fmt.Sprint(r.(map[string]any)["id"])] = true + if len(ids) != len(res)+1 { + t.Fatalf("offer.resourceIds = %v, want the selected id plus one per advisory (%d)", ids, len(res)) } - for _, id := range ids { - if !returned[fmt.Sprint(id)] { - t.Errorf("offer references %q, which is not in resources[] -- a caller cannot resolve it", id) - } + if fmt.Sprint(ids[0]) != selectedResourceID { + t.Errorf("offer.resourceIds[0] = %v, want the selected id %q", ids[0], selectedResourceID) } - for want := range returned { + // Every minted advisory must appear, or a caller cannot resolve it. + for i := range res { + want := fmt.Sprint(res[i].(map[string]any)["id"]) found := false - for _, got := range ids { + for _, got := range ids[1:] { if fmt.Sprint(got) == want { found = true } @@ -467,4 +467,12 @@ func TestShippedMappingReferencesOnlyTheResourcesItReturns(t *testing.T) { t.Errorf("resource %q is missing from offer.resourceIds %v", want, ids) } } + // No duplicates: $distinct has to be doing its job. + seen := map[string]bool{} + for _, id := range ids { + if seen[fmt.Sprint(id)] { + t.Errorf("offer.resourceIds repeats %v", id) + } + seen[fmt.Sprint(id)] = true + } } From a23eee05d2a2104ce6f725a9e44ef86669dca587 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 12:35:17 +0530 Subject: [PATCH 100/130] feat(mappings): emit one advisory per hit, sourced from its own document [#7] Three changes to the answer's shape, all narrowing what a resource may claim. ONE RESOURCE PER HIT, keyed on hits[]._id, replacing one per doc_id. The live corpus returns four hits carrying four DISTINCT passages -- no two share a text checksum -- so collapsing by document was discarding content. _id rather than doc_id identifies a resource because two hits can share a doc_id and still be separate passages from different ingestion runs, which the live data shows: the same doc_id arrives under two filenames. source now names the DOCUMENT rather than a participant: sourceId <- hits[].doc_id sourceName <- hits[].filename That removes the last invented value in this file -- sourceId was a hardcoded participant string derived from nothing upstream. The two fields that look like better candidates stay unmapped, and the implementation guide says why: hits[].instance is a provider-internal code and hits[].source names the ingestion pipeline. offer.resourceIds no longer retains the OnDemand id the caller selected, so every reference resolves inside resources[]. _id and filename are both on the guide's never-mapped list. They are mapped here deliberately: _id is the only per-hit identifier, and filename the only per-hit label for the document a passage came from. Every label a resource carries is now read from its own hit -- descriptor, language, topics, subjectCategories and source alike -- so nothing is pooled across hits and no advisory can describe itself with another's metadata. All four validate against the published KnowledgeAdvisory pack under a validator that does evaluate if/then. --- .../knowledge/knowledge-advisory.select.yaml | 225 ++++++++---------- .../KnowledgeAdvisory/mappings_test.go | 202 ++++++++-------- 2 files changed, 212 insertions(+), 215 deletions(-) diff --git a/config/mappings/knowledge/knowledge-advisory.select.yaml b/config/mappings/knowledge/knowledge-advisory.select.yaml index 01df49d1..9f1a3386 100644 --- a/config/mappings/knowledge/knowledge-advisory.select.yaml +++ b/config/mappings/knowledge/knowledge-advisory.select.yaml @@ -42,31 +42,30 @@ request: | # Response half derived from the OpenAgriNet non-normative implementation guide -# for this capability (knowledge-mapper/on_select.jsonata). Its derivation -# rules are followed; its INPUT SHAPE is not, because it reads a hand-built -# { search, pn } object and this adapter supplies { beckn, _local, response }. +# for this capability (knowledge-mapper/on_select.jsonata). Its per-field +# derivation rules are followed; three things deliberately are not. # -# Divergences from the guide, each deliberate and marked at its site: -# - one Resource PER DOCUMENT, not one for the top-scoring hit -# - hits[].text IS the recommendation message; nothing here synthesizes -# - no $assert: an empty result is an answer, not an error -# - senderId/receiverId, not bapId/bppId, per the runtime Beckn spec -# - `res:` id prefix, matching the other OAN capabilities -# - Contract.consideration omitted; see the note at the emit block +# INPUT SHAPE. The guide reads a hand-built { search, pn }; this adapter +# supplies { beckn, _local, response }, and there is no `pn` to supply the +# values it treats as Provider responsibility. +# +# ONE RESOURCE PER HIT, keyed on hits[]._id. The guide emits a single +# Resource for the top-scoring hit. Per-hit is what the network wants here: +# in the live corpus the four hits carry four DISTINCT passages (no two share +# a text checksum), so collapsing them loses content. +# +# hits[]._id and hits[].filename ARE mapped, though the guide lists both as +# never-mapped. _id is the only per-hit identifier, and filename is the only +# per-hit human label for the document a passage came from. +# +# Also divergent, each marked at its site: hits[].text IS the recommendation +# message, since nothing in this path synthesizes; no $assert, because an empty +# result is an answer; Contract.consideration omitted. response: | ( $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; $ctx := $ra.`@context`; $offer := beckn.message.contract.commitments[0].offer; - $sel := beckn.message.contract.commitments[0].resources[0].id; - - /* DEPLOYMENT CONSTANT, like index_name in the request half: the - network-stable participant id of the authority behind this corpus. - SourceReference.sourceId is required and retrieval cannot supply it. - Two upstream fields look like candidates and are traps: - hits[].instance "bv" provider-internal, collides across PNs - hits[].source "docs-pipeline" the ingestion pipeline, not the authority */ - $sourceId := "participant:bharat-vistaar"; /* This API returns "" for absent strings -- doc_language, category_tags, title_en, doc_short_description and description are all "" on every hit @@ -90,17 +89,15 @@ response: | /* Every hit the provider returned, less its own reference chunks. It ran the query and chose what to send; applying a score floor here would be this adapter overriding the provider's relevance judgement. The guide - argues for a floor (unrelated documents can land within 0.005 of the top + argues for one (unrelated documents can land within 0.005 of the top hit) -- that is one $filter to reinstate if a deployment wants it. */ $usable := $filter(response.hits, function($h) { $h.is_reference != true }); - /* ONE ADVISORY PER DOCUMENT. doc_id repeats: the engine returns up to - max_chunks_per_doc chunks per document, and this corpus also indexes one - chunk twice, as type "document" and again as type "scheme". */ - $docIds := $distinct($usable.doc_id); - $hitsOf := function($id) { $filter($usable, function($h) { $h.doc_id = $id }) }; + /* The query the engine reports having run. Sole fallback for topics, which + Direct mode requires: category_tags is "" throughout this corpus. */ + $query := $text(response.effective_config.query); - /* Counts are whole-response facts, so the rationale reads the same on every + /* Counts are whole-response facts, so this reads the same on every advisory. It cites aggregate provenance only -- naming the index, the ranking method or its parameters would leak engine internals. */ $counts := "Retrieved " & $string($count($usable)) & " of " @@ -115,84 +112,74 @@ response: | priced quote, but its own caveat is the reason to leave it out: Beckn v2.0.0 never DEFINES beckn:PriceSpecification, so any considerationAttributes shape here is unverifiable. Contract - requires only `commitments`, so omitting it is valid, and an - advisory served at no charge is better said by saying nothing than - by inventing a schema. */ + requires only `commitments`, so omitting it is valid. */ "commitments": [{ "status": { "descriptor": { "code": "DRAFT", "name": "Draft" } }, - /* The offer is echoed; its references are rebuilt as the guide - prescribes -- the OnDemand id the caller selected, RETAINED so it - can correlate the callback, plus the Direct id minted per - document. OnDemand and Direct require different fields, so the - answer cannot restate the selected resource; the caller resolves - that id against its own request, not against resources[] here. - - $distinct because a re-selected advisory would otherwise appear - twice. With no hits this is just the selected id, and resources[] - is [] -- the offer still names the question it was asked. */ + /* The offer is echoed; its references are rebuilt to name exactly + the resources this answer carries. The OnDemand id the caller + selected is NOT retained -- it resolves to nothing in + resources[], and context.transactionId already correlates the + callback. With no hits this is [], matching resources[]. */ "offer": $merge([ $offer, - { "resourceIds": $distinct($append([$sel], - [$map($docIds, function($id) { "res:knowledge-advisory:" & $id })])) } + { "resourceIds": [$map($usable, function($h) { $h.`_id` })] } ]), - "resources": [$map($docIds, function($id) { ( - $h := $hitsOf($id); - - /* Per document, the metadata-bearing hit: the scheme-typed one if - this document has it, else the highest scoring. _score orders - only and is never emitted. Without the first clause a document - indexed both ways loses its scheme_name to an untyped chunk. */ - $sch := $h[$exists(scheme_code)]; - $top := $exists($sch) ? $sch[0] : ($h^(>`_score`))[0]; - + /* ONE RESOURCE PER HIT. Each passage stands on its own: its id is + that hit's _id, its message that hit's text, and every label it + carries is read from that same hit. Nothing is pooled across + hits, so no resource can describe itself with another's + metadata. + + NO RESULTS IS AN ANSWER: with nothing to map this is [], which + Commitment.resources allows -- required, but with no minItems. */ + "resources": [$map($usable, function($h) { ( /* doc_language, then the guide's "en" default. recommendations[] - requires language with minLength 2, so there is no honest empty - to fall back to -- the corpus leaves it blank on every hit. */ - $lang := $exists($text($top.doc_language)) ? $text($top.doc_language) : "en"; - - /* category_tags is a COMMA-JOINED STRING, not an array, and is "" - throughout this corpus. Falling back to the query the engine - reports having run keeps topics sourced from the response - rather than echoed from the request. topics is required in - Direct mode with minItems 1. */ - $tags := $distinct($h.category_tags[$ != ""].$split($, ",").$trim($)[$ != ""]); - $query := $text(response.effective_config.query); + requires language with minLength 2, and this corpus leaves the + field blank on every hit, so there is no honest empty here. */ + $lang := $exists($text($h.doc_language)) ? $text($h.doc_language) : "en"; + + /* category_tags is a COMMA-JOINED STRING, not an array. Blank + throughout this corpus, so the engine's own reported query is + what keeps topics sourced from the response rather than + echoed from the request. */ + $tags := $distinct($split($text($h.category_tags), ",").$trim($)[$ != ""]); $topics := $exists($tags) ? $tags : ($exists($query) ? [ $query ] : undefined); - $cats := $distinct($h.$categoryOf(type)); + $cat := $categoryOf($h.type); /* scheme_name, then the human titles, then name_en/name_gu LAST: - this corpus fills those with the ingestion filename slug - ("Praja-Palana-Application-Form"), which reads badly to a - farmer. Only this document's hits are consulted -- borrowing - another document's name is what made an earlier shape lie. */ - $label := $exists($text($top.scheme_name)) ? $text($top.scheme_name) - : $exists($text($top.title_en)) ? $text($top.title_en) - : $exists($text($top.title_gu)) ? $text($top.title_gu) - : $exists($text($top.name_en)) ? $text($top.name_en) - : $exists($text($top.name_gu)) ? $text($top.name_gu) + this corpus fills those with the ingestion filename slug, + which reads badly to a farmer. This hit's own values only. */ + $label := $exists($text($h.scheme_name)) ? $text($h.scheme_name) + : $exists($text($h.title_en)) ? $text($h.title_en) + : $exists($text($h.title_gu)) ? $text($h.title_gu) + : $exists($text($h.name_en)) ? $text($h.name_en) + : $exists($text($h.name_gu)) ? $text($h.name_gu) : undefined; { - "id": "res:knowledge-advisory:" & $id, + /* The hit's own identifier. Unique per passage -- verified + distinct across every hit in the live corpus, where two hits + can share a doc_id yet come from different ingestion runs. */ + "id": $h.`_id`, /* Required by Commitment.resources, which declares no quantity property and carries no Quantity schema at all -- a defect upstream. One resource is one advisory. */ "quantity": 1, "descriptor": $merge([ { - "code": $exists($text($top.scheme_code)) - ? $toCode($text($top.scheme_code)) & "_ADVISORY" + "code": $exists($text($h.scheme_code)) + ? $toCode($text($h.scheme_code)) & "_ADVISORY" : "KNOWLEDGE_ADVISORY" }, - $exists($label) ? { "name": $label & " advisory" } : {}, - /* Blank on every hit here, so $text drops them both today. */ - $exists($text($top.doc_short_description)) - ? { "shortDesc": $text($top.doc_short_description) } : {}, - $exists($text($top.doc_llm_description)) - ? { "longDesc": $text($top.doc_llm_description) } - : ($exists($text($top.description)) - ? { "longDesc": $text($top.description) } : {}) + $exists($label) ? { "name": $label & " advisory" } : {}, + /* Blank on every hit here, so both drop out today. */ + $exists($text($h.doc_short_description)) + ? { "shortDesc": $text($h.doc_short_description) } : {}, + $exists($text($h.doc_llm_description)) + ? { "longDesc": $text($h.doc_llm_description) } + : ($exists($text($h.description)) + ? { "longDesc": $text($h.description) } : {}) ]), "resourceAttributes": $merge([ { @@ -205,54 +192,52 @@ response: | /* Not in the search response: the advisory is issued by this answer, at this moment. */ "issuedAt": $now(), - /* This document's passages, joined. The engine returns up - to max_chunks_per_doc of them and the same chunk can - arrive twice under two type labels, hence $distinct. - - The guide holds that hits[].text is evidence and the - message must be synthesized upstream. Nothing in this - path synthesizes, and /search returns passages, so the - retrieved text IS the message here -- a deliberate - divergence, recorded rather than hidden. */ + /* This passage, as retrieved. The guide holds that + hits[].text is evidence and the message must be + synthesized upstream; nothing in this path synthesizes + and /search returns passages, so the retrieved text IS + the message here -- a divergence recorded, not hidden. */ "recommendations": [{ - "message": $join($distinct($h.text), "\n\n"), + "message": $h.text, "language": $lang }], - /* This document only, as a resource IDENTIFIER rather than - the raw hash. doc_id never chunk_id: uniqueItems is true - and one document contributes several chunks. They resolve - only once those documents are themselves published as - KnowledgeResource resources, which today they are not. */ - "supportingResourceIds": ["res:knowledge:" & $id], - "rationale": $exists($text($top.instance_name)) - ? $counts & " from the " & $text($top.instance_name) & " corpus." + /* The document this passage was drawn from, as a resource + IDENTIFIER rather than the raw hash. doc_id never + chunk_id. It resolves only once those documents are + themselves published as KnowledgeResource resources, + which today they are not. */ + "supportingResourceIds": ["res:knowledge:" & $h.doc_id], + "rationale": $exists($text($h.instance_name)) + ? $counts & " from the " & $text($h.instance_name) & " corpus." : $counts & ".", + /* The document behind this passage IS its source: + doc_id identifies it, filename is its human label. Two + upstream fields that look like better candidates are + traps, and the guide says so -- hits[].instance is a + provider-internal code ("bv") and hits[].source names + the ingestion pipeline ("docs-pipeline"). */ "source": $merge([ - { "sourceId": $sourceId }, - /* This document's own instance, not the first across all - of them. */ - $exists($text($top.instance_name)) - ? { "sourceName": $text($top.instance_name) } : {} + { "sourceId": $h.doc_id }, + $exists($text($h.filename)) + ? { "sourceName": $text($h.filename) } : {} ]) }, - /* Optional, so emitted only when the corpus yields one. */ - $exists($cats) ? { "subjectCategories": [ $cats ] } : {}, + /* Optional, so emitted only when this hit's type maps. */ + $exists($cat) ? { "subjectCategories": [ $cat ] } : {}, /* REQUIRED in Direct mode, yet category_tags is blank across - this corpus and effective_config.query is the only other - source. When neither speaks, the key is OMITTED rather than + this corpus and the reported query is the only other + source. When neither speaks the key is OMITTED rather than emitted as [] -- an empty array asserts "this advisory has no topics", where absence says nothing was available to - say. Both fail the pack; only one of them is honest, and - the absence is what makes the gap visible. */ + say. Both fail the pack; only one is honest, and the + absence is what makes the gap visible. + + agricultureSubjects stays absent although scheme_code and + scheme_name are real data: AgricultureSubjectReference + requires a subjectId taxonomy URI nothing here supplies. + coverageAreas has no counterpart at all, and deriving it + from a scheme's name ("Telangana" -> IN-TG) is guesswork. */ $exists($topics) ? { "topics": [ $topics ] } : {} - /* agricultureSubjects is deliberately absent even though - scheme_code and scheme_name are real data here: - AgricultureSubjectReference requires subjectId, a taxonomy - URI retrieval does not supply. Emitting the array without - it would be invalid; inventing a base URI would be a guess. - coverageAreas is absent for the same reason -- it has no - counterpart in the response, and deriving it from a - scheme's name ("Telangana" -> IN-TG) is guesswork. */ ]) } ) })] diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go index 9ca0dd48..ac99483f 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go @@ -91,6 +91,7 @@ const providerResponse = `{ "hits": [ { "_id": "26860172-2762-4b0b-cabf-3c947846c2e1", + "filename": "doc-379e706ade8b-rerun-1788847883", "_score": 0.77918214, "doc_id": "74b89b5f69dcd56f6566d4b1a9392ffd", "text": "![House icon](a2bc7bb7c74e387ff445bcc6cf9b2a5a_1_img.webp)\n# Gruha Jyoti Scheme:\n1. 200 units of free electricity per family per month\n1.1. Your monthly household electricity consumption:\n0-100 Units\n100-200 Units\nAbove 200 Units\n1.2. Household consumption electricity meter connection number _____\n![Hand icon](a2bc7bb7c74e387ff445bcc6cf9b2a5a_7_img.webp)\n# Chayata Scheme:\nTo receive Rs.4000 per month under Chayata Scheme and Rs.6000 Disability Pension, provide the following details.\n( Those currently receiving pension do not need to apply )\n1. Disability Certificate Number: _____\n2. Others:\nOld Age\nWidow\nWeavers\nCotton Workers\nDiabetics\nAIDS Affected\nBeedi Workers' Livelihood Pension\nFilariasis Affected\nSingle Woman Livelihood Pension\nBeedi Contractor Livelihood Pension\nPath of Progress.. Welfare of All.. Our People's Government\n![Government of Telangana logo](a2bc7bb7c74e387ff445bcc6cf9b2a5a_16_img.webp)\nGovernment of Telangana\n![Circular graphic with Nirmala Sitharaman's portrait and icons for various schemes: Mahalaxmi, Rashtra Janjane, Grahajyoti, Chayata, Yuvavikasan, and Indiramandir](a2bc7bb7c74e387ff445bcc6cf9b2a5a_18_img.webp)![Portrait of Nirmala Sitharaman and N. Chandrababu Naidu](a2bc7bb7c74e387ff445bcc6cf9b2a5a_19_img.webp)\nPeople's Governance\nFrom 28.12.2023 to 06.01.2024", @@ -102,6 +103,7 @@ const providerResponse = `{ }, { "_id": "0e753c2b-040a-4af9-3dc4-c9fca3b01e4d", + "filename": "doc-379e706ade8b-rerun-1788785422", "_score": 0.7783708, "doc_id": "74b89b5f69dcd56f6566d4b1a9392ffd", "text": "![House icon](71c6c13ee057c1204dd8980597d79d5b_1_img.webp)\n# Gruha Jyoti Scheme:\n1. 200 units of free electricity per family per month\n1.1. Your monthly household electricity consumption:\n0-100 Units\n100-200 Units\nAbove 200 Units\n1.2. Household consumption electricity meter connection number _____\n![Hand icon](71c6c13ee057c1204dd8980597d79d5b_7_img.webp)\n# Chayatha Scheme:\nTo receive Rs.4000 per month under Chayatha Scheme and Rs.6000 pension for disabled persons, provide the following details.\n(Those currently receiving pension do not need to apply)\n1. Disabled person certificate number: _____\n2. Others:\nOld Age\nWidow\nGita Workers\nAgricultural Laborers\nDiabetes Affected\nAIDS Affected\nBeedi Workers' Livelihood Pension\nFilaria Affected\nBantali Women Livelihood Pension\nBeedi Contractors' Livelihood Pension\nPath of Progress.. Welfare of All.. Our People's Government\n![Government of Telangana logo](71c6c13ee057c1204dd8980597d79d5b_16_img.webp)\nGovernment of Telangana\n![Circular graphic with Nirmala Sitharaman's portrait and icons for various schemes: Mahalaxmi, Rashtra Janjane, Grahajyoti, Chayath, Yuvavikasan, and Indiramandir](71c6c13ee057c1204dd8980597d79d5b_18_img.webp)![Portrait of Nirmala Sitharaman and N. Chandrababu Naidu](71c6c13ee057c1204dd8980597d79d5b_19_img.webp)\nPeople's Governance\nFrom 28.12.2023 to 06.01.2024", @@ -115,6 +117,7 @@ const providerResponse = `{ }, { "_id": "b9ec5c75-7331-0424-f741-4b3294c6b75e", + "filename": "doc-d7124029cb60-rerun-1788841718", "_score": 0.77453613, "doc_id": "2cc58f8a1c39a64fc132338be3547f52", "text": "- (d) Under the Anna Bhagya Scheme, from which state is the rice being brought; what is the rate of this rice; what is the transportation and other cost? (Details to be provided)\n- (e) Following what standard has the rice brought from outside the state been entrusted for transportation; and which company/agency is currently transporting the rice?\n- (f) At what rate is the levy quantity of rice being collected in the state for the Anna Bhagya Scheme being purchased; for how many months is this collected rice sufficient? (Detailed information to be provided)\n**Smart City Scheme**\n**Shri Kavatagi Matha Mahantesh Mallikarjuna (Local Institutions Department):-**\n1053 (1274) Will the Hon'ble Minister of Urban Development kindly inform the following matters :-\n- (a) Has the fact that the ambitious Smart City Scheme of the Central Government is not being implemented properly in the state come to the government's notice;\n- (b) If it has come to notice, what steps has the government taken to implement the Smart City Scheme in the state and complete it at a rapid pace; for the delay", @@ -126,6 +129,7 @@ const providerResponse = `{ }, { "_id": "4c6466a5-6975-a3be-4a62-637ae1a611d3", + "filename": "doc-d7124029cb60-rerun-1788841718", "_score": 0.7743588, "doc_id": "2cc58f8a1c39a64fc132338be3547f52", "text": "**Regarding the 'Jalashree' Scheme being implemented by KUIDFC in the State, Shri N. Appajigaoud (Local Institutions Department) :-**\n1031 (1244) Hon'ble Urban Development Minister, please inform on these matters :-\n- (a) What are the urban areas selected under the 24 x 7 drinking water supply scheme under the 'Jalashree' scheme being implemented by KUIDFC in the State, and what is the financial assistance allocated for their implementation; (Provide complete information)\n- (b) On what criteria were the urban areas included in this scheme selected; (Provide details)\n- (c) At what stage is the implementation of works in the urban areas selected under this scheme; (Provide complete information)\n- (d) Many urban areas selected under this scheme have been taken up under the drinking water development program under the Central Government's 'AMRUT' scheme, is it correct to include these urban areas again under the 'Jalashree' scheme; (Provide complete information)\n\n(d) Has the Government considered including the taluks in the districts where drought occurs frequently and the groundwater level is depleting in the state under the earlier 'Jalashree' scheme; if so, provide complete information?\n**Regarding Industrial Clusters in the state and industries established therein, Shri N. Appajigaoud (Local Bodies Department) :-**\n1032 (1248) Hon'ble Minister of Large and Medium Industries, please inform on these matters :-\n- (a) How many Industrial Clusters are there in the state; (Provide district-wise information including area)\n- (b) Out of these Industrial Clusters, in how many clusters has the Automobile industry been established; Provide complete information for each cluster including factories/enterprises;\n- (c) In several Small and Medium Industrial Estates located in the state for several decades such as Peethya Industrial Estate, Bidadi Industrial Estate, Mysore-Hebbal Industrial Estate, Belur-Hubballi Industrial Estate, etc., due to recent economic recession", @@ -283,77 +287,73 @@ func TestShippedMappingAnswersWithADirectAdvisory(t *testing.T) { } } -// ONE recommendation per DOCUMENT, not per hit. Four hits arrive over two -// documents, because the engine returns several chunks per document and this -// corpus indexes one chunk twice. Per-hit would say the same thing twice. -func TestShippedMappingCollapsesHitsToDocuments(t *testing.T) { +// ONE RESOURCE PER HIT, keyed on hits[]._id. Four hits carry four distinct +// passages in this corpus -- no two share a text checksum -- so each becomes +// its own advisory. Two hits can share a doc_id and still be separate +// passages from different ingestion runs, which is why _id and not doc_id +// identifies a resource. +func TestShippedMappingEmitsOneResourcePerHit(t *testing.T) { _, answer := runShipped(t, selectRequest) res := resourcesOf(t, answer) - if len(res) != 2 { - t.Fatalf("resources = %d, want 2 -- one per document, from 4 hits", len(res)) + if len(res) != 4 { + t.Fatalf("resources = %d, want 4 -- one per hit", len(res)) } - ids := map[string]bool{} + + wantIDs := []string{ + "26860172-2762-4b0b-cabf-3c947846c2e1", + "0e753c2b-040a-4af9-3dc4-c9fca3b01e4d", + "b9ec5c75-7331-0424-f741-4b3294c6b75e", + "4c6466a5-6975-a3be-4a62-637ae1a611d3", + } + got := map[string]bool{} + for i := range res { + got[fmt.Sprint(res[i].(map[string]any)["id"])] = true + } + for _, want := range wantIDs { + if !got[want] { + t.Errorf("no resource carries hit id %q", want) + } + } + + // Every advisory holds exactly its own passage, and they are all different. + messages := map[string]bool{} for i := range res { attrs := attributesOf(t, answer, i) recs, _ := attrs["recommendations"].([]any) if len(recs) != 1 { - t.Errorf("advisory %d carries %d recommendations, want 1 -- its own document's", i, len(recs)) - } - // supportingResourceIds declares uniqueItems. One document per - // advisory means one id, and it must be that advisory's own. - sup, _ := attrs["supportingResourceIds"].([]any) - if len(sup) != 1 { - t.Errorf("advisory %d cites %d documents, want 1", i, len(sup)) + t.Fatalf("resource %d has %d recommendations, want exactly 1", i, len(recs)) } - id := fmt.Sprint(res[i].(map[string]any)["id"]) - if ids[id] { - t.Errorf("two advisories share the id %q", id) + msg, _ := recs[0].(map[string]any)["message"].(string) + if strings.TrimSpace(msg) == "" { + t.Errorf("resource %d carries no message", i) } - ids[id] = true - // The resource id and the document it cites name the same document. - if len(sup) == 1 && !strings.HasSuffix(id, strings.TrimPrefix(fmt.Sprint(sup[0]), "res:knowledge:")) { - t.Errorf("advisory %q cites %v, which is a different document", id, sup[0]) + if messages[msg] { + t.Errorf("resource %d repeats another advisory's passage", i) } + messages[msg] = true } } -// AN ADVISORY MUST NOT BORROW ANOTHER DOCUMENT'S NAME. The first shape here -// was one resource holding every document's text, which took its descriptor -// from whichever hit happened to carry a scheme_name -- so on this fixture it -// came back named "Telangana Gruha Jyoti" while carrying a recommendation from -// an unrelated Karnataka legislative document. Per document, a name can only -// come from the document it names. -func TestShippedMappingNeverBorrowsAnotherDocumentsName(t *testing.T) { +// No advisory may take a label from a hit other than its own. Only the +// scheme-typed hit carries a scheme_name, so exactly one advisory may show it; +// anything else naming itself "Telangana Gruha Jyoti" is borrowing. +func TestShippedMappingNeverBorrowsAnotherHitsName(t *testing.T) { _, answer := runShipped(t, selectRequest) res := resourcesOf(t, answer) - named := 0 + scheme := 0 for i := range res { d, _ := res[i].(map[string]any)["descriptor"].(map[string]any) name, _ := d["name"].(string) - if name == "" { - continue - } - named++ - // Only the scheme-typed hit carries a scheme_name, and it belongs to - // the first document. Any other advisory claiming a name is borrowing. - id := fmt.Sprint(res[i].(map[string]any)["id"]) - if !strings.Contains(id, "74b89b5f69dcd56f6566d4b1a9392ffd") { - t.Errorf("advisory %q is named %q, but only the other document has a scheme name", id, name) + if strings.Contains(name, "Telangana Gruha Jyoti") { + scheme++ } } - if named != 1 { - t.Errorf("%d advisories carry a name, want 1 -- exactly one document has a scheme_name", named) + if scheme != 1 { + t.Errorf("%d advisories claim the scheme name, want 1 -- only one hit carries scheme_name", scheme) } } - -// topics and language are DERIVED FROM THE RESPONSE, never echoed from the -// request. This corpus returns category_tags and doc_language blank on every -// hit, so topics falls back to effective_config.query -- the engine reporting -// the query it actually ran -- and language to the implementation guide's "en". -// A value copied from the request would read as provider data in the answer, -// and a consumer could not tell the difference. func TestShippedMappingDerivesTopicsAndLanguageFromTheResponse(t *testing.T) { _, answer := runShipped(t, selectRequest) attrs := attributesOf(t, answer, 0) @@ -386,33 +386,60 @@ func TestShippedMappingDerivesTopicsAndLanguageFromTheResponse(t *testing.T) { func TestShippedMappingOmitsTopicsWhenNothingSuppliesThem(t *testing.T) { _, answer := runShippedWith(t, selectRequest, `{"candidate_count":1,"final_count":1,"hits":[ - {"doc_id":"d1","_score":0.5,"text":"some passage","instance_name":"Bharat Vistaar"}]}`) + {"_id":"h1","doc_id":"d1","_score":0.5,"text":"some passage","filename":"doc-d1","instance_name":"Bharat Vistaar"}]}`) attrs := attributesOf(t, answer, 0) if v, present := attrs["topics"]; present { t.Errorf("topics = %v, want the key absent when no source supplies it", v) } } -func TestShippedMappingNamesTheParticipantAsTheSource(t *testing.T) { + +// The document behind a passage IS its source: doc_id identifies it, filename +// labels it. Both come from the hit, so no participant id is hardcoded here. +// The two upstream fields that look like better candidates are traps -- +// hits[].instance is a provider-internal code and hits[].source names the +// ingestion pipeline. +func TestShippedMappingNamesTheDocumentAsTheSource(t *testing.T) { _, answer := runShipped(t, selectRequest) - attrs := attributesOf(t, answer, 0) + res := resourcesOf(t, answer) - source, ok := attrs["source"].(map[string]any) - if !ok { - t.Fatal("the advisory carries no source") + // doc_id per hit, in the fixture's order. + wantDocIDs := []string{ + "74b89b5f69dcd56f6566d4b1a9392ffd", + "74b89b5f69dcd56f6566d4b1a9392ffd", + "2cc58f8a1c39a64fc132338be3547f52", + "2cc58f8a1c39a64fc132338be3547f52", } - id, _ := source["sourceId"].(string) - if id == "" { - t.Error("source carries no sourceId") + seenIDs := map[string]bool{} + for i := range res { + attrs := attributesOf(t, answer, i) + source, ok := attrs["source"].(map[string]any) + if !ok { + t.Fatalf("resource %d carries no source", i) + } + id, _ := source["sourceId"].(string) + if id == "" { + t.Errorf("resource %d source carries no sourceId", i) + } + if strings.Contains(id, "docs-pipeline") || id == "bv" { + t.Errorf("sourceId = %q, which is the pipeline or the instance code, not the document", id) + } + seenIDs[id] = true + + // sourceName is the document's filename, never the instance name. + name, _ := source["sourceName"].(string) + if name == "" { + t.Errorf("resource %d source carries no sourceName", i) + } + if name == "Bharat Vistaar" { + t.Errorf("sourceName = %q, which is the instance, not the document", name) + } } - if strings.Contains(id, "docs-pipeline") { - t.Errorf("sourceId = %q, which is the ingestion pipeline, not the source", id) + for _, want := range wantDocIDs { + if !seenIDs[want] { + t.Errorf("no advisory names doc_id %q as its source", want) + } } } - -// NO RESULTS IS AN ANSWER, not a failure. Commitment.resources is required but -// carries no minItems, so an empty array says "nothing matched" without -// inventing an advisory -- and without the dangling resource id that building -// one anyway produced. func TestShippedMappingAnswersAnEmptyResultWithNoResources(t *testing.T) { _, answer := runShippedWith(t, selectRequest, `{"candidate_count":0,"final_count":0,"effective_config":{"query":"gruha jyoti scheme eligibility"},"hits":[]}`) @@ -423,56 +450,41 @@ func TestShippedMappingAnswersAnEmptyResultWithNoResources(t *testing.T) { commitments := answer["message"].(map[string]any)["contract"].(map[string]any)["commitments"].([]any) offer := commitments[0].(map[string]any)["offer"].(map[string]any) ids, _ := offer["resourceIds"].([]any) - if len(ids) != 1 || fmt.Sprint(ids[0]) != selectedResourceID { - t.Fatalf("offer.resourceIds = %v, want just the selected id %q", ids, selectedResourceID) + if len(ids) != 0 { + t.Fatalf("offer.resourceIds = %v, want none: the offer covers no resource", ids) } } -// The offer is echoed and its references are rebuilt, per the implementation -// guide: the OnDemand id the caller selected is RETAINED so it can correlate -// the callback, alongside the Direct id minted for each document. The selected -// id deliberately does not resolve inside resources[] -- OnDemand and Direct -// require different fields, so the answer cannot restate it, and the caller -// matches it against its own request. $distinct guards a re-selected advisory -// appearing twice. -func TestShippedMappingRetainsTheSelectedIdAlongsideTheMintedOnes(t *testing.T) { +// The offer is echoed and its references rebuilt to name exactly the resources +// this answer carries -- one per hit. The OnDemand id the caller selected is +// NOT retained: it resolves to nothing in resources[], and correlation is +// context.transactionId's job. +func TestShippedMappingReferencesEveryHitAndNothingElse(t *testing.T) { _, answer := runShipped(t, selectRequest) commitments := answer["message"].(map[string]any)["contract"].(map[string]any)["commitments"].([]any) offer := commitments[0].(map[string]any)["offer"].(map[string]any) - // The request's own offer id, carried through rather than minted. if got := fmt.Sprint(offer["id"]); got != requestOfferID { t.Errorf("offer.id = %q, want the request's %q echoed", got, requestOfferID) } ids, _ := offer["resourceIds"].([]any) res := resourcesOf(t, answer) - if len(ids) != len(res)+1 { - t.Fatalf("offer.resourceIds = %v, want the selected id plus one per advisory (%d)", ids, len(res)) + if len(ids) != len(res) { + t.Fatalf("offer.resourceIds = %v, want exactly one per resource (%d)", ids, len(res)) } - if fmt.Sprint(ids[0]) != selectedResourceID { - t.Errorf("offer.resourceIds[0] = %v, want the selected id %q", ids[0], selectedResourceID) - } - // Every minted advisory must appear, or a caller cannot resolve it. + + returned := map[string]bool{} for i := range res { - want := fmt.Sprint(res[i].(map[string]any)["id"]) - found := false - for _, got := range ids[1:] { - if fmt.Sprint(got) == want { - found = true - } - } - if !found { - t.Errorf("resource %q is missing from offer.resourceIds %v", want, ids) - } + returned[fmt.Sprint(res[i].(map[string]any)["id"])] = true } - // No duplicates: $distinct has to be doing its job. - seen := map[string]bool{} for _, id := range ids { - if seen[fmt.Sprint(id)] { - t.Errorf("offer.resourceIds repeats %v", id) + if fmt.Sprint(id) == selectedResourceID { + t.Errorf("offer retains the selected id %q, which resolves to nothing here", id) + } + if !returned[fmt.Sprint(id)] { + t.Errorf("offer references %q, which is not in resources[]", id) } - seen[fmt.Sprint(id)] = true } } From bc2cafc0debdfeb61b65a6b86098d6f508ffc6a6 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 12:56:05 +0530 Subject: [PATCH 101/130] feat(mappings): describe each passage by its section, drop supportingResourceIds [#7] descriptor.shortDesc now reads hits[].section, the heading a passage sits under. doc_short_description is the field that looks right for it and is the one the guide names, but it is "" on every hit in the live corpus while section is always populated -- so section is the only per-hit short label there is. It carries raw ingested markdown, images included, which is a corpus property rather than something to clean up here. supportingResourceIds is removed. It minted "res:knowledge:" per advisory, which named the document the passage came from -- exactly what source.sourceId now says directly, since that became doc_id. The minted ids also resolved to nothing: those documents are not published as KnowledgeResource resources, so a caller could correlate but never dereference them. hits[].section joins _id and filename as a field mapped against the guide's never-mapped list, recorded in the header with the reason. --- .../knowledge/knowledge-advisory.select.yaml | 30 +++++++++------ .../KnowledgeAdvisory/mappings_test.go | 38 +++++++++++++++++++ 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/config/mappings/knowledge/knowledge-advisory.select.yaml b/config/mappings/knowledge/knowledge-advisory.select.yaml index 9f1a3386..32df0e6f 100644 --- a/config/mappings/knowledge/knowledge-advisory.select.yaml +++ b/config/mappings/knowledge/knowledge-advisory.select.yaml @@ -54,9 +54,16 @@ request: | # in the live corpus the four hits carry four DISTINCT passages (no two share # a text checksum), so collapsing them loses content. # -# hits[]._id and hits[].filename ARE mapped, though the guide lists both as -# never-mapped. _id is the only per-hit identifier, and filename is the only -# per-hit human label for the document a passage came from. +# hits[]._id, hits[].filename and hits[].section ARE mapped, though the guide +# lists all three as never-mapped. _id is the only per-hit identifier, +# filename the only per-hit label for the document a passage came from, and +# section the only per-hit short label -- doc_short_description, the field +# that looks right for it, is "" on every hit. +# +# supportingResourceIds is NOT emitted. It named the document a passage came +# from, which source.sourceId now does directly, and the ids it minted +# resolved to nothing: those documents are not published as KnowledgeResource +# resources. # # Also divergent, each marked at its site: hits[].text IS the recommendation # message, since nothing in this path synthesizes; no $assert, because an empty @@ -173,9 +180,14 @@ response: | : "KNOWLEDGE_ADVISORY" }, $exists($label) ? { "name": $label & " advisory" } : {}, - /* Blank on every hit here, so both drop out today. */ - $exists($text($h.doc_short_description)) - ? { "shortDesc": $text($h.doc_short_description) } : {}, + /* The heading the passage sits under. doc_short_description + is the field this looks like it should read, but that is "" + on every hit in the corpus while section is always + populated -- so section is the only per-hit short label + there is. Note it carries the passage's raw ingested + markdown, images included. */ + $exists($text($h.section)) + ? { "shortDesc": $text($h.section) } : {}, $exists($text($h.doc_llm_description)) ? { "longDesc": $text($h.doc_llm_description) } : ($exists($text($h.description)) @@ -201,12 +213,6 @@ response: | "message": $h.text, "language": $lang }], - /* The document this passage was drawn from, as a resource - IDENTIFIER rather than the raw hash. doc_id never - chunk_id. It resolves only once those documents are - themselves published as KnowledgeResource resources, - which today they are not. */ - "supportingResourceIds": ["res:knowledge:" & $h.doc_id], "rationale": $exists($text($h.instance_name)) ? $counts & " from the " & $text($h.instance_name) & " corpus." : $counts & ".", diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go index ac99483f..8865caa5 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go @@ -91,6 +91,7 @@ const providerResponse = `{ "hits": [ { "_id": "26860172-2762-4b0b-cabf-3c947846c2e1", + "section": "![House icon](a2bc7bb7c74e387ff445bcc6cf9b2a5a_1_img.webp)", "filename": "doc-379e706ade8b-rerun-1788847883", "_score": 0.77918214, "doc_id": "74b89b5f69dcd56f6566d4b1a9392ffd", @@ -103,6 +104,7 @@ const providerResponse = `{ }, { "_id": "0e753c2b-040a-4af9-3dc4-c9fca3b01e4d", + "section": "![House icon](71c6c13ee057c1204dd8980597d79d5b_1_img.webp)", "filename": "doc-379e706ade8b-rerun-1788785422", "_score": 0.7783708, "doc_id": "74b89b5f69dcd56f6566d4b1a9392ffd", @@ -117,6 +119,7 @@ const providerResponse = `{ }, { "_id": "b9ec5c75-7331-0424-f741-4b3294c6b75e", + "section": "- (d) Under the Anna Bhagya Scheme, from which state is the rice being brought", "filename": "doc-d7124029cb60-rerun-1788841718", "_score": 0.77453613, "doc_id": "2cc58f8a1c39a64fc132338be3547f52", @@ -129,6 +132,7 @@ const providerResponse = `{ }, { "_id": "4c6466a5-6975-a3be-4a62-637ae1a611d3", + "section": "**Regarding the 'Jalashree' Scheme being implemented by KUIDFC in the State**", "filename": "doc-d7124029cb60-rerun-1788841718", "_score": 0.7743588, "doc_id": "2cc58f8a1c39a64fc132338be3547f52", @@ -488,3 +492,37 @@ func TestShippedMappingReferencesEveryHitAndNothingElse(t *testing.T) { } } } + +// shortDesc is the passage's own section heading, and supportingResourceIds is +// gone. The heading is the only per-hit short label the corpus offers -- +// doc_short_description, the field that looks right for it, is "" throughout. +// supportingResourceIds named the document a passage came from, which +// source.sourceId now does directly, and the ids it minted resolved to nothing. +func TestShippedMappingDescribesEachPassageBySection(t *testing.T) { + _, answer := runShipped(t, selectRequest) + res := resourcesOf(t, answer) + + wantSections := map[string]bool{ + "![House icon](a2bc7bb7c74e387ff445bcc6cf9b2a5a_1_img.webp)": true, + "![House icon](71c6c13ee057c1204dd8980597d79d5b_1_img.webp)": true, + "- (d) Under the Anna Bhagya Scheme, from which state is the rice being brought": true, + "**Regarding the 'Jalashree' Scheme being implemented by KUIDFC in the State**": true, + } + for i := range res { + d, _ := res[i].(map[string]any)["descriptor"].(map[string]any) + short, _ := d["shortDesc"].(string) + if short == "" { + t.Errorf("resource %d carries no shortDesc", i) + continue + } + if !wantSections[short] { + t.Errorf("resource %d shortDesc = %q, which is not that hit's section", i, short) + } + + // supportingResourceIds must be gone from every advisory. + attrs := attributesOf(t, answer, i) + if v, present := attrs["supportingResourceIds"]; present { + t.Errorf("resource %d still carries supportingResourceIds = %v", i, v) + } + } +} From b27494558a60bace35e51eed0c22da87898d1440 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 15:35:11 +0530 Subject: [PATCH 102/130] feat(plugin): accept one level of nesting in a plugin config block [#7] Groundwork for per-provider auth on a provider step. A step can serve several providers, but its auth is one scheme for the whole step, so two providers that authenticate differently cannot both be configured. Naming the provider in each setting is the way out, and this makes the readable spelling of that possible: knowledge-provider: authScheme: oauth2 tokenUrl: https://issuer.example/token which arrives at the plugin as authScheme-knowledge-provider and tokenUrl-knowledge-provider. The nesting is a way of WRITING config, not a shape any plugin handles: by the time one is called its settings are flat strings exactly as before. Config is now `Settings`, which is map[string]string underneath. That is the point -- a named map type stays assignable to a map[string]string parameter, so every plugin's New signature, all 25 call sites in manager.go and the read in stdHandler.go compile untouched. Nothing in core changed, and no .so needs rebuilding for the config format. Settings stays ignorant of which keys mean anything. It knows the format, not the vocabulary: listing valid fields here would collect every plugin's settings in one file and make it a dependency of all of them. Rejecting an unknown field belongs to the plugin that defines it. Four things are refused rather than absorbed, each because the alternative loses a setting the operator wrote: - two levels of nesting, which has no flat spelling - a dashed field inside a block, which would flatten to -- and resolve to a provider that does not exist - the same setting written both nested and flat, a contradiction with no right answer - an unnamed block Integers render as digits, not an exponent: a float formatting would turn maxResponseBytes into 1.048576e+06 and the plugin's ParseInt would reject it. Two test expectations move from map[string]string to Settings. reflect.DeepEqual is type-sensitive, so those were comparing the type rather than the contents; no behaviour changed. --- pkg/plugin/config.go | 123 +++++++++++- pkg/plugin/config_test.go | 187 ++++++++++++++++++ .../otelsetup/otelsetup_test.go | 9 +- pkg/plugin/manager_test.go | 2 +- 4 files changed, 308 insertions(+), 13 deletions(-) create mode 100644 pkg/plugin/config_test.go diff --git a/pkg/plugin/config.go b/pkg/plugin/config.go index c5b87c7e..9a594a5d 100644 --- a/pkg/plugin/config.go +++ b/pkg/plugin/config.go @@ -1,23 +1,130 @@ package plugin +import ( + "fmt" + "sort" + "strings" + + "gopkg.in/yaml.v3" +) + +// Settings is a plugin's own configuration block. +// +// It is map[string]string underneath, deliberately: every plugin's New takes a +// map[string]string, and a named map type stays assignable to one, so widening +// this changed no call site and no plugin contract. +// +// What it adds is ONE level of nesting on the way in. A block keyed by +// something a plugin recognises -- a participant id, for the provider steps -- +// is flattened to `-`, so +// +// knowledge-provider: +// authScheme: oauth2 +// tokenUrl: https://issuer.example/token +// +// arrives as authScheme-knowledge-provider and tokenUrl-knowledge-provider. +// The nesting is therefore a way of WRITING config, not a shape any plugin has +// to handle: by the time one is called, its settings are flat strings as +// before. +// +// This type stays deliberately ignorant of which keys mean anything. It knows +// the format, not the vocabulary -- naming valid fields here would put every +// plugin's settings in one place and make this file a dependency of all of +// them. Rejecting an unknown field is the plugin's job, where the field is +// already defined. +type Settings map[string]string + +// UnmarshalYAML accepts scalars at the top level and one level of nesting. +func (s *Settings) UnmarshalYAML(node *yaml.Node) error { + var raw map[string]any + if err := node.Decode(&raw); err != nil { + return err + } + + out := Settings{} + // Sorted so that a duplicate is reported against the same pair whichever + // order the map happened to iterate in. + blocks := make([]string, 0, len(raw)) + for key := range raw { + blocks = append(blocks, key) + } + sort.Strings(blocks) + + for _, key := range blocks { + value := raw[key] + + block, nested := value.(map[string]any) + if !nested { + out[key] = scalar(value) + continue + } + + // A block key carrying a dash would produce a flattened key that reads + // as another field's, so the split on the first dash could no longer + // tell them apart. Participant ids DO carry dashes, which is why the + // dash goes between field and key rather than inside either. + if strings.TrimSpace(key) == "" { + return fmt.Errorf("plugin config: a settings block has an empty name") + } + + fields := make([]string, 0, len(block)) + for field := range block { + fields = append(fields, field) + } + sort.Strings(fields) + + for _, field := range fields { + if _, deeper := block[field].(map[string]any); deeper { + return fmt.Errorf( + "plugin config: %s.%s nests further; only one level is supported", key, field) + } + if strings.Contains(field, "-") { + return fmt.Errorf( + "plugin config: %s.%s carries a dash; inside a block a setting is named "+ + "plainly, and the block already says which one it belongs to", key, field) + } + flat := field + "-" + key + if _, taken := out[flat]; taken { + return fmt.Errorf( + "plugin config: %s is set twice, once as %s.%s and once directly", + flat, key, field) + } + out[flat] = scalar(block[field]) + } + } + + *s = out + return nil +} + +// scalar renders a YAML scalar the way the flat form would have carried it. +// %v on an int gives "1048576" rather than an exponent, which a float would; +// YAML decodes an integer literal as int, so a byte count survives intact. +func scalar(v any) string { + if v == nil { + return "" + } + return fmt.Sprintf("%v", v) +} + type PublisherCfg struct { - ID string `yaml:"id"` - Config map[string]string `yaml:"config"` + ID string `yaml:"id"` + Config Settings `yaml:"config"` } type ValidatorCfg struct { - ID string `yaml:"id"` - Config map[string]string `yaml:"config"` + ID string `yaml:"id"` + Config Settings `yaml:"config"` } type Config struct { - ID string `yaml:"id"` - Config map[string]string `yaml:"config"` + ID string `yaml:"id"` + Config Settings `yaml:"config"` } type ManagerConfig struct { - Root string `yaml:"root"` - RemoteRoot string `yaml:"remoteRoot"` + Root string `yaml:"root"` + RemoteRoot string `yaml:"remoteRoot"` BecknConstants *BecknConstantsConfig `yaml:"becknConstants,omitempty"` } diff --git a/pkg/plugin/config_test.go b/pkg/plugin/config_test.go new file mode 100644 index 00000000..6dc2ac1d --- /dev/null +++ b/pkg/plugin/config_test.go @@ -0,0 +1,187 @@ +package plugin + +import ( + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// decode is the path a real config takes: YAML -> Config.Config. +func decode(t *testing.T, body string) (Settings, error) { + t.Helper() + var c Config + err := yaml.Unmarshal([]byte("id: X\nconfig:\n"+body), &c) + return c.Config, err +} + +// The flat form is what every shipped config uses, and it has to survive +// untouched -- this type was introduced to ADD nesting, not to change what +// already worked. +func TestSettingsKeepsTheFlatFormUntouched(t *testing.T) { + got, err := decode(t, ` + bindingKeys: "a|cap,b|cap" + authScheme: oauth2 + clientIdEnv: KNOWLEDGE_CLIENT_ID +`) + if err != nil { + t.Fatalf("decode: %v", err) + } + want := Settings{ + "bindingKeys": "a|cap,b|cap", + "authScheme": "oauth2", + "clientIdEnv": "KNOWLEDGE_CLIENT_ID", + } + if len(got) != len(want) { + t.Fatalf("got %v, want %v", got, want) + } + for k, v := range want { + if got[k] != v { + t.Errorf("%s = %q, want %q", k, got[k], v) + } + } +} + +// One level of nesting flattens to -, which is what turns a +// per-provider block into settings a plugin can read without knowing YAML. +func TestSettingsFlattensOneLevelOfNesting(t *testing.T) { + got, err := decode(t, ` + bindingKeys: "knowledge-provider|cap,vistaar-two|cap" + knowledge-provider: + authScheme: oauth2 + tokenUrl: "https://issuer.example/token" + vistaar-two: + authScheme: query + queryValueEnv: VISTAAR_TWO_TOKEN +`) + if err != nil { + t.Fatalf("decode: %v", err) + } + for key, want := range map[string]string{ + "authScheme-knowledge-provider": "oauth2", + "tokenUrl-knowledge-provider": "https://issuer.example/token", + "authScheme-vistaar-two": "query", + "queryValueEnv-vistaar-two": "VISTAAR_TWO_TOKEN", + "bindingKeys": "knowledge-provider|cap,vistaar-two|cap", + } { + if got[key] != want { + t.Errorf("%s = %q, want %q", key, got[key], want) + } + } + // The block names themselves must not survive as keys. + for _, gone := range []string{"knowledge-provider", "vistaar-two"} { + if _, present := got[gone]; present { + t.Errorf("block name %q leaked through as a key", gone) + } + } +} + +// A participant id is hostname-shaped, so dots are legal and must not confuse +// the flattening -- the dash between field and key is the only separator. +func TestSettingsHandlesADottedBlockName(t *testing.T) { + got, err := decode(t, ` + provider.oan.dev: + authScheme: basic + usernameEnv: P_USER +`) + if err != nil { + t.Fatalf("decode: %v", err) + } + if got["authScheme-provider.oan.dev"] != "basic" { + t.Errorf("dotted block name did not flatten: %v", got) + } + if got["usernameEnv-provider.oan.dev"] != "P_USER" { + t.Errorf("dotted block name did not flatten: %v", got) + } +} + +// An integer must arrive as its digits. A float formatting would turn a byte +// count into 1.048576e+06 and the plugin's ParseInt would reject it. +func TestSettingsRendersAnIntegerWithoutAnExponent(t *testing.T) { + got, err := decode(t, " maxResponseBytes: 1048576\n enabled: true\n") + if err != nil { + t.Fatalf("decode: %v", err) + } + if got["maxResponseBytes"] != "1048576" { + t.Errorf("maxResponseBytes = %q, want 1048576", got["maxResponseBytes"]) + } + if got["enabled"] != "true" { + t.Errorf("enabled = %q, want true", got["enabled"]) + } +} + +// Two levels have no flat spelling, so they are refused rather than dropped: +// silently discarding a setting an operator wrote is the worse failure. +func TestSettingsRefusesDeeperNesting(t *testing.T) { + _, err := decode(t, ` + knowledge-provider: + retry: + attempts: 3 +`) + if err == nil { + t.Fatal("two levels of nesting was accepted") + } + if !strings.Contains(err.Error(), "only one level") { + t.Errorf("error %q should say one level is supported", err) + } +} + +// Inside a block the id is already said by the block name. A dashed field +// would flatten to --, resolving to a provider that does not +// exist, so it is refused with the reason rather than accepted and lost. +func TestSettingsRefusesADashedFieldInsideABlock(t *testing.T) { + _, err := decode(t, ` + vistaar-two: + authScheme-vistaar-two: query +`) + if err == nil { + t.Fatal("a dashed field inside a block was accepted") + } + if !strings.Contains(err.Error(), "dash") { + t.Errorf("error %q should name the dash", err) + } +} + +// The same setting written both ways is a contradiction with no right answer, +// so neither wins: it is reported. +func TestSettingsRefusesASettingGivenTwice(t *testing.T) { + _, err := decode(t, ` + authScheme-vistaar-two: oauth2 + vistaar-two: + authScheme: query +`) + if err == nil { + t.Fatal("the same setting given twice was accepted") + } + if !strings.Contains(err.Error(), "set twice") { + t.Errorf("error %q should say it is set twice", err) + } +} + +// An absent config block is not an error: most plugins take none. +func TestSettingsAcceptsNoConfigAtAll(t *testing.T) { + var c Config + if err := yaml.Unmarshal([]byte("id: X\n"), &c); err != nil { + t.Fatalf("decode: %v", err) + } + if len(c.Config) != 0 { + t.Errorf("Config = %v, want empty", c.Config) + } +} + +// Settings must stay usable exactly where map[string]string was, because every +// plugin's New takes one and none of them changed. +func TestSettingsPassesWhereAPlainMapIsWanted(t *testing.T) { + got, err := decode(t, " authScheme: oauth2\n") + if err != nil { + t.Fatalf("decode: %v", err) + } + takesPlainMap := func(m map[string]string) string { return m["authScheme"] } + if takesPlainMap(got) != "oauth2" { + t.Error("Settings no longer passes as a map[string]string") + } + var assigned map[string]string = got + if assigned["authScheme"] != "oauth2" { + t.Error("Settings no longer assigns to a map[string]string") + } +} diff --git a/pkg/plugin/implementation/otelsetup/otelsetup_test.go b/pkg/plugin/implementation/otelsetup/otelsetup_test.go index 3131b682..a61a34cb 100644 --- a/pkg/plugin/implementation/otelsetup/otelsetup_test.go +++ b/pkg/plugin/implementation/otelsetup/otelsetup_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/beckn-one/beckn-onix/pkg/plugin" "github.com/beckn-one/beckn-onix/pkg/version" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -203,7 +204,7 @@ func TestToPluginConfig_Success(t *testing.T) { name string cfg *Config expectedID string - expectedConfig map[string]string + expectedConfig plugin.Settings }{ { name: "Valid config with all fields", @@ -220,7 +221,7 @@ func TestToPluginConfig_Success(t *testing.T) { TimeInterval: 5, }, expectedID: "otelsetup", - expectedConfig: map[string]string{ + expectedConfig: plugin.Settings{ "serviceName": "test-service", "serviceVersion": "1.0.0", "environment": "test", @@ -244,7 +245,7 @@ func TestToPluginConfig_Success(t *testing.T) { Environment: "production", }, expectedID: "otelsetup", - expectedConfig: map[string]string{ + expectedConfig: plugin.Settings{ "serviceName": "my-service", "serviceVersion": "2.0.0", "environment": "production", @@ -271,7 +272,7 @@ func TestToPluginConfig_Success(t *testing.T) { OtlpEndpoint: "", }, expectedID: "otelsetup", - expectedConfig: map[string]string{ + expectedConfig: plugin.Settings{ "serviceName": "", "serviceVersion": "", "environment": "", diff --git a/pkg/plugin/manager_test.go b/pkg/plugin/manager_test.go index 15edb7a8..655134a4 100644 --- a/pkg/plugin/manager_test.go +++ b/pkg/plugin/manager_test.go @@ -2588,7 +2588,7 @@ func TestApplyConstants_UnknownPlugin_NoChange(t *testing.T) { m := makeManager(map[string]map[string]string{"dediregistry": {"url": "https://x.com"}}, nil) c := cfg("cache", map[string]string{"addr": "redis:6379"}) require.NoError(t, m.applyConstants(context.Background(), c)) - assert.Equal(t, map[string]string{"addr": "redis:6379"}, c.Config) + assert.Equal(t, Settings{"addr": "redis:6379"}, c.Config) } // --- nil constants --- From 61f5a4b0b6e9d0e833446fed8f4c04894f271fec Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 16:06:51 +0530 Subject: [PATCH 103/130] feat(upstream): give each provider its own credentials [#7] A step serves several binding keys, but its auth was one scheme for the whole step, so two providers of the same capability could not both be configured when they authenticate differently. Everything else that varies per provider already comes from the registry -- endpoint, path, mapping -- so auth was the only thing pinned to the step, and the only thing making a second provider impossible. Config now carries one profile per provider, keyed by participant id. What an operator writes is a block per provider: knowledge-provider: authScheme: oauth2 tokenUrl: <> clientIdEnv: KNOWLEDGE_CLIENT_ID clientSecretEnv: KNOWLEDGE_CLIENT_SECRET vistaar-two: authScheme: query queryName: token queryValueEnv: VISTAAR_TWO_TOKEN pkg/plugin flattens that to authScheme-knowledge-provider and friends, and ParseAuth reads it back. No plugin contract changed and nothing in core was touched. NO STEP-WIDE DEFAULT. A profile per provider means there is never a question of which setting applies. A served provider without a block is refused at startup, and a bare step-wide authScheme is refused too rather than ignored -- ignoring it leaves every provider on no credential at all, which reads as the provider rejecting us rather than as our config being stale. THE TOKEN CACHE MOVED ONTO THE PROFILE, and that is the change that matters. A step-wide cache shared between two oauth2 providers hands the first provider's token to the second, which is authenticating as somebody else -- and no test of a single provider can see it. The negative control for the new test reproduces it exactly: "alpha was sent token-B". Each profile now holds its own, keeping the atomic-read/mutex-exchange split that exists because guarding the value with the mutex deadlocked through secretForms. REDACTION UNIONS EVERY PROFILE, not the one being served. An error raised while serving one provider can quote another's credential, and scoping this to the active profile would let it reach the log unredacted. Sorting is across the merged set, because one provider's token can be a substring of another's and replacing the short one first leaves the longer half-redacted. Validation runs per profile and names the provider, both directions: a profile for a provider not in bindingKeys is a typo applying to nothing, and a served provider with no profile would fall through to sending nothing. ParseAuth is shared rather than repeated: all three capability plugins copied the same field list out of the same map, so a scheme added in one had to be remembered in three. All three shrink. The shipped config's three entries are rewritten nested, and a test reads that real file rather than a copy so the config and the parser cannot drift. 66 packages green, no races. Every new test landed with a negative control. --- config/provider-adapter.yaml | 46 +- .../KnowledgeAdvisory/cmd/plugin.go | 24 +- .../KnowledgeAdvisory/cmd/plugin_test.go | 171 +++++-- .../cmd/shippedconfig_test.go | 79 ++++ .../KnowledgeAdvisory/live_test.go | 15 +- .../KnowledgeAdvisory/mappings_test.go | 10 +- .../implementation/MandiPrice/cmd/plugin.go | 18 +- .../MandiPrice/cmd/plugin_test.go | 70 +-- .../MandiPrice/mappings_test.go | 28 +- .../WeatherObservation/cmd/plugin.go | 18 +- .../WeatherObservation/cmd/plugin_test.go | 44 +- .../WeatherObservation/mappings_test.go | 10 +- .../internal/upstream/dispatch_test.go | 9 +- .../internal/upstream/perprovider_test.go | 264 +++++++++++ .../internal/upstream/upstream.go | 425 ++++++++++++++---- .../internal/upstream/upstream_test.go | 177 ++++---- 16 files changed, 1100 insertions(+), 308 deletions(-) create mode 100644 pkg/plugin/implementation/KnowledgeAdvisory/cmd/shippedconfig_test.go create mode 100644 pkg/plugin/implementation/internal/upstream/perprovider_test.go diff --git a/config/provider-adapter.yaml b/config/provider-adapter.yaml index e336a3c1..f6c798c9 100644 --- a/config/provider-adapter.yaml +++ b/config/provider-adapter.yaml @@ -319,19 +319,27 @@ modules: # header a named header; headerName + headerValueEnv # query a named query parameter; queryName + queryValueEnv. # Redacted from the URL the step logs. - authScheme: basic - usernameEnv: MAUSAMGRAM_USER - passwordEnv: MAUSAMGRAM_X_API_KEY - - # The other two schemes, for reference: + # oauth2 a client id and secret exchanged for a bearer token; + # tokenUrl + clientIdEnv + clientSecretEnv. # - # authScheme: header - # headerName: X-API-Key - # headerValueEnv: MAUSAMGRAM_X_API_KEY + # AUTH IS PER PROVIDER, in a block named for the participant id -- + # the left half of a binding key. A step serves several binding + # keys and the providers behind them need not authenticate alike, + # so there is no step-wide setting: every provider declares its + # own, and one that is served without a block is refused at + # startup rather than quietly sending nothing. + mausamgram: + authScheme: basic + usernameEnv: MAUSAMGRAM_USER + passwordEnv: MAUSAMGRAM_X_API_KEY + + # The other schemes, for reference. A second provider of this same + # capability goes beside the block above, with its own: # - # authScheme: query - # queryName: token - # queryValueEnv: MAUSAMGRAM_TOKEN + # some-other-imd: + # authScheme: header + # headerName: X-API-Key + # headerValueEnv: SOME_OTHER_IMD_KEY # A ceiling on the provider's RESPONSE, so one that streams or # misbehaves cannot exhaust memory. The body is rejected past @@ -354,9 +362,10 @@ modules: # parameter's NAME and the name of the variable carrying the # value -- never the value -- and redacts it from the URL it # logs, so a token cannot reach the log by way of the request. - authScheme: query - queryName: token - queryValueEnv: MANDI_TOKEN + agmarknet: + authScheme: query + queryName: token + queryValueEnv: MANDI_TOKEN # MANDI_TOKEN MUST BE SET IN THE ADAPTER'S ENVIRONMENT. Without it # every mandi call fails immediately and permanently: the step @@ -396,7 +405,8 @@ modules: # pastes another. authScheme oauth2 exchanges the client # credentials for a token, holds it until just before it expires, # and exchanges a fresh one on the next request after that. - authScheme: oauth2 + knowledge-provider: + authScheme: oauth2 # A PLACEHOLDER, because this is per-deployment and no host # belongs in this file. Whatever goes here is the provider's @@ -404,7 +414,7 @@ modules: # token path: # # https:///auth/realms//protocol/openid-connect/token - tokenUrl: <> + tokenUrl: <> # The VARIABLES holding the client credentials, never the values. # Both must be set in the adapter's environment; with either @@ -416,8 +426,8 @@ modules: # docker: -e KNOWLEDGE_CLIENT_ID=... -e KNOWLEDGE_CLIENT_SECRET=... # kubernetes: a Secret, mounted as environment variables # local: KNOWLEDGE_CLIENT_ID=... KNOWLEDGE_CLIENT_SECRET=... ./server - clientIdEnv: KNOWLEDGE_CLIENT_ID - clientSecretEnv: KNOWLEDGE_CLIENT_SECRET + clientIdEnv: KNOWLEDGE_CLIENT_ID + clientSecretEnv: KNOWLEDGE_CLIENT_SECRET # A retrieval answer is larger than a price row: four passages # measured 15.7 KB against the live corpus, and top_k defaults to diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin.go b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin.go index 5bcefed5..b6ee51f4 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin.go @@ -16,6 +16,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/KnowledgeAdvisory" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" ) // knowledgeAdvisoryProvider implements definition.ProviderStepProvider. @@ -34,19 +35,6 @@ func (p knowledgeAdvisoryProvider) parseConfig(config map[string]string) (*Knowl // this is a default rather than something to set. ProviderIDAt: config["providerIdAt"], CapabilityCodeAt: config["capabilityCodeAt"], - AuthScheme: config["authScheme"], - UsernameEnv: config["usernameEnv"], - PasswordEnv: config["passwordEnv"], - HeaderName: config["headerName"], - HeaderValueEnv: config["headerValueEnv"], - QueryName: config["queryName"], - QueryValueEnv: config["queryValueEnv"], - // authScheme oauth2. tokenUrl is not a credential and is named here; - // the client id and secret are named only as the variables holding - // them, so neither value appears in a config file or the registry. - TokenURL: config["tokenUrl"], - ClientIDEnv: config["clientIdEnv"], - ClientSecretEnv: config["clientSecretEnv"], } if raw, exists := config["maxResponseBytes"]; exists && raw != "" { @@ -60,6 +48,16 @@ func (p knowledgeAdvisoryProvider) parseConfig(config map[string]string) (*Knowl cfg.MaxResponseBytes = value } + // One credential profile per provider, read from the flattened + // authScheme- settings. Shared with the other capability + // plugins: each used to copy the same field list, so a scheme added in one + // had to be remembered in three. + auth, err := upstream.ParseAuth(config) + if err != nil { + return nil, err + } + cfg.Auth = auth + return cfg, nil } diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin_test.go index 70e101a8..438729b6 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin_test.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin_test.go @@ -10,6 +10,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/KnowledgeAdvisory" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" ) type stubRegistry struct{} @@ -37,10 +38,11 @@ func TestParseConfig(t *testing.T) { }{ { // Everything absent is left zero: KnowledgeAdvisory.New defaults - // it, so the rules are defined in exactly one place. + // it, so the rules are defined in exactly one place. Auth is an + // empty map rather than nil -- ParseAuth always returns one. name: "leaves everything unset for New to default", config: map[string]string{}, - expected: &KnowledgeAdvisory.Config{}, + expected: &KnowledgeAdvisory.Config{Auth: map[string]*upstream.Auth{}}, }, { // The scheme this capability exists to use. Its provider sits @@ -48,48 +50,145 @@ func TestParseConfig(t *testing.T) { // static value in an environment variable is wrong twice a day. name: "reads the oauth2 settings this capability needs", config: map[string]string{ - "bindingKeys": "knowledge-provider|openagrinet:KnowledgeAdvisory", - "authScheme": "oauth2", - "tokenUrl": "https://issuer.invalid/token", - "clientIdEnv": "KNOWLEDGE_CLIENT_ID", - "clientSecretEnv": "KNOWLEDGE_CLIENT_SECRET", + "bindingKeys": "knowledge-provider|openagrinet:KnowledgeAdvisory", + "authScheme-knowledge-provider": "oauth2", + "tokenUrl-knowledge-provider": "https://issuer.invalid/token", + "clientIdEnv-knowledge-provider": "KNOWLEDGE_CLIENT_ID", + "clientSecretEnv-knowledge-provider": "KNOWLEDGE_CLIENT_SECRET", + }, + expected: &KnowledgeAdvisory.Config{ + BindingKeys: []string{"knowledge-provider|openagrinet:KnowledgeAdvisory"}, + Auth: map[string]*upstream.Auth{ + "knowledge-provider": { + Provider: "knowledge-provider", + Scheme: "oauth2", + TokenURL: "https://issuer.invalid/token", + ClientIDEnv: "KNOWLEDGE_CLIENT_ID", + ClientSecretEnv: "KNOWLEDGE_CLIENT_SECRET", + }, + }, + }, + }, + { + // TWO PROVIDERS, DIFFERENT SCHEMES -- the reason this shape exists. + // One step serves both binding keys, and each authenticates as + // itself. + name: "reads a profile per provider", + config: map[string]string{ + "bindingKeys": "knowledge-provider|openagrinet:KnowledgeAdvisory," + + "vistaar-two|openagrinet:KnowledgeAdvisory", + "authScheme-knowledge-provider": "oauth2", + "tokenUrl-knowledge-provider": "https://issuer.invalid/token", + "clientIdEnv-knowledge-provider": "ID", + "clientSecretEnv-knowledge-provider": "SECRET", + "authScheme-vistaar-two": "query", + "queryName-vistaar-two": "token", + "queryValueEnv-vistaar-two": "VISTAAR_TWO_TOKEN", + }, + expected: &KnowledgeAdvisory.Config{ + BindingKeys: []string{ + "knowledge-provider|openagrinet:KnowledgeAdvisory", + "vistaar-two|openagrinet:KnowledgeAdvisory", + }, + Auth: map[string]*upstream.Auth{ + "knowledge-provider": { + Provider: "knowledge-provider", + Scheme: "oauth2", + TokenURL: "https://issuer.invalid/token", + ClientIDEnv: "ID", + ClientSecretEnv: "SECRET", + }, + "vistaar-two": { + Provider: "vistaar-two", + Scheme: "query", + QueryName: "token", + QueryValueEnv: "VISTAAR_TWO_TOKEN", + }, + }, + }, + }, + { + // A participant id is hostname-shaped, so dots are legal. The + // field name in front is what the split anchors on. + name: "reads a dotted participant id", + config: map[string]string{ + "bindingKeys": "provider.oan.dev|openagrinet:KnowledgeAdvisory", + "authScheme-provider.oan.dev": "basic", + "usernameEnv-provider.oan.dev": "P_USER", + "passwordEnv-provider.oan.dev": "P_PASS", }, expected: &KnowledgeAdvisory.Config{ - BindingKeys: []string{"knowledge-provider|openagrinet:KnowledgeAdvisory"}, - AuthScheme: "oauth2", - TokenURL: "https://issuer.invalid/token", - ClientIDEnv: "KNOWLEDGE_CLIENT_ID", - ClientSecretEnv: "KNOWLEDGE_CLIENT_SECRET", + BindingKeys: []string{"provider.oan.dev|openagrinet:KnowledgeAdvisory"}, + Auth: map[string]*upstream.Auth{ + "provider.oan.dev": { + Provider: "provider.oan.dev", + Scheme: "basic", + UsernameEnv: "P_USER", + PasswordEnv: "P_PASS", + }, + }, + }, + }, + { + // The old step-wide spelling. Refused rather than ignored: dropping + // it silently leaves every provider on no credential at all, which + // reads as the provider rejecting us. + name: "refuses a step-wide authScheme", + config: map[string]string{ + "bindingKeys": "knowledge-provider|openagrinet:KnowledgeAdvisory", + "authScheme": "oauth2", + }, + expectedErr: "auth is per provider now", + }, + { + name: "refuses a misspelled credential setting", + config: map[string]string{ + "bindingKeys": "knowledge-provider|openagrinet:KnowledgeAdvisory", + "authScheeme-knowledge-provider": "oauth2", + }, + expectedErr: "is not a credential setting", + }, + { + name: "refuses a setting that names no provider", + config: map[string]string{ + "bindingKeys": "knowledge-provider|openagrinet:KnowledgeAdvisory", + "authScheme-": "oauth2", }, + expectedErr: "names no provider", }, { name: "reads every supported setting", config: map[string]string{ - "bindingKeys": "other|capability", - "authScheme": "basic", - "usernameEnv": "U", - "passwordEnv": "P", - "headerName": "X-Key", - "headerValueEnv": "V", - "queryName": "q", - "queryValueEnv": "Q", - "tokenUrl": "https://issuer.invalid/token", - "clientIdEnv": "ID", - "clientSecretEnv": "SECRET", - "maxResponseBytes": "2048", + "bindingKeys": "other|capability", + "authScheme-other": "basic", + "usernameEnv-other": "U", + "passwordEnv-other": "P", + "headerName-other": "X-Key", + "headerValueEnv-other": "V", + "queryName-other": "q", + "queryValueEnv-other": "Q", + "tokenUrl-other": "https://issuer.invalid/token", + "clientIdEnv-other": "ID", + "clientSecretEnv-other": "SECRET", + "maxResponseBytes": "2048", }, expected: &KnowledgeAdvisory.Config{ - BindingKeys: []string{"other|capability"}, - AuthScheme: "basic", - UsernameEnv: "U", - PasswordEnv: "P", - HeaderName: "X-Key", - HeaderValueEnv: "V", - QueryName: "q", - QueryValueEnv: "Q", - TokenURL: "https://issuer.invalid/token", - ClientIDEnv: "ID", - ClientSecretEnv: "SECRET", + BindingKeys: []string{"other|capability"}, + Auth: map[string]*upstream.Auth{ + "other": { + Provider: "other", + Scheme: "basic", + UsernameEnv: "U", + PasswordEnv: "P", + HeaderName: "X-Key", + HeaderValueEnv: "V", + QueryName: "q", + QueryValueEnv: "Q", + TokenURL: "https://issuer.invalid/token", + ClientIDEnv: "ID", + ClientSecretEnv: "SECRET", + }, + }, MaxResponseBytes: 2048, }, }, @@ -108,7 +207,7 @@ func TestParseConfig(t *testing.T) { // unset variable produces this, and it should read as "unset". name: "treats an empty response cap as unset", config: map[string]string{"maxResponseBytes": ""}, - expected: &KnowledgeAdvisory.Config{}, + expected: &KnowledgeAdvisory.Config{Auth: map[string]*upstream.Auth{}}, }, } diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/cmd/shippedconfig_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/shippedconfig_test.go new file mode 100644 index 00000000..f530a84e --- /dev/null +++ b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/shippedconfig_test.go @@ -0,0 +1,79 @@ +package main + +import ( + "os" + "testing" + + "gopkg.in/yaml.v3" + + "github.com/beckn-one/beckn-onix/pkg/plugin" +) + +// The shipped config and this code move together: the nested per-provider +// block is only valid if pkg/plugin flattens it and ParseAuth reads it back. +// This reads the real file rather than a copy, so the two cannot drift. +func TestShippedConfigParsesIntoAProfilePerProvider(t *testing.T) { + t.Parallel() + + raw, err := os.ReadFile("../../../../../config/provider-adapter.yaml") + if err != nil { + t.Skipf("reference config unavailable: %v", err) + } + + // Only the providerSteps entries: the rest of the file is other modules'. + var file struct { + Modules []struct { + Handler struct { + Plugins struct { + ProviderSteps []plugin.Config `yaml:"providerSteps"` + } `yaml:"plugins"` + } `yaml:"handler"` + } `yaml:"modules"` + } + if err := yaml.Unmarshal(raw, &file); err != nil { + t.Fatalf("the shipped config does not decode: %v", err) + } + + var steps []plugin.Config + for _, m := range file.Modules { + steps = append(steps, m.Handler.Plugins.ProviderSteps...) + } + if len(steps) == 0 { + t.Fatal("no providerSteps found in the shipped config") + } + + for _, step := range steps { + t.Run(step.ID, func(t *testing.T) { + // Flattened on the way in: a plugin never sees a block, so every + // value here is a string. parseConfig failing would mean the + // nesting did not survive. + cfg, err := knowledgeAdvisoryProvider{}.parseConfig(step.Config) + if err != nil { + t.Fatalf("parseConfig on the shipped %s entry: %v", step.ID, err) + } + if len(cfg.BindingKeys) == 0 { + t.Fatal("no bindingKeys") + } + // Every provider it serves must have a profile, which is what + // New would refuse it for otherwise. + for _, key := range cfg.BindingKeys { + provider := key + for i := 0; i < len(key); i++ { + if key[i] == '|' { + provider = key[:i] + break + } + } + profile, ok := cfg.Auth[provider] + if !ok { + t.Errorf("%s serves %q with no auth block", step.ID, provider) + continue + } + if profile.Scheme == "" { + t.Errorf("%s: %q has no authScheme", step.ID, provider) + } + t.Logf("%-20s %-22s authScheme=%s", step.ID, provider, profile.Scheme) + } + }) + } +} diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/live_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/live_test.go index ddce11a5..89410ac3 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/live_test.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/live_test.go @@ -15,6 +15,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/KnowledgeAdvisory" + upstreamstep "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" ) @@ -101,11 +102,15 @@ func TestLiveEndToEnd(t *testing.T) { step, closeStep, err := KnowledgeAdvisory.New(context.Background(), registry, mapper, &KnowledgeAdvisory.Config{ - BindingKeys: []string{key}, - AuthScheme: "oauth2", - TokenURL: issuer.URL + "/token", - ClientIDEnv: "KNOWLEDGE_CLIENT_ID", - ClientSecretEnv: "KNOWLEDGE_CLIENT_SECRET", + BindingKeys: []string{key}, + Auth: map[string]*upstreamstep.Auth{ + strings.Split(key, "|")[0]: { + Scheme: "oauth2", + TokenURL: issuer.URL + "/token", + ClientIDEnv: "KNOWLEDGE_CLIENT_ID", + ClientSecretEnv: "KNOWLEDGE_CLIENT_SECRET", + }, + }, MaxResponseBytes: 1048576, }) if err != nil { diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go index 8865caa5..d621a166 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go @@ -22,6 +22,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/KnowledgeAdvisory" + upstreamstep "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" ) @@ -207,7 +208,14 @@ func runShippedWith(t *testing.T, request, providerBody string) (map[string]any, }} step, closeStep, err := KnowledgeAdvisory.New(context.Background(), registry, mapper, - &KnowledgeAdvisory.Config{BindingKeys: []string{shippedBindingKey}}) + &KnowledgeAdvisory.Config{ + BindingKeys: []string{shippedBindingKey}, + // Auth is per provider; these upstreams are stubs needing + // no credential, and that is declared rather than defaulted. + Auth: map[string]*upstreamstep.Auth{ + strings.Split(shippedBindingKey, "|")[0]: {Scheme: upstreamstep.AuthSchemeNone}, + }, + }) if err != nil { t.Fatalf("failed to build the step: %v", err) } diff --git a/pkg/plugin/implementation/MandiPrice/cmd/plugin.go b/pkg/plugin/implementation/MandiPrice/cmd/plugin.go index 8dcce23f..b033d17a 100644 --- a/pkg/plugin/implementation/MandiPrice/cmd/plugin.go +++ b/pkg/plugin/implementation/MandiPrice/cmd/plugin.go @@ -14,6 +14,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/MandiPrice" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" ) // mandiProvider implements definition.ProviderStepProvider. @@ -32,13 +33,6 @@ func (p mandiProvider) parseConfig(config map[string]string) (*MandiPrice.Config // this is a default rather than something to set. ProviderIDAt: config["providerIdAt"], CapabilityCodeAt: config["capabilityCodeAt"], - AuthScheme: config["authScheme"], - UsernameEnv: config["usernameEnv"], - PasswordEnv: config["passwordEnv"], - HeaderName: config["headerName"], - HeaderValueEnv: config["headerValueEnv"], - QueryName: config["queryName"], - QueryValueEnv: config["queryValueEnv"], } if raw, exists := config["maxResponseBytes"]; exists && raw != "" { @@ -52,6 +46,16 @@ func (p mandiProvider) parseConfig(config map[string]string) (*MandiPrice.Config cfg.MaxResponseBytes = value } + // One credential profile per provider, read from the flattened + // authScheme- settings. Shared with the other capability + // plugins: each used to copy the same field list, so a scheme added in one + // had to be remembered in three. + auth, err := upstream.ParseAuth(config) + if err != nil { + return nil, err + } + cfg.Auth = auth + return cfg, nil } diff --git a/pkg/plugin/implementation/MandiPrice/cmd/plugin_test.go b/pkg/plugin/implementation/MandiPrice/cmd/plugin_test.go index 0a3c1a50..7592a0b7 100644 --- a/pkg/plugin/implementation/MandiPrice/cmd/plugin_test.go +++ b/pkg/plugin/implementation/MandiPrice/cmd/plugin_test.go @@ -10,6 +10,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/MandiPrice" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" ) type stubRegistry struct{} @@ -40,7 +41,7 @@ func TestParseConfig(t *testing.T) { // rules are defined in exactly one place. name: "leaves everything unset for New to default", config: map[string]string{}, - expected: &MandiPrice.Config{}, + expected: &MandiPrice.Config{Auth: map[string]*upstream.Auth{}}, }, { // Query auth is why this capability has its own entry rather than @@ -49,40 +50,50 @@ func TestParseConfig(t *testing.T) { // expressible. name: "reads the query auth scheme this capability needs", config: map[string]string{ - "bindingKeys": "agmarknet|openagrinet:MandiPrice", - "authScheme": "query", - "queryName": "api-key", - "queryValueEnv": "MANDI_TOKEN", + "bindingKeys": "agmarknet|openagrinet:MandiPrice", + "authScheme-agmarknet": "query", + "queryName-agmarknet": "api-key", + "queryValueEnv-agmarknet": "MANDI_TOKEN", }, expected: &MandiPrice.Config{ - BindingKeys: []string{"agmarknet|openagrinet:MandiPrice"}, - AuthScheme: "query", - QueryName: "api-key", - QueryValueEnv: "MANDI_TOKEN", + BindingKeys: []string{"agmarknet|openagrinet:MandiPrice"}, + Auth: map[string]*upstream.Auth{ + "agmarknet": { + Provider: "agmarknet", + Scheme: "query", + QueryName: "api-key", + QueryValueEnv: "MANDI_TOKEN", + }, + }, }, }, { name: "reads every supported setting", config: map[string]string{ - "bindingKeys": "other|capability", - "authScheme": "basic", - "usernameEnv": "U", - "passwordEnv": "P", - "headerName": "X-Key", - "headerValueEnv": "V", - "queryName": "q", - "queryValueEnv": "Q", - "maxResponseBytes": "2048", + "bindingKeys": "other|capability", + "authScheme-other": "basic", + "usernameEnv-other": "U", + "passwordEnv-other": "P", + "headerName-other": "X-Key", + "headerValueEnv-other": "V", + "queryName-other": "q", + "queryValueEnv-other": "Q", + "maxResponseBytes": "2048", }, expected: &MandiPrice.Config{ - BindingKeys: []string{"other|capability"}, - AuthScheme: "basic", - UsernameEnv: "U", - PasswordEnv: "P", - HeaderName: "X-Key", - HeaderValueEnv: "V", - QueryName: "q", - QueryValueEnv: "Q", + BindingKeys: []string{"other|capability"}, + Auth: map[string]*upstream.Auth{ + "other": { + Provider: "other", + Scheme: "basic", + UsernameEnv: "U", + PasswordEnv: "P", + HeaderName: "X-Key", + HeaderValueEnv: "V", + QueryName: "q", + QueryValueEnv: "Q", + }, + }, MaxResponseBytes: 2048, }, }, @@ -102,7 +113,7 @@ func TestParseConfig(t *testing.T) { // as "unset" rather than failing startup. name: "treats an empty response cap as unset", config: map[string]string{"maxResponseBytes": ""}, - expected: &MandiPrice.Config{}, + expected: &MandiPrice.Config{Auth: map[string]*upstream.Auth{}}, }, } @@ -220,7 +231,10 @@ func TestNew(t *testing.T) { t.Parallel() _, _, err := mandiProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, - map[string]string{"authScheme": "oauth"}) + map[string]string{ + "bindingKeys": "agmarknet|openagrinet:MandiPrice", + "authScheme-agmarknet": "oauth", + }) if err == nil { t.Fatal("expected an unknown auth scheme to be refused") } diff --git a/pkg/plugin/implementation/MandiPrice/mappings_test.go b/pkg/plugin/implementation/MandiPrice/mappings_test.go index ef91a40c..d7ad6557 100644 --- a/pkg/plugin/implementation/MandiPrice/mappings_test.go +++ b/pkg/plugin/implementation/MandiPrice/mappings_test.go @@ -23,6 +23,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/MandiPrice" + upstreamstep "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" ) @@ -192,7 +193,14 @@ func runShippedWith(t *testing.T, request, providerBody string) (url.Values, map }} step, closeStep, err := MandiPrice.New(context.Background(), registry, mapper, - &MandiPrice.Config{BindingKeys: []string{shippedBindingKey}}) + &MandiPrice.Config{ + BindingKeys: []string{shippedBindingKey}, + // Auth is per provider; these upstreams are stubs needing + // no credential, and that is declared rather than defaulted. + Auth: map[string]*upstreamstep.Auth{ + strings.Split(shippedBindingKey, "|")[0]: {Scheme: upstreamstep.AuthSchemeNone}, + }, + }) if err != nil { t.Fatalf("failed to build the step: %v", err) } @@ -432,7 +440,14 @@ func TestShippedMappingRefusesWhatItCannotServe(t *testing.T) { }, }} step, closeStep, err := MandiPrice.New(context.Background(), registry, mapper, - &MandiPrice.Config{BindingKeys: []string{shippedBindingKey}}) + &MandiPrice.Config{ + BindingKeys: []string{shippedBindingKey}, + // Auth is per provider; these upstreams are stubs needing + // no credential, and that is declared rather than defaulted. + Auth: map[string]*upstreamstep.Auth{ + strings.Split(shippedBindingKey, "|")[0]: {Scheme: upstreamstep.AuthSchemeNone}, + }, + }) if err != nil { t.Fatalf("failed to build the step: %v", err) } @@ -741,7 +756,14 @@ func TestShippedMappingRefusesPayloadsItCannotAnswer(t *testing.T) { }, }} step, closeStep, err := MandiPrice.New(context.Background(), registry, mapper, - &MandiPrice.Config{BindingKeys: []string{shippedBindingKey}}) + &MandiPrice.Config{ + BindingKeys: []string{shippedBindingKey}, + // Auth is per provider; these upstreams are stubs needing + // no credential, and that is declared rather than defaulted. + Auth: map[string]*upstreamstep.Auth{ + strings.Split(shippedBindingKey, "|")[0]: {Scheme: upstreamstep.AuthSchemeNone}, + }, + }) if err != nil { t.Fatalf("failed to build the step: %v", err) } diff --git a/pkg/plugin/implementation/WeatherObservation/cmd/plugin.go b/pkg/plugin/implementation/WeatherObservation/cmd/plugin.go index 9a53bf97..fa406f7a 100644 --- a/pkg/plugin/implementation/WeatherObservation/cmd/plugin.go +++ b/pkg/plugin/implementation/WeatherObservation/cmd/plugin.go @@ -10,6 +10,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/WeatherObservation" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" ) // weatherProvider implements definition.ProviderStepProvider. @@ -28,13 +29,6 @@ func (p weatherProvider) parseConfig(config map[string]string) (*WeatherObservat // this is a default rather than something to set. ProviderIDAt: config["providerIdAt"], CapabilityCodeAt: config["capabilityCodeAt"], - AuthScheme: config["authScheme"], - UsernameEnv: config["usernameEnv"], - PasswordEnv: config["passwordEnv"], - HeaderName: config["headerName"], - HeaderValueEnv: config["headerValueEnv"], - QueryName: config["queryName"], - QueryValueEnv: config["queryValueEnv"], } if raw, exists := config["maxResponseBytes"]; exists && raw != "" { @@ -48,6 +42,16 @@ func (p weatherProvider) parseConfig(config map[string]string) (*WeatherObservat cfg.MaxResponseBytes = value } + // One credential profile per provider, read from the flattened + // authScheme- settings. Shared with the other capability + // plugins: each used to copy the same field list, so a scheme added in one + // had to be remembered in three. + auth, err := upstream.ParseAuth(config) + if err != nil { + return nil, err + } + cfg.Auth = auth + return cfg, nil } diff --git a/pkg/plugin/implementation/WeatherObservation/cmd/plugin_test.go b/pkg/plugin/implementation/WeatherObservation/cmd/plugin_test.go index f95323da..73a8087b 100644 --- a/pkg/plugin/implementation/WeatherObservation/cmd/plugin_test.go +++ b/pkg/plugin/implementation/WeatherObservation/cmd/plugin_test.go @@ -10,6 +10,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/WeatherObservation" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" ) type stubRegistry struct{} @@ -40,26 +41,31 @@ func TestParseConfig(t *testing.T) { // rules are defined in exactly one place. name: "leaves everything unset for New to default", config: map[string]string{}, - expected: &WeatherObservation.Config{}, + expected: &WeatherObservation.Config{Auth: map[string]*upstream.Auth{}}, }, { name: "reads every supported setting", config: map[string]string{ - "bindingKeys": "other|capability", - "authScheme": "basic", - "usernameEnv": "U", - "passwordEnv": "P", - "headerName": "X-Key", - "headerValueEnv": "V", - "maxResponseBytes": "2048", + "bindingKeys": "other|capability", + "authScheme-other": "basic", + "usernameEnv-other": "U", + "passwordEnv-other": "P", + "headerName-other": "X-Key", + "headerValueEnv-other": "V", + "maxResponseBytes": "2048", }, expected: &WeatherObservation.Config{ - BindingKeys: []string{"other|capability"}, - AuthScheme: "basic", - UsernameEnv: "U", - PasswordEnv: "P", - HeaderName: "X-Key", - HeaderValueEnv: "V", + BindingKeys: []string{"other|capability"}, + Auth: map[string]*upstream.Auth{ + "other": { + Provider: "other", + Scheme: "basic", + UsernameEnv: "U", + PasswordEnv: "P", + HeaderName: "X-Key", + HeaderValueEnv: "V", + }, + }, MaxResponseBytes: 2048, }, }, @@ -182,7 +188,10 @@ func TestNew(t *testing.T) { t.Parallel() _, _, err := weatherProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, - map[string]string{"authScheme": "oauth"}) + map[string]string{ + "bindingKeys": "mausamgram|openagrinet:WeatherObservation", + "authScheme-mausamgram": "oauth", + }) if err == nil { t.Fatal("expected an unknown auth scheme to be refused") } @@ -204,7 +213,10 @@ func TestNew(t *testing.T) { t.Parallel() step, closer, err := weatherProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, - map[string]string{"bindingKeys": "imd|openagrinet:WeatherObservation"}) + map[string]string{ + "bindingKeys": "imd|openagrinet:WeatherObservation", + "authScheme-imd": "none", + }) if err != nil { t.Fatalf("expected no error, got: %v", err) } diff --git a/pkg/plugin/implementation/WeatherObservation/mappings_test.go b/pkg/plugin/implementation/WeatherObservation/mappings_test.go index d8688193..564534cd 100644 --- a/pkg/plugin/implementation/WeatherObservation/mappings_test.go +++ b/pkg/plugin/implementation/WeatherObservation/mappings_test.go @@ -23,6 +23,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/WeatherObservation" + upstreamstep "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" ) @@ -143,7 +144,14 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { }} step, closeStep, err := WeatherObservation.New(context.Background(), registry, mapper, - &WeatherObservation.Config{BindingKeys: []string{shippedBindingKey}}) + &WeatherObservation.Config{ + BindingKeys: []string{shippedBindingKey}, + // Auth is per provider; these upstreams are stubs needing + // no credential, and that is declared rather than defaulted. + Auth: map[string]*upstreamstep.Auth{ + strings.Split(shippedBindingKey, "|")[0]: {Scheme: upstreamstep.AuthSchemeNone}, + }, + }) if err != nil { t.Fatalf("failed to build the step: %v", err) } diff --git a/pkg/plugin/implementation/internal/upstream/dispatch_test.go b/pkg/plugin/implementation/internal/upstream/dispatch_test.go index 09cceb3e..7509bc8f 100644 --- a/pkg/plugin/implementation/internal/upstream/dispatch_test.go +++ b/pkg/plugin/implementation/internal/upstream/dispatch_test.go @@ -79,7 +79,14 @@ func TestTwoProviderStepsDispatchByBindingKey(t *testing.T) { &stubRegistry{plan: plan}, fixedMapper{answer: answer}, nil, - &upstream.Config{BindingKeys: []string{bindingKey}}) + &upstream.Config{ + BindingKeys: []string{bindingKey}, + // Auth is per provider, so each step declares its own even + // when neither upstream needs a credential. + Auth: map[string]*upstream.Auth{ + strings.Split(bindingKey, "|")[0]: {Scheme: upstream.AuthSchemeNone}, + }, + }) if err != nil { t.Fatal(err) } diff --git a/pkg/plugin/implementation/internal/upstream/perprovider_test.go b/pkg/plugin/implementation/internal/upstream/perprovider_test.go new file mode 100644 index 00000000..3bd3e785 --- /dev/null +++ b/pkg/plugin/implementation/internal/upstream/perprovider_test.go @@ -0,0 +1,264 @@ +package upstream + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/beckn-one/beckn-onix/pkg/model" +) + +// bodyFor builds a select naming a particular provider, so one step serving +// two binding keys can be sent a request for either. +func bodyFor(provider, capability string) string { + return fmt.Sprintf(`{ + "context": { "version": "2.0.0", "action": "select", "transactionId": "txn-1" }, + "message": { "contract": { "commitments": [{ + "resources": [{ "resourceAttributes": { "@type": %q } }], + "offer": { "provider": { "id": %q } } + }] } } + }`, capability, provider) +} + +// planFor is testPlan for an arbitrary provider. +func planFor(provider, capability, baseURL string) *model.ProviderRecord { + return &model.ProviderRecord{ + BindingKey: provider + "|" + capability, + ParticipantID: provider, + CapabilityCode: capability, + BaseURL: baseURL, + Actions: map[string]model.ActionPlan{ + "select": {Method: http.MethodGet, Path: "/x", Mappings: testMappingRef, + TimeoutMs: 2000}, + }, + } +} + +// routingRegistry answers with a different record per binding key, the way the +// real registry does. +type routingRegistry struct { + plans map[string]*model.ProviderRecord +} + +func (r *routingRegistry) ProviderRecord(_ context.Context, key string) (*model.ProviderRecord, error) { + plan, ok := r.plans[key] + if !ok { + return nil, fmt.Errorf("no record for %s", key) + } + return plan, nil +} + +// THE WHOLE POINT OF PER-PROVIDER AUTH. One step, two providers of the same +// capability, each authenticating its own way -- which was impossible while a +// step carried a single scheme. +func TestTwoProvidersOnOneStepAuthenticateDifferently(t *testing.T) { + t.Setenv("TEST_PP_QUERY_TOKEN", "query-secret") + t.Setenv("TEST_PP_HEADER_TOKEN", "header-secret") + + const capability = "openagrinet:KnowledgeAdvisory" + + var gotQuery, gotHeader string + var queryCalls, headerCalls int32 + queryProvider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&queryCalls, 1) + gotQuery = r.URL.Query().Get("token") + fmt.Fprint(w, `{"from":"query"}`) + })) + defer queryProvider.Close() + headerProvider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&headerCalls, 1) + gotHeader = r.Header.Get("X-Api-Key") + fmt.Fprint(w, `{"from":"header"}`) + })) + defer headerProvider.Close() + + cfg := &Config{ + BindingKeys: []string{"alpha|" + capability, "beta|" + capability}, + Auth: map[string]*Auth{ + "alpha": {Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_PP_QUERY_TOKEN"}, + "beta": {Scheme: AuthSchemeHeader, HeaderName: "X-Api-Key", HeaderValueEnv: "TEST_PP_HEADER_TOKEN"}, + }, + } + registry := &routingRegistry{plans: map[string]*model.ProviderRecord{ + "alpha|" + capability: planFor("alpha", capability, queryProvider.URL), + "beta|" + capability: planFor("beta", capability, headerProvider.URL), + }} + step, closer, err := New(context.Background(), registry, + &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{"ok":true}`)}, nil, cfg) + if err != nil { + t.Fatalf("New: %v", err) + } + t.Cleanup(func() { _ = closer() }) + + if _, err := runStep(t, step, bodyFor("alpha", capability)); err != nil { + t.Fatalf("alpha: %v", err) + } + if _, err := runStep(t, step, bodyFor("beta", capability)); err != nil { + t.Fatalf("beta: %v", err) + } + + if queryCalls != 1 || headerCalls != 1 { + t.Fatalf("calls: query=%d header=%d, want 1 each", queryCalls, headerCalls) + } + if gotQuery != "query-secret" { + t.Errorf("alpha received token %q, want its own query credential", gotQuery) + } + if gotHeader != "header-secret" { + t.Errorf("beta received X-Api-Key %q, want its own header credential", gotHeader) + } +} + +// A step-wide token cache would hand the first provider's token to the second, +// which is authenticating as somebody else. Each profile holds its own, so two +// issuers are exchanged with separately and neither token crosses over. +func TestTwoOAuth2ProvidersDoNotShareAToken(t *testing.T) { + t.Setenv("TEST_PP_ID_A", "id-a") + t.Setenv("TEST_PP_SECRET_A", "secret-a") + t.Setenv("TEST_PP_ID_B", "id-b") + t.Setenv("TEST_PP_SECRET_B", "secret-b") + + const capability = "openagrinet:KnowledgeAdvisory" + + var exchangesA, exchangesB int32 + issuerA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&exchangesA, 1) + fmt.Fprint(w, `{"access_token":"token-A","expires_in":3600}`) + })) + defer issuerA.Close() + issuerB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&exchangesB, 1) + fmt.Fprint(w, `{"access_token":"token-B","expires_in":3600}`) + })) + defer issuerB.Close() + + var mu sync.Mutex + seen := map[string][]string{} + record := func(provider string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + seen[provider] = append(seen[provider], + strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")) + mu.Unlock() + fmt.Fprint(w, `{"ok":true}`) + } + } + providerA := httptest.NewServer(record("alpha")) + defer providerA.Close() + providerB := httptest.NewServer(record("beta")) + defer providerB.Close() + + cfg := &Config{ + BindingKeys: []string{"alpha|" + capability, "beta|" + capability}, + Auth: map[string]*Auth{ + "alpha": {Scheme: AuthSchemeOAuth2, TokenURL: issuerA.URL, + ClientIDEnv: "TEST_PP_ID_A", ClientSecretEnv: "TEST_PP_SECRET_A"}, + "beta": {Scheme: AuthSchemeOAuth2, TokenURL: issuerB.URL, + ClientIDEnv: "TEST_PP_ID_B", ClientSecretEnv: "TEST_PP_SECRET_B"}, + }, + } + registry := &routingRegistry{plans: map[string]*model.ProviderRecord{ + "alpha|" + capability: planFor("alpha", capability, providerA.URL), + "beta|" + capability: planFor("beta", capability, providerB.URL), + }} + step, closer, err := New(context.Background(), registry, + &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{"ok":true}`)}, nil, cfg) + if err != nil { + t.Fatalf("New: %v", err) + } + t.Cleanup(func() { _ = closer() }) + + // Twice each: the second call of a pair must reuse that provider's token. + for i := 0; i < 2; i++ { + if _, err := runStep(t, step, bodyFor("alpha", capability)); err != nil { + t.Fatalf("alpha call %d: %v", i, err) + } + if _, err := runStep(t, step, bodyFor("beta", capability)); err != nil { + t.Fatalf("beta call %d: %v", i, err) + } + } + + if exchangesA != 1 || exchangesB != 1 { + t.Errorf("exchanges: A=%d B=%d, want 1 each -- each profile caches its own", + exchangesA, exchangesB) + } + mu.Lock() + defer mu.Unlock() + for _, got := range seen["alpha"] { + if got != "token-A" { + t.Errorf("alpha was sent %q, want token-A -- a token crossed between providers", got) + } + } + for _, got := range seen["beta"] { + if got != "token-B" { + t.Errorf("beta was sent %q, want token-B -- a token crossed between providers", got) + } + } + if len(seen["alpha"]) != 2 || len(seen["beta"]) != 2 { + t.Errorf("calls: alpha=%d beta=%d, want 2 each", len(seen["alpha"]), len(seen["beta"])) + } +} + +// Redaction is about what could appear in a piece of text, not about which +// provider the request was for. An error raised while serving one provider can +// quote another's credential, so every profile's secrets are covered. +func TestRedactionCoversEveryProvidersSecret(t *testing.T) { + t.Setenv("TEST_PP_RED_A", "secret-of-alpha") + t.Setenv("TEST_PP_RED_B", "secret-of-beta") + + const capability = "openagrinet:KnowledgeAdvisory" + cfg := &Config{ + BindingKeys: []string{"alpha|" + capability, "beta|" + capability}, + Auth: map[string]*Auth{ + "alpha": {Scheme: AuthSchemeHeader, HeaderName: "X-A", HeaderValueEnv: "TEST_PP_RED_A"}, + "beta": {Scheme: AuthSchemeHeader, HeaderName: "X-B", HeaderValueEnv: "TEST_PP_RED_B"}, + }, + } + step, closer, err := New(context.Background(), &routingRegistry{}, &stubMapper{}, nil, cfg) + if err != nil { + t.Fatalf("New: %v", err) + } + t.Cleanup(func() { _ = closer() }) + + text := "provider said no: alpha=secret-of-alpha beta=secret-of-beta" + got := step.redactString(text) + if strings.Contains(got, "secret-of-alpha") { + t.Errorf("alpha's credential survived redaction: %s", got) + } + if strings.Contains(got, "secret-of-beta") { + t.Errorf("beta's credential survived redaction: %s", got) + } + if !strings.Contains(got, "provider said no") { + t.Errorf("redaction ate the message: %s", got) + } +} + +// One provider's credential can be a substring of another's. The longer has to +// be replaced first, or it is left half-redacted with the tail exposed. +func TestRedactionSortsAcrossProvidersNotWithinOne(t *testing.T) { + t.Setenv("TEST_PP_SHORT", "abc") + t.Setenv("TEST_PP_LONG", "abcdef") + + const capability = "openagrinet:KnowledgeAdvisory" + cfg := &Config{ + BindingKeys: []string{"shorty|" + capability, "longy|" + capability}, + Auth: map[string]*Auth{ + "shorty": {Scheme: AuthSchemeHeader, HeaderName: "X-S", HeaderValueEnv: "TEST_PP_SHORT"}, + "longy": {Scheme: AuthSchemeHeader, HeaderName: "X-L", HeaderValueEnv: "TEST_PP_LONG"}, + }, + } + step, closer, err := New(context.Background(), &routingRegistry{}, &stubMapper{}, nil, cfg) + if err != nil { + t.Fatalf("New: %v", err) + } + t.Cleanup(func() { _ = closer() }) + + got := step.redactString("token=abcdef") + if strings.Contains(got, "def") { + t.Errorf("the longer credential was half-redacted, leaving its tail: %s", got) + } +} diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index 2478aca1..893633da 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -152,39 +152,88 @@ type Config struct { ProviderIDAt string `yaml:"providerIdAt" json:"providerIdAt"` CapabilityCodeAt string `yaml:"capabilityCodeAt" json:"capabilityCodeAt"` - // AuthScheme is how credentials are presented upstream: none, basic or - // header. Providers differ here -- basic auth, a raw token header, a field - // in the body -- which is why it is configuration and not an assumption. - AuthScheme string `yaml:"authScheme" json:"authScheme"` + // Auth carries one credential profile PER PROVIDER, keyed by participant + // id -- the left half of a binding key. + // + // Per provider rather than per step because a step serves several binding + // keys and the providers behind them need not authenticate alike: one may + // take an oauth2 client id and secret, the next a token in a query + // parameter. Everything else that differs per provider already comes from + // the registry -- the endpoint, the path, the mapping -- so auth was the + // only thing pinned to the step, and the only thing that made a second + // provider of the same capability impossible to configure. + // + // There is deliberately NO step-wide default. A profile per provider means + // there is never a question of which setting applies, and a provider whose + // profile is missing is refused at startup rather than quietly falling + // through to sending nothing. + // + // Built by ParseAuth from the flattened config, not decoded from YAML + // directly: what an operator writes is a nested block per provider, which + // pkg/plugin flattens on the way in. + Auth map[string]*Auth `yaml:"-" json:"-"` + + // MaxResponseBytes caps what is read from the provider. + MaxResponseBytes int64 `yaml:"maxResponseBytes" json:"maxResponseBytes"` +} + +// Auth is how one provider's credentials are presented upstream. +type Auth struct { + // Provider is the participant id this profile belongs to. Held so an error + // can name it: with several profiles on one step, "authScheme query + // requires queryName" would otherwise leave an operator guessing which + // provider it meant. + Provider string + + // Scheme is one of none, basic, header, query or oauth2. Providers differ + // here, which is why it is configuration and not an assumption. + Scheme string // UsernameEnv and PasswordEnv name the environment variables holding basic // credentials. They are variable NAMES, never the values. - UsernameEnv string `yaml:"usernameEnv" json:"usernameEnv"` - PasswordEnv string `yaml:"passwordEnv" json:"passwordEnv"` + UsernameEnv string + PasswordEnv string - // HeaderName and HeaderValueEnv configure the header scheme: which header to - // set, and which environment variable holds its value. - HeaderName string `yaml:"headerName" json:"headerName"` - HeaderValueEnv string `yaml:"headerValueEnv" json:"headerValueEnv"` + // HeaderName and HeaderValueEnv configure the header scheme: which header + // to set, and which environment variable holds its value. + HeaderName string + HeaderValueEnv string - // QueryName and QueryValueEnv configure authScheme query: the parameter + // QueryName and QueryValueEnv configure the query scheme: the parameter // name to add, and the environment variable holding its value. Named the // same way as the header pair, for the same reason -- the credential is // never in this config, only the name of the variable carrying it. - QueryName string `yaml:"queryName" json:"queryName"` - QueryValueEnv string `yaml:"queryValueEnv" json:"queryValueEnv"` + QueryName string + QueryValueEnv string // TokenURL is the OAuth2 token endpoint. Not a credential, so it is named // here rather than through an environment variable -- but it IS // deployment-specific, so the reference config carries a placeholder. - TokenURL string `yaml:"tokenUrl" json:"tokenUrl"` + TokenURL string // ClientIDEnv and ClientSecretEnv name the variables holding the client // credentials. The values never appear in config, the registry, or a log. - ClientIDEnv string `yaml:"clientIdEnv" json:"clientIdEnv"` - ClientSecretEnv string `yaml:"clientSecretEnv" json:"clientSecretEnv"` + ClientIDEnv string + ClientSecretEnv string +} - // MaxResponseBytes caps what is read from the provider. - MaxResponseBytes int64 `yaml:"maxResponseBytes" json:"maxResponseBytes"` +// authenticator is one provider's profile plus the token it holds. +// +// The token cache lives HERE rather than on the Step, and that is the whole +// reason this type exists. A step-wide cache shared between two oauth2 +// providers would hand the first provider's token to the second, which is +// authenticating as somebody else -- a failure no test of a single provider +// can see. +type authenticator struct { + cfg Auth + + // Two mechanisms because there are two jobs. tokenMu serialises the + // EXCHANGE, so a cold start sends one request to the issuer rather than one + // per concurrent caller. token is atomic so READERS never take that mutex, + // which matters because secretForms is one of them and it is reached from + // inside the exchange -- guarding the value with tokenMu instead deadlocked + // on the first failing exchange, which is how this was found. + tokenMu sync.Mutex + token atomic.Pointer[cachedToken] } // Step serves whatever capabilities a domain package configures it for. It is @@ -197,16 +246,10 @@ type Step struct { mapper definition.Mapper httpClient *http.Client - // The oauth2 token this step holds. - // - // Two mechanisms because there are two jobs. tokenMu serialises the - // EXCHANGE, so a cold start sends one request to the issuer rather than one - // per concurrent caller. token is atomic so READERS never take that mutex, - // which matters because secretForms is one of them and it is reached from - // inside the exchange -- guarding the value with tokenMu instead deadlocked - // on the first failing exchange, which is how this was found. - tokenMu sync.Mutex - token atomic.Pointer[cachedToken] + // One authenticator per provider, keyed by participant id. Built once at + // startup, so a request only looks one up -- and each holds its own token, + // so two oauth2 providers cannot share one. + auth map[string]*authenticator } // New creates the step. @@ -239,6 +282,10 @@ func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper d // Timeout is set per request from the registry's own budget, so the // client carries none of its own. httpClient: &http.Client{}, + auth: make(map[string]*authenticator, len(cfg.Auth)), + } + for provider, profile := range cfg.Auth { + step.auth[provider] = &authenticator{cfg: *profile} } closer := func() error { @@ -286,39 +333,192 @@ func applyDefaults(cfg *Config) error { return errors.New("upstream: bindingKeys carries an empty entry") } } - if cfg.AuthScheme == "" { - cfg.AuthScheme = AuthSchemeNone - } if cfg.MaxResponseBytes <= 0 { cfg.MaxResponseBytes = DefaultMaxResponseBytes } + if cfg.Auth == nil { + cfg.Auth = map[string]*Auth{} + } + + // Both directions, because each catches a different mistake and both are + // silent at runtime. A profile for a provider this step does not serve is a + // typo that would apply to nothing; a served provider with no profile would + // fall through to sending no credential and read as the provider rejecting + // us. + served := map[string]bool{} + for _, key := range cfg.BindingKeys { + served[participantOf(key)] = true + } + for provider := range cfg.Auth { + if !served[provider] { + return fmt.Errorf( + "upstream: auth is configured for %q, which is not a provider in bindingKeys (%s)", + provider, strings.Join(cfg.BindingKeys, ", ")) + } + } + for provider := range served { + profile, ok := cfg.Auth[provider] + if !ok { + return fmt.Errorf( + "upstream: %q is served but has no auth block; every provider declares its own, "+ + "using authScheme none where the upstream needs no credential", provider) + } + profile.Provider = provider + if err := profile.validate(); err != nil { + return err + } + } + return nil +} - switch cfg.AuthScheme { +// validate refuses a profile whose scheme and fields disagree. Every message +// names the provider: with several profiles on one step, the field alone would +// leave an operator guessing which block to look at. +func (a *Auth) validate() error { + switch a.Scheme { case AuthSchemeNone: case AuthSchemeBasic: - if cfg.UsernameEnv == "" || cfg.PasswordEnv == "" { - return errors.New("upstream: authScheme basic requires usernameEnv and passwordEnv") + if a.UsernameEnv == "" || a.PasswordEnv == "" { + return fmt.Errorf( + "upstream: %s: authScheme basic requires usernameEnv and passwordEnv", a.Provider) } case AuthSchemeHeader: - if cfg.HeaderName == "" || cfg.HeaderValueEnv == "" { - return errors.New("upstream: authScheme header requires headerName and headerValueEnv") + if a.HeaderName == "" || a.HeaderValueEnv == "" { + return fmt.Errorf( + "upstream: %s: authScheme header requires headerName and headerValueEnv", a.Provider) } case AuthSchemeQuery: - if cfg.QueryName == "" || cfg.QueryValueEnv == "" { - return errors.New("upstream: authScheme query requires queryName and queryValueEnv") + if a.QueryName == "" || a.QueryValueEnv == "" { + return fmt.Errorf( + "upstream: %s: authScheme query requires queryName and queryValueEnv", a.Provider) } case AuthSchemeOAuth2: - if cfg.TokenURL == "" || cfg.ClientIDEnv == "" || cfg.ClientSecretEnv == "" { - return errors.New( - "upstream: authScheme oauth2 requires tokenUrl, clientIdEnv and clientSecretEnv") + if a.TokenURL == "" || a.ClientIDEnv == "" || a.ClientSecretEnv == "" { + return fmt.Errorf( + "upstream: %s: authScheme oauth2 requires tokenUrl, clientIdEnv and clientSecretEnv", + a.Provider) } + case "": + return fmt.Errorf("upstream: %s: authScheme is required, "+ + "and is none where the upstream needs no credential", a.Provider) default: return fmt.Errorf( - "upstream: unknown authScheme %q: must be none, basic, header, query or oauth2", cfg.AuthScheme) + "upstream: %s: unknown authScheme %q: must be none, basic, header, query or oauth2", + a.Provider, a.Scheme) } return nil } +// ParseAuth builds one credential profile per provider from a plugin's +// flattened settings. +// +// What an operator writes is a block per provider: +// +// knowledge-provider: +// authScheme: oauth2 +// tokenUrl: https://issuer.example/token +// +// which pkg/plugin flattens to authScheme-knowledge-provider and +// tokenUrl-knowledge-provider before any plugin sees it. This reads that form +// back into profiles. +// +// Shared rather than repeated in each capability plugin: all of them copied +// the same field list out of the same map, so a scheme added in one place had +// to be remembered in three. +// +// The split is on the FIRST dash, which is unambiguous because no setting name +// contains one while a participant id routinely does -- knowledge-provider, +// provider.oan.dev. So the field is always the part before it. +func ParseAuth(config map[string]string) (map[string]*Auth, error) { + profiles := map[string]*Auth{} + + // Sorted so a config with two mistakes reports the same one every run. + keys := make([]string, 0, len(config)) + for key := range config { + keys = append(keys, key) + } + sort.Strings(keys) + + for _, key := range keys { + field, provider, dashed := strings.Cut(key, "-") + if !dashed { + // A bare auth field is the old step-wide form. Refused rather than + // ignored: silently dropping it leaves every provider on no + // credential at all, which reads as the provider rejecting us. + if authFields[key] { + return nil, fmt.Errorf( + "upstream: %q is set for the whole step; auth is per provider now, "+ + "so it belongs in a block named for the participant id", key) + } + continue + } + if !authFields[field] { + // Not an auth setting, and nothing else on a provider step carries + // a dash -- so this is a misspelled field rather than something to + // pass through. + return nil, fmt.Errorf("upstream: %q is not a credential setting", key) + } + if strings.TrimSpace(provider) == "" { + return nil, fmt.Errorf("upstream: %q names no provider after the dash", key) + } + + profile, seen := profiles[provider] + if !seen { + profile = &Auth{Provider: provider} + profiles[provider] = profile + } + value := config[key] + switch field { + case "authScheme": + profile.Scheme = value + case "usernameEnv": + profile.UsernameEnv = value + case "passwordEnv": + profile.PasswordEnv = value + case "headerName": + profile.HeaderName = value + case "headerValueEnv": + profile.HeaderValueEnv = value + case "queryName": + profile.QueryName = value + case "queryValueEnv": + profile.QueryValueEnv = value + case "tokenUrl": + profile.TokenURL = value + case "clientIdEnv": + profile.ClientIDEnv = value + case "clientSecretEnv": + profile.ClientSecretEnv = value + } + } + return profiles, nil +} + +// authFields is the closed set of per-provider credential settings. Closed on +// purpose: it is what makes the split on the first dash decidable, and what +// turns a misspelled field into a startup error rather than a setting that +// quietly does nothing. +var authFields = map[string]bool{ + "authScheme": true, + "usernameEnv": true, + "passwordEnv": true, + "headerName": true, + "headerValueEnv": true, + "queryName": true, + "queryValueEnv": true, + "tokenUrl": true, + "clientIdEnv": true, + "clientSecretEnv": true, +} + +// participantOf returns the provider half of a binding key. The format is +// "|", and a participant id carries dashes and +// dots but never a pipe, so the first one separates them. +func participantOf(bindingKey string) string { + provider, _, _ := strings.Cut(bindingKey, "|") + return strings.TrimSpace(provider) +} + // Run serves the request when it is for this step's capability, and does // nothing when it is not. // @@ -429,7 +629,16 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { return err } - upstreamResponse, err := s.call(ctx, plan.BaseURL, call, upstreamRequest) + // Which credential this provider takes. Resolved from the binding key, so + // one step serving several providers authenticates each as its own. + // Startup guarantees a profile per served provider; this guards the case + // where a record arrives for a key the config never declared. + auth, configured := s.auth[participantOf(plan.BindingKey)] + if !configured { + return fmt.Errorf("upstream: no credential is configured for %s", plan.BindingKey) + } + + upstreamResponse, err := s.call(ctx, auth, plan.BaseURL, call, upstreamRequest) if err != nil { return err } @@ -541,7 +750,7 @@ func budget(call model.ActionPlan) (time.Duration, int) { return timeout, retries } -func (s *Step) call(ctx context.Context, baseURL string, call model.ActionPlan, mapped []byte) ([]byte, error) { +func (s *Step) call(ctx context.Context, auth *authenticator, baseURL string, call model.ActionPlan, mapped []byte) ([]byte, error) { endpoint, err := buildEndpoint(baseURL, call, mapped) if err != nil { return nil, err @@ -569,7 +778,7 @@ func (s *Step) call(ctx context.Context, baseURL string, call model.ActionPlan, break } - body, err := s.attempt(ctx, call, endpoint, mapped, timeout) + body, err := s.attempt(ctx, auth, call, endpoint, mapped, timeout) if err == nil { return body, nil } @@ -649,7 +858,7 @@ func sleep(ctx context.Context, d time.Duration) error { } // attempt makes one upstream request. -func (s *Step) attempt(ctx context.Context, call model.ActionPlan, endpoint string, mapped []byte, timeout time.Duration) ([]byte, error) { +func (s *Step) attempt(ctx context.Context, auth *authenticator, call model.ActionPlan, endpoint string, mapped []byte, timeout time.Duration) ([]byte, error) { attemptCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() @@ -661,7 +870,7 @@ func (s *Step) attempt(ctx context.Context, call model.ActionPlan, endpoint stri if hasBody(method) { req.Header.Set("Content-Type", "application/json") } - if err := s.authenticate(req); err != nil { + if err := s.authenticate(auth, req); err != nil { // A missing or unreadable credential is configuration, not weather. return nil, doNotRetry(err) } @@ -750,40 +959,42 @@ func explain(body []byte) string { // else's stack for no benefit to the caller -- the caller cannot set it, and // the fix is entirely the operator's. So the name goes to the log, where the // operator is, and the wire gets the scheme that failed. -func (s *Step) missingCredential(ctx context.Context, scheme, envNames string) error { - err := fmt.Errorf("upstream: this provider's %s credential is not configured", scheme) - log.Errorf(ctx, err, "upstream: %s auth is configured but %s is not set", scheme, envNames) +func (s *Step) missingCredential(ctx context.Context, provider, scheme, envNames string) error { + err := fmt.Errorf("upstream: the %s credential for %s is not configured", scheme, provider) + log.Errorf(ctx, err, "upstream: %s auth is configured for %s but %s is not set", + scheme, provider, envNames) return err } -func (s *Step) authenticate(req *http.Request) error { - switch s.config.AuthScheme { +func (s *Step) authenticate(auth *authenticator, req *http.Request) error { + cfg := auth.cfg + switch cfg.Scheme { case AuthSchemeBasic: - username, password := os.Getenv(s.config.UsernameEnv), os.Getenv(s.config.PasswordEnv) + username, password := os.Getenv(cfg.UsernameEnv), os.Getenv(cfg.PasswordEnv) if username == "" || password == "" { - return s.missingCredential(req.Context(), "basic", - s.config.UsernameEnv+" and "+s.config.PasswordEnv) + return s.missingCredential(req.Context(), cfg.Provider, "basic", + cfg.UsernameEnv+" and "+cfg.PasswordEnv) } req.SetBasicAuth(username, password) case AuthSchemeHeader: - value := os.Getenv(s.config.HeaderValueEnv) + value := os.Getenv(cfg.HeaderValueEnv) if value == "" { - return s.missingCredential(req.Context(), "header", s.config.HeaderValueEnv) + return s.missingCredential(req.Context(), cfg.Provider, "header", cfg.HeaderValueEnv) } - req.Header.Set(s.config.HeaderName, value) + req.Header.Set(cfg.HeaderName, value) case AuthSchemeQuery: - value := os.Getenv(s.config.QueryValueEnv) + value := os.Getenv(cfg.QueryValueEnv) if value == "" { - return s.missingCredential(req.Context(), "query", s.config.QueryValueEnv) + return s.missingCredential(req.Context(), cfg.Provider, "query", cfg.QueryValueEnv) } // Set rather than Add: a second copy of the parameter is not a // credential, it is an ambiguity, and which one an upstream reads is // its own business. query := req.URL.Query() - query.Set(s.config.QueryName, value) + query.Set(cfg.QueryName, value) req.URL.RawQuery = query.Encode() case AuthSchemeOAuth2: - token, err := s.bearerToken(req.Context()) + token, err := s.bearerToken(req.Context(), auth) if err != nil { return err } @@ -850,25 +1061,25 @@ func (e redactedErr) Unwrap() error { return e.err } // NOTHING IS CACHED ON FAILURE. A transient outage at the token endpoint must // not leave this step holding a failure for the lifetime it never obtained -- // the next request tries again. -func (s *Step) bearerToken(ctx context.Context) (string, error) { - if held := s.token.Load(); held != nil && time.Now().Before(held.expiry) { +func (s *Step) bearerToken(ctx context.Context, auth *authenticator) (string, error) { + if held := auth.token.Load(); held != nil && time.Now().Before(held.expiry) { return held.value, nil } - s.tokenMu.Lock() - defer s.tokenMu.Unlock() + auth.tokenMu.Lock() + defer auth.tokenMu.Unlock() // Re-checked after acquiring: while this caller waited, whoever held the // mutex may already have exchanged a fresh token. - if held := s.token.Load(); held != nil && time.Now().Before(held.expiry) { + if held := auth.token.Load(); held != nil && time.Now().Before(held.expiry) { return held.value, nil } - token, lifetime, err := s.exchangeToken(ctx) + token, lifetime, err := s.exchangeToken(ctx, auth) if err != nil { return "", err } - s.token.Store(&cachedToken{value: token, expiry: time.Now().Add(lifetime)}) + auth.token.Store(&cachedToken{value: token, expiry: time.Now().Add(lifetime)}) return token, nil } @@ -889,11 +1100,12 @@ type tokenResponse struct { // Every failure here is the UPSTREAM exchange failing, so all of them carry // 502. Unclassified they would surface as a 500, which tells a network peer // this adapter broke when in fact the provider's issuer did. -func (s *Step) exchangeToken(ctx context.Context) (string, time.Duration, error) { - clientID, clientSecret := os.Getenv(s.config.ClientIDEnv), os.Getenv(s.config.ClientSecretEnv) +func (s *Step) exchangeToken(ctx context.Context, auth *authenticator) (string, time.Duration, error) { + cfg := auth.cfg + clientID, clientSecret := os.Getenv(cfg.ClientIDEnv), os.Getenv(cfg.ClientSecretEnv) if clientID == "" || clientSecret == "" { - return "", 0, s.missingCredential(ctx, "oauth2", - s.config.ClientIDEnv+" and "+s.config.ClientSecretEnv) + return "", 0, s.missingCredential(ctx, cfg.Provider, "oauth2", + cfg.ClientIDEnv+" and "+cfg.ClientSecretEnv) } form := url.Values{ @@ -901,7 +1113,7 @@ func (s *Step) exchangeToken(ctx context.Context) (string, time.Duration, error) "client_id": {clientID}, "client_secret": {clientSecret}, } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.config.TokenURL, + req, err := http.NewRequestWithContext(ctx, http.MethodPost, cfg.TokenURL, strings.NewReader(form.Encode())) if err != nil { return "", 0, s.tokenErr(fmt.Errorf("token request could not be built: %w", err)) @@ -911,7 +1123,7 @@ func (s *Step) exchangeToken(ctx context.Context) (string, time.Duration, error) resp, err := s.httpClient.Do(req) if err != nil { return "", 0, s.tokenErr(fmt.Errorf("token endpoint %s could not be reached: %w", - s.config.TokenURL, err)) + cfg.TokenURL, err)) } defer resp.Body.Close() @@ -925,24 +1137,24 @@ func (s *Step) exchangeToken(ctx context.Context) (string, time.Duration, error) // The status, not the body: what an issuer puts in a failure body is its // own business, and it routinely quotes the request back. log.Warnf(ctx, "upstream: token endpoint %s returned %s: %s", - s.config.TokenURL, resp.Status, s.redactString(explain(body))) + cfg.TokenURL, resp.Status, s.redactString(explain(body))) return "", 0, s.tokenErr(fmt.Errorf("token endpoint returned %s", resp.Status)) } var parsed tokenResponse if err := json.Unmarshal(body, &parsed); err != nil { return "", 0, s.tokenErr(fmt.Errorf("token response from %s is not JSON: %w", - s.config.TokenURL, err)) + cfg.TokenURL, err)) } if parsed.AccessToken == "" { return "", 0, s.tokenErr(fmt.Errorf("token response from %s carries no access_token", - s.config.TokenURL)) + cfg.TokenURL)) } lifetime, ok := tokenLifetime(parsed.ExpiresIn, tokenRefreshSkew) if !ok { return "", 0, s.tokenErr(fmt.Errorf( "token response from %s carries no usable expires_in, so its lifetime is unknown", - s.config.TokenURL)) + cfg.TokenURL)) } return parsed.AccessToken, lifetime, nil } @@ -1025,9 +1237,46 @@ func (s *Step) redactString(text string) string { // from the config rather than from the request still quotes the credential // unwrapped. func (s *Step) secretForms() []string { - switch s.config.AuthScheme { + // EVERY profile, not the one being served. + // + // Redaction is about what could appear in a piece of text, not about which + // provider a request happened to be for. An error or a log line built while + // serving one provider can quote another's credential -- a shared client + // echoing a header, an issuer naming the wrong caller -- and scoping this + // to the active profile would let that reach the log unredacted. There are + // a handful of profiles and this runs on a failure path, so walking all of + // them costs nothing worth measuring. + var forms []string + for _, auth := range s.auth { + forms = append(forms, auth.secretForms()...) + } + // Sorted across the merged set rather than per profile: one provider's + // short token can be a substring of another's, and replacing the short one + // first would leave the longer half-redacted. + return longestFirst(forms) +} + +// secretForms returns every form this profile's credential can appear in. +// +// Per scheme, because the schemes leak differently and redacting the value we +// hold is not enough on its own: +// +// - basic wraps the pair: SetBasicAuth sends base64(user:pass), so the +// password alone does not appear on the wire and replacing it misses the +// echoed header entirely. +// - query escapes: authenticate goes through url.Values.Encode, so a base64 +// token carrying "+", "/" or "=" appears as "a%2Bb%2Fc%3D". Escaping what +// we hold is exact -- same function Encode used, so the two agree by +// construction rather than by a guess about which characters matter. +// - header sends the value as-is. +// +// The raw form is kept alongside the wrapped one in both cases: an error built +// from the config rather than from the request still quotes the credential +// unwrapped. +func (a *authenticator) secretForms() []string { + switch a.cfg.Scheme { case AuthSchemeBasic: - username, password := os.Getenv(s.config.UsernameEnv), os.Getenv(s.config.PasswordEnv) + username, password := os.Getenv(a.cfg.UsernameEnv), os.Getenv(a.cfg.PasswordEnv) if password == "" { return nil } @@ -1041,9 +1290,9 @@ func (s *Step) secretForms() []string { // authenticates, and it is routinely a short common word -- redacting // "user" or "admin" would eat unrelated text and cost the operator the // log line they came for. The pair and the password are the secrets. - return longestFirst(forms) + return forms case AuthSchemeHeader: - value := os.Getenv(s.config.HeaderValueEnv) + value := os.Getenv(a.cfg.HeaderValueEnv) if value == "" { return nil } @@ -1053,19 +1302,19 @@ func (s *Step) secretForms() []string { // it gave back. The token is the one that reaches the provider, so it // is the one an echoing 401 body quotes. var forms []string - if secret := os.Getenv(s.config.ClientSecretEnv); secret != "" { + if secret := os.Getenv(a.cfg.ClientSecretEnv); secret != "" { forms = append(forms, secret) } // Read without tokenMu: this is reached from inside the exchange, which // holds it. - if held := s.token.Load(); held != nil && held.value != "" { + if held := a.token.Load(); held != nil && held.value != "" { forms = append(forms, held.value) } // The client id is deliberately NOT redacted: it identifies, it does // not authenticate, and it is what makes a log line useful. - return longestFirst(forms) + return forms case AuthSchemeQuery: - value := os.Getenv(s.config.QueryValueEnv) + value := os.Getenv(a.cfg.QueryValueEnv) if value == "" { return nil } @@ -1073,7 +1322,7 @@ func (s *Step) secretForms() []string { if encoded := url.QueryEscape(value); encoded != value { forms = append(forms, encoded) } - return longestFirst(forms) + return forms } return nil } diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go index 3031c554..d1479a0b 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -48,6 +48,39 @@ func (s *stubRegistry) ProviderRecord(context.Context, string) (*model.ProviderR // default any more: this package serves whatever a domain package points it at. const testBindingKey = "mausamgram|openagrinet:WeatherObservation" +// testProvider is the participant half of testBindingKey. Auth is per +// provider, so a step config needs a profile under this key. +const testProvider = "mausamgram" + +// setAuth gives every provider this config serves the same profile. Keyed off +// BindingKeys rather than a constant, so a test that serves a different +// provider does not also have to remember to move its auth. +func (c *Config) setAuth(a Auth) { + c.Auth = map[string]*Auth{} + for _, key := range c.BindingKeys { + provider := participantOf(key) + profile := a + profile.Provider = provider + c.Auth[provider] = &profile + } +} + +// authFor is setAuth for a Config literal. +func authFor(a Auth) map[string]*Auth { + a.Provider = testProvider + return map[string]*Auth{testProvider: &a} +} + +// stepWithAuth builds a Step directly, for the handful of tests that exercise +// redaction without a request. New() is the path everything else takes. +func stepWithAuth(a Auth) *Step { + a.Provider = testProvider + return &Step{ + config: &Config{BindingKeys: []string{testBindingKey}, Auth: authFor(a)}, + auth: map[string]*authenticator{testProvider: {cfg: a}}, + } +} + const testMappingRef = "https://mappings.example.com/mausamgram/weather-observation.select.yaml" // assert what reached the mapping without writing one. @@ -110,6 +143,12 @@ func newStep(t *testing.T, registry definition.ProviderRecordLookup, mapper defi for _, apply := range tweak { apply(cfg) } + // There is no step-wide default, so a served provider without a profile is + // refused at startup. Filled in after the tweaks, because a tweak may + // replace BindingKeys and the profile has to follow. + if len(cfg.Auth) == 0 { + cfg.setAuth(Auth{Scheme: AuthSchemeNone}) + } step, closer, err := New(context.Background(), registry, mapper, nil, cfg) if err != nil { t.Fatalf("New() returned an unexpected error: %v", err) @@ -145,21 +184,26 @@ func TestNewValidatesTheAuthScheme(t *testing.T) { config *Config valid bool }{ - {"none by default", &Config{}, true}, - {"basic with both variables", &Config{AuthScheme: AuthSchemeBasic, UsernameEnv: "U", PasswordEnv: "P"}, true}, - {"basic missing the password variable", &Config{AuthScheme: AuthSchemeBasic, UsernameEnv: "U"}, false}, - {"basic missing the username variable", &Config{AuthScheme: AuthSchemeBasic, PasswordEnv: "P"}, false}, - {"header with both settings", &Config{AuthScheme: AuthSchemeHeader, HeaderName: "X-Key", HeaderValueEnv: "V"}, true}, - {"header missing the value variable", &Config{AuthScheme: AuthSchemeHeader, HeaderName: "X-Key"}, false}, - {"an unknown scheme", &Config{AuthScheme: "oauth"}, false}, - {"oauth2 with all three settings", &Config{AuthScheme: AuthSchemeOAuth2, - TokenURL: "https://issuer.invalid/token", ClientIDEnv: "ID", ClientSecretEnv: "SECRET"}, true}, - {"oauth2 missing the token url", &Config{AuthScheme: AuthSchemeOAuth2, - ClientIDEnv: "ID", ClientSecretEnv: "SECRET"}, false}, - {"oauth2 missing the client id variable", &Config{AuthScheme: AuthSchemeOAuth2, - TokenURL: "https://issuer.invalid/token", ClientSecretEnv: "SECRET"}, false}, - {"oauth2 missing the client secret variable", &Config{AuthScheme: AuthSchemeOAuth2, - TokenURL: "https://issuer.invalid/token", ClientIDEnv: "ID"}, false}, + {"none, declared", &Config{Auth: authFor(Auth{Scheme: AuthSchemeNone})}, true}, + {"no profile at all for a served provider", &Config{}, false}, + {"an empty scheme", &Config{Auth: authFor(Auth{})}, false}, + {"basic with both variables", &Config{Auth: authFor(Auth{Scheme: AuthSchemeBasic, UsernameEnv: "U", PasswordEnv: "P"})}, true}, + {"basic missing the password variable", &Config{Auth: authFor(Auth{Scheme: AuthSchemeBasic, UsernameEnv: "U"})}, false}, + {"basic missing the username variable", &Config{Auth: authFor(Auth{Scheme: AuthSchemeBasic, PasswordEnv: "P"})}, false}, + {"header with both settings", &Config{Auth: authFor(Auth{Scheme: AuthSchemeHeader, HeaderName: "X-Key", HeaderValueEnv: "V"})}, true}, + {"header missing the value variable", &Config{Auth: authFor(Auth{Scheme: AuthSchemeHeader, HeaderName: "X-Key"})}, false}, + {"query with both settings", &Config{Auth: authFor(Auth{Scheme: AuthSchemeQuery, QueryName: "t", QueryValueEnv: "V"})}, true}, + {"an unknown scheme", &Config{Auth: authFor(Auth{Scheme: "oauth"})}, false}, + {"oauth2 with all three settings", &Config{Auth: authFor(Auth{Scheme: AuthSchemeOAuth2, + TokenURL: "https://issuer.invalid/token", ClientIDEnv: "ID", ClientSecretEnv: "SECRET"})}, true}, + {"oauth2 missing the token url", &Config{Auth: authFor(Auth{Scheme: AuthSchemeOAuth2, + ClientIDEnv: "ID", ClientSecretEnv: "SECRET"})}, false}, + {"oauth2 missing the client id variable", &Config{Auth: authFor(Auth{Scheme: AuthSchemeOAuth2, + TokenURL: "https://issuer.invalid/token", ClientSecretEnv: "SECRET"})}, false}, + {"oauth2 missing the client secret variable", &Config{Auth: authFor(Auth{Scheme: AuthSchemeOAuth2, + TokenURL: "https://issuer.invalid/token", ClientIDEnv: "ID"})}, false}, + {"a profile for a provider not in bindingKeys", &Config{ + Auth: map[string]*Auth{"someone-else": {Scheme: AuthSchemeNone}}}, false}, } for _, tc := range testCases { @@ -293,9 +337,7 @@ func TestRunDoesNotRetryAMissingCredential(t *testing.T) { } mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} step := newStep(t, &stubRegistry{plan: plan}, mapper, func(c *Config) { - c.AuthScheme = AuthSchemeBasic - c.UsernameEnv = "TEST_ABSENT_USER_FOR_RETRY" - c.PasswordEnv = "TEST_ABSENT_PASS_FOR_RETRY" + c.setAuth(Auth{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_ABSENT_USER_FOR_RETRY", PasswordEnv: "TEST_ABSENT_PASS_FOR_RETRY"}) }) _, err := runStep(t, step, selectBody) @@ -511,9 +553,7 @@ func TestRunRedactsACredentialEchoedInABody(t *testing.T) { mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper, func(c *Config) { - c.AuthScheme = AuthSchemeQuery - c.QueryName = "token" - c.QueryValueEnv = "TEST_ECHO_TOKEN" + c.setAuth(Auth{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_ECHO_TOKEN"}) }) _, err := runStep(t, step, selectBody) @@ -557,9 +597,7 @@ func TestRunSendsTheCredentialAsAQueryParameter(t *testing.T) { mapper := &stubMapper{requestResult: []byte(`{"statecode":"CG"}`), responseResult: []byte(`{}`)} step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper, func(c *Config) { - c.AuthScheme = AuthSchemeQuery - c.QueryName = "token" - c.QueryValueEnv = "TEST_MANDI_TOKEN" + c.setAuth(Auth{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) }) if _, err := runStep(t, step, selectBody); err != nil { @@ -589,9 +627,7 @@ func TestRunRedactsAQueryCredentialFromAnError(t *testing.T) { } mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} step := newStep(t, &stubRegistry{plan: plan}, mapper, func(c *Config) { - c.AuthScheme = AuthSchemeQuery - c.QueryName = "token" - c.QueryValueEnv = "TEST_MANDI_TOKEN" + c.setAuth(Auth{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) }) _, err := runStep(t, step, selectBody) @@ -613,11 +649,7 @@ func TestRedactStringRemovesTheCredentialFromTheURL(t *testing.T) { // No t.Parallel: t.Setenv forbids it. t.Setenv("TEST_MANDI_TOKEN", "s3cr3t") - step := &Step{config: &Config{ - AuthScheme: AuthSchemeQuery, - QueryName: "token", - QueryValueEnv: "TEST_MANDI_TOKEN", - }} + step := stepWithAuth(Auth{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) got := step.redactString("http://host/v1/x?statecode=CG&token=s3cr3t") if strings.Contains(got, "s3cr3t") { t.Errorf("the credential survived redaction: %s", got) @@ -627,7 +659,7 @@ func TestRedactStringRemovesTheCredentialFromTheURL(t *testing.T) { } // Any other scheme has nothing to hide in a URL, so the text is untouched. - plain := &Step{config: &Config{AuthScheme: AuthSchemeNone}} + plain := stepWithAuth(Auth{Scheme: AuthSchemeNone}) if out := plain.redactString("http://host/v1/x?statecode=CG"); out != "http://host/v1/x?statecode=CG" { t.Errorf("a url with no credential must pass through unchanged, got %q", out) } @@ -643,11 +675,7 @@ func TestRedactStringRemovesThePercentEncodedCredential(t *testing.T) { const token = "a+b/c=d e" // every character Encode treats specially t.Setenv("TEST_MANDI_TOKEN", token) - step := &Step{config: &Config{ - AuthScheme: AuthSchemeQuery, - QueryName: "token", - QueryValueEnv: "TEST_MANDI_TOKEN", - }} + step := stepWithAuth(Auth{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) // Exactly how the credential appears once authenticate has run: Encode // escapes it, so this is the string a transport error quotes. @@ -673,11 +701,7 @@ func TestRedactStringStillRemovesAnUnescapedCredential(t *testing.T) { // No t.Parallel: t.Setenv forbids it. t.Setenv("TEST_MANDI_TOKEN", "plaintoken123") - step := &Step{config: &Config{ - AuthScheme: AuthSchemeQuery, - QueryName: "token", - QueryValueEnv: "TEST_MANDI_TOKEN", - }} + step := stepWithAuth(Auth{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) got := step.redactString("http://host/v1/x?token=plaintoken123") if strings.Contains(got, "plaintoken123") { t.Errorf("the credential survived redaction: %s", got) @@ -694,9 +718,9 @@ func TestNewRefusesAHalfConfiguredQueryScheme(t *testing.T) { cfg *Config }{ {"no queryName", &Config{BindingKeys: []string{testBindingKey}, - AuthScheme: AuthSchemeQuery, QueryValueEnv: "TEST_MANDI_TOKEN"}}, + Auth: authFor(Auth{Scheme: AuthSchemeQuery, QueryValueEnv: "TEST_MANDI_TOKEN"})}}, {"no queryValueEnv", &Config{BindingKeys: []string{testBindingKey}, - AuthScheme: AuthSchemeQuery, QueryName: "token"}}, + Auth: authFor(Auth{Scheme: AuthSchemeQuery, QueryName: "token"})}}, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -1135,6 +1159,7 @@ func TestNewRefusesAHalfConfiguredOverride(t *testing.T) { _, _, err := New(context.Background(), &stubRegistry{}, &stubMapper{}, nil, &Config{ BindingKeys: []string{testBindingKey}, ProviderIDAt: "who.provider", + Auth: authFor(Auth{Scheme: AuthSchemeNone}), }) if err == nil { t.Fatal("expected one path without the other to be refused") @@ -1383,9 +1408,7 @@ func TestRunPresentsBasicCredentialsFromTheEnvironment(t *testing.T) { step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, func(c *Config) { - c.AuthScheme = AuthSchemeBasic - c.UsernameEnv = "TEST_MAUSAMGRAM_USER" - c.PasswordEnv = "TEST_MAUSAMGRAM_KEY" + c.setAuth(Auth{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_MAUSAMGRAM_USER", PasswordEnv: "TEST_MAUSAMGRAM_KEY"}) }) if _, err := runStep(t, step, selectBody); err != nil { @@ -1409,9 +1432,7 @@ func TestRunPresentsAHeaderCredentialFromTheEnvironment(t *testing.T) { step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, func(c *Config) { - c.AuthScheme = AuthSchemeHeader - c.HeaderName = "X-Api-Key" - c.HeaderValueEnv = "TEST_MAUSAMGRAM_TOKEN" + c.setAuth(Auth{Scheme: AuthSchemeHeader, HeaderName: "X-Api-Key", HeaderValueEnv: "TEST_MAUSAMGRAM_TOKEN"}) }) if _, err := runStep(t, step, selectBody); err != nil { @@ -1433,9 +1454,7 @@ func TestRunFailsWhenAConfiguredCredentialIsAbsent(t *testing.T) { step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, func(c *Config) { - c.AuthScheme = AuthSchemeBasic - c.UsernameEnv = "TEST_MAUSAMGRAM_ABSENT_USER" - c.PasswordEnv = "TEST_MAUSAMGRAM_ABSENT_KEY" + c.setAuth(Auth{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_MAUSAMGRAM_ABSENT_USER", PasswordEnv: "TEST_MAUSAMGRAM_ABSENT_KEY"}) }) if _, err := runStep(t, step, selectBody); err == nil { @@ -1739,11 +1758,7 @@ func TestRedactKeepsTheErrorChainMatchable(t *testing.T) { // No t.Parallel: t.Setenv forbids it. t.Setenv("TEST_CHAIN_TOKEN", "s3cr3t") - step := &Step{config: &Config{ - AuthScheme: AuthSchemeQuery, - QueryName: "token", - QueryValueEnv: "TEST_CHAIN_TOKEN", - }} + step := stepWithAuth(Auth{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_CHAIN_TOKEN"}) // The shape net/http produces: the cause wrapped behind text that quotes // the whole URL, credential and all. @@ -1777,11 +1792,7 @@ func TestRedactLeavesAnUnchangedErrorAlone(t *testing.T) { // No t.Parallel: t.Setenv forbids it. t.Setenv("TEST_CHAIN_TOKEN_2", "s3cr3t") - step := &Step{config: &Config{ - AuthScheme: AuthSchemeQuery, - QueryName: "token", - QueryValueEnv: "TEST_CHAIN_TOKEN_2", - }} + step := stepWithAuth(Auth{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_CHAIN_TOKEN_2"}) original := errors.New("nothing sensitive here") if got := step.redact(original); got != original { t.Errorf("redact returned a different error for text it did not change: %v", got) @@ -1956,8 +1967,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { { name: "basic, the wire form a gateway echoes", tweak: func(c *Config) { - c.AuthScheme = AuthSchemeBasic - c.UsernameEnv, c.PasswordEnv = "TEST_USER", "TEST_PASS" + c.setAuth(Auth{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_USER", PasswordEnv: "TEST_PASS"}) }, body: `{"error":"invalid Authorization: Basic ` + wire + `"}`, secret: wire, @@ -1965,8 +1975,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { { name: "basic, the password quoted raw", tweak: func(c *Config) { - c.AuthScheme = AuthSchemeBasic - c.UsernameEnv, c.PasswordEnv = "TEST_USER", "TEST_PASS" + c.setAuth(Auth{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_USER", PasswordEnv: "TEST_PASS"}) }, body: `{"error":"bad password s3cr3t"}`, secret: "s3cr3t", @@ -1974,8 +1983,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { { name: "header, the value as sent", tweak: func(c *Config) { - c.AuthScheme = AuthSchemeHeader - c.HeaderName, c.HeaderValueEnv = "X-API-Key", "TEST_HDR" + c.setAuth(Auth{Scheme: AuthSchemeHeader, HeaderName: "X-API-Key", HeaderValueEnv: "TEST_HDR"}) }, body: `{"error":"bad X-API-Key: hdr-k3y"}`, secret: "hdr-k3y", @@ -1983,8 +1991,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { { name: "query, still covered, raw form", tweak: func(c *Config) { - c.AuthScheme = AuthSchemeQuery - c.QueryName, c.QueryValueEnv = "token", "TEST_QRY" + c.setAuth(Auth{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_QRY"}) }, body: `{"rejected":"token=a+b/c="}`, secret: "a+b/c=", @@ -1992,8 +1999,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { { name: "query, still covered, percent-encoded form", tweak: func(c *Config) { - c.AuthScheme = AuthSchemeQuery - c.QueryName, c.QueryValueEnv = "token", "TEST_QRY" + c.setAuth(Auth{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_QRY"}) }, body: `{"rejected":"token=` + url.QueryEscape("a+b/c=") + `"}`, secret: url.QueryEscape("a+b/c="), @@ -2027,8 +2033,7 @@ func TestRedactStringLeavesTheBasicUsername(t *testing.T) { step := newStep(t, &stubRegistry{plan: testPlan("http://provider.invalid", http.MethodGet)}, &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, func(c *Config) { - c.AuthScheme = AuthSchemeBasic - c.UsernameEnv, c.PasswordEnv = "TEST_USER", "TEST_PASS" + c.setAuth(Auth{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_USER", PasswordEnv: "TEST_PASS"}) }) got := step.redactString(`user mausam failed to authenticate with s3cr3t`) @@ -2049,11 +2054,16 @@ func TestRedactStringWithNoCredentialConfigured(t *testing.T) { step := newStep(t, &stubRegistry{plan: testPlan("http://provider.invalid", http.MethodGet)}, &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, func(c *Config) { - c.AuthScheme = scheme // Env vars named but deliberately unset. - c.UsernameEnv, c.PasswordEnv = "TEST_UNSET_U", "TEST_UNSET_P" - c.HeaderName, c.HeaderValueEnv = "X-K", "TEST_UNSET_H" - c.QueryName, c.QueryValueEnv = "t", "TEST_UNSET_Q" + c.setAuth(Auth{ + Scheme: scheme, + UsernameEnv: "TEST_UNSET_U", + PasswordEnv: "TEST_UNSET_P", + HeaderName: "X-K", + HeaderValueEnv: "TEST_UNSET_H", + QueryName: "t", + QueryValueEnv: "TEST_UNSET_Q", + }) }) if got := step.redactString(text); got != text { @@ -2085,7 +2095,8 @@ func TestRunHandsResolvedPrerequisitesToTheMappingAsLocal(t *testing.T) { mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{"answered":true}`)} step, closer, err := New(context.Background(), &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper, prerequisites, - &Config{BindingKeys: []string{testBindingKey}}) + &Config{BindingKeys: []string{testBindingKey}, + Auth: authFor(Auth{Scheme: AuthSchemeNone})}) if err != nil { t.Fatalf("New() returned an unexpected error: %v", err) } @@ -2252,9 +2263,7 @@ func oauth2Step(t *testing.T, tokenURL string, seen *[]string) *Step { return newStep(t, &stubRegistry{plan: testPlan(provider.URL, http.MethodGet)}, &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{"a":1}`)}, func(c *Config) { - c.AuthScheme = AuthSchemeOAuth2 - c.TokenURL = tokenURL - c.ClientIDEnv, c.ClientSecretEnv = "TEST_OAUTH_ID", "TEST_OAUTH_SECRET" + c.setAuth(Auth{Scheme: AuthSchemeOAuth2, TokenURL: tokenURL, ClientIDEnv: "TEST_OAUTH_ID", ClientSecretEnv: "TEST_OAUTH_SECRET"}) }) } From 12d1e54a0f0c247df92d2e601e6fa09cba997f29 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 16:53:14 +0530 Subject: [PATCH 104/130] fix(plugin): unmarshal settings with the yaml version the adapter uses [#7] The nested config parsed in tests and was refused by the running adapter: fatal: could not decode config: yaml: unmarshal errors: line 142: cannot unmarshal !!map into string cmd/adapter/main.go decodes with gopkg.in/yaml.v2; Settings.UnmarshalYAML was written against v3. The two declare incompatible interfaces -- UnmarshalYAML(*yaml.Node) versus UnmarshalYAML(func(any) error) -- and the mismatch is SILENT. Go does not complain, yaml.v2 simply never calls the method, and every nested block reaches the plain map. Both versions are in go.mod, so nothing flagged it. The tests missed it because they imported v3 and called it directly, so nine green tests exercised a path the adapter never takes. They now use v2, with a comment saying why, and a compile-time assertion makes a future mismatch a build failure rather than a startup one: var _ yaml.Unmarshaler = (*Settings)(nil) Also adds asBlock. v2 decodes a nested mapping as map[interface{}]interface{} while the top level arrives as map[string]any, so the v3 type assertion would have failed on every nested block even with the right signature. Block names are rendered through the same helper as values, so a non-string key cannot quietly become a different setting. Verified against the running stack rather than in tests alone. All three capabilities answer through one adapter, each on its own scheme: KnowledgeAdvisory oauth2 200, 12 resources, real external provider MandiPrice query 200, 2 resources, token=REDACTED in the log WeatherObservation none 200, 3 resources Zero occurrences of the client secret or the mandi token anywhere in the logs. --- pkg/plugin/config.go | 39 ++++++++++++++++--- pkg/plugin/config_test.go | 8 +++- .../cmd/shippedconfig_test.go | 6 ++- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/pkg/plugin/config.go b/pkg/plugin/config.go index 9a594a5d..17972f6e 100644 --- a/pkg/plugin/config.go +++ b/pkg/plugin/config.go @@ -5,7 +5,7 @@ import ( "sort" "strings" - "gopkg.in/yaml.v3" + yaml "gopkg.in/yaml.v2" ) // Settings is a plugin's own configuration block. @@ -34,10 +34,18 @@ import ( // already defined. type Settings map[string]string +// Settings must satisfy the unmarshaler of the SAME yaml package the adapter +// decodes its config with -- gopkg.in/yaml.v2, in cmd/adapter/main.go. The two +// versions declare incompatible interfaces, and a mismatch is silent: the +// method is simply never called and a nested block reaches the plain map, +// failing with "cannot unmarshal !!map into string" at startup. This assertion +// turns that into a compile error instead. +var _ yaml.Unmarshaler = (*Settings)(nil) + // UnmarshalYAML accepts scalars at the top level and one level of nesting. -func (s *Settings) UnmarshalYAML(node *yaml.Node) error { +func (s *Settings) UnmarshalYAML(unmarshal func(any) error) error { var raw map[string]any - if err := node.Decode(&raw); err != nil { + if err := unmarshal(&raw); err != nil { return err } @@ -53,7 +61,7 @@ func (s *Settings) UnmarshalYAML(node *yaml.Node) error { for _, key := range blocks { value := raw[key] - block, nested := value.(map[string]any) + block, nested := asBlock(value) if !nested { out[key] = scalar(value) continue @@ -74,7 +82,7 @@ func (s *Settings) UnmarshalYAML(node *yaml.Node) error { sort.Strings(fields) for _, field := range fields { - if _, deeper := block[field].(map[string]any); deeper { + if _, deeper := asBlock(block[field]); deeper { return fmt.Errorf( "plugin config: %s.%s nests further; only one level is supported", key, field) } @@ -97,6 +105,27 @@ func (s *Settings) UnmarshalYAML(node *yaml.Node) error { return nil } +// asBlock recognises a nested mapping. +// +// Two shapes because yaml.v2 decodes a mapping into map[interface{}]interface{} +// unless the target says otherwise, so the top level arrives with string keys +// and anything under it does not. Normalised here rather than at each use, and +// the keys are rendered the same way values are so a non-string key cannot +// silently become a different setting. +func asBlock(v any) (map[string]any, bool) { + switch typed := v.(type) { + case map[string]any: + return typed, true + case map[any]any: + out := make(map[string]any, len(typed)) + for key, value := range typed { + out[scalar(key)] = value + } + return out, true + } + return nil, false +} + // scalar renders a YAML scalar the way the flat form would have carried it. // %v on an int gives "1048576" rather than an exponent, which a float would; // YAML decodes an integer literal as int, so a byte count survives intact. diff --git a/pkg/plugin/config_test.go b/pkg/plugin/config_test.go index 6dc2ac1d..66ce2f6a 100644 --- a/pkg/plugin/config_test.go +++ b/pkg/plugin/config_test.go @@ -4,10 +4,16 @@ import ( "strings" "testing" - "gopkg.in/yaml.v3" + yaml "gopkg.in/yaml.v2" ) // decode is the path a real config takes: YAML -> Config.Config. +// +// yaml.v2 ON PURPOSE. cmd/adapter/main.go decodes the adapter config with v2, +// and the two versions declare incompatible unmarshaler interfaces -- so a +// test written against v3 passes while the adapter refuses the same file at +// startup. That happened; these tests are the reason it cannot happen quietly +// again. func decode(t *testing.T, body string) (Settings, error) { t.Helper() var c Config diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/cmd/shippedconfig_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/shippedconfig_test.go index f530a84e..df4ef08f 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/cmd/shippedconfig_test.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/shippedconfig_test.go @@ -4,7 +4,7 @@ import ( "os" "testing" - "gopkg.in/yaml.v3" + yaml "gopkg.in/yaml.v2" "github.com/beckn-one/beckn-onix/pkg/plugin" ) @@ -12,6 +12,10 @@ import ( // The shipped config and this code move together: the nested per-provider // block is only valid if pkg/plugin flattens it and ParseAuth reads it back. // This reads the real file rather than a copy, so the two cannot drift. +// +// Decoded with yaml.v2, the version cmd/adapter/main.go uses. A test on v3 +// passes while the adapter refuses the same file, because the two declare +// incompatible unmarshaler interfaces. func TestShippedConfigParsesIntoAProfilePerProvider(t *testing.T) { t.Parallel() From 16f4f48ed514c1bc5772044a0a0b0f9f96ecadf9 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 17:09:55 +0530 Subject: [PATCH 105/130] refactor: name the config block and credential profiles for what they are [#7] Renames only -- no behaviour change, no config change. The previous names read as jargon at the call site, particularly in tests where `Settings{...}` gave a reader nothing to go on. Settings -> ConfigBlock the `config:` block of a plugin entry scalar -> asString renders a YAML scalar as a string asBlock -> asNestedBlock recognises a nested mapping Auth -> AuthProfile ONE provider's credentials Config.Auth -> Config.AuthByProvider keyed by participant id ParseAuth -> ParseProviderAuth participantOf -> providerIDFrom authFor -> authForTestProvider setAuth -> setProviderAuth stepWithAuth -> stepWithProviderAuth AuthByProvider is the one worth the extra length: `cfg.Auth` looked like the step's own credential, which is exactly the thing this change stopped it being. The unmarshaler's locals follow: raw -> written (what the operator wrote), out -> flattened (what a plugin receives), blocks -> names, flat -> flatName, and the receiver s -> block. Scoped deliberately to the nineteen files this work owns. A repo-wide sweep of the same identifiers reached pkg/telemetry and pkg/model, which have their own Settings, and rewrote resp.Auth on Vault's api.Secret -- so those were reverted and the rename redone file by file. pkg/telemetry, pkg/model, core/module/handler/healthcheck.go and the keymanager are verified untouched. 66 packages green, vet clean. --- pkg/plugin/config.go | 101 +++++++++-------- pkg/plugin/config_test.go | 10 +- .../KnowledgeAdvisory/cmd/plugin.go | 4 +- .../KnowledgeAdvisory/cmd/plugin_test.go | 14 +-- .../cmd/shippedconfig_test.go | 4 +- .../KnowledgeAdvisory/live_test.go | 2 +- .../KnowledgeAdvisory/mappings_test.go | 2 +- .../implementation/MandiPrice/cmd/plugin.go | 4 +- .../MandiPrice/cmd/plugin_test.go | 8 +- .../MandiPrice/mappings_test.go | 6 +- .../WeatherObservation/cmd/plugin.go | 4 +- .../WeatherObservation/cmd/plugin_test.go | 4 +- .../WeatherObservation/mappings_test.go | 2 +- .../internal/upstream/dispatch_test.go | 2 +- .../internal/upstream/perprovider_test.go | 8 +- .../internal/upstream/upstream.go | 45 ++++---- .../internal/upstream/upstream_test.go | 106 +++++++++--------- .../otelsetup/otelsetup_test.go | 8 +- pkg/plugin/manager_test.go | 2 +- 19 files changed, 169 insertions(+), 167 deletions(-) diff --git a/pkg/plugin/config.go b/pkg/plugin/config.go index 17972f6e..607d81b3 100644 --- a/pkg/plugin/config.go +++ b/pkg/plugin/config.go @@ -8,7 +8,7 @@ import ( yaml "gopkg.in/yaml.v2" ) -// Settings is a plugin's own configuration block. +// ConfigBlock is a plugin's own configuration block. // // It is map[string]string underneath, deliberately: every plugin's New takes a // map[string]string, and a named map type stays assignable to one, so widening @@ -32,94 +32,93 @@ import ( // plugin's settings in one place and make this file a dependency of all of // them. Rejecting an unknown field is the plugin's job, where the field is // already defined. -type Settings map[string]string +type ConfigBlock map[string]string -// Settings must satisfy the unmarshaler of the SAME yaml package the adapter +// ConfigBlock must satisfy the unmarshaler of the SAME yaml package the adapter // decodes its config with -- gopkg.in/yaml.v2, in cmd/adapter/main.go. The two // versions declare incompatible interfaces, and a mismatch is silent: the // method is simply never called and a nested block reaches the plain map, // failing with "cannot unmarshal !!map into string" at startup. This assertion // turns that into a compile error instead. -var _ yaml.Unmarshaler = (*Settings)(nil) +var _ yaml.Unmarshaler = (*ConfigBlock)(nil) // UnmarshalYAML accepts scalars at the top level and one level of nesting. -func (s *Settings) UnmarshalYAML(unmarshal func(any) error) error { - var raw map[string]any - if err := unmarshal(&raw); err != nil { +func (block *ConfigBlock) UnmarshalYAML(unmarshal func(any) error) error { + var written map[string]any + if err := unmarshal(&written); err != nil { return err } - out := Settings{} - // Sorted so that a duplicate is reported against the same pair whichever - // order the map happened to iterate in. - blocks := make([]string, 0, len(raw)) - for key := range raw { - blocks = append(blocks, key) - } - sort.Strings(blocks) + flattened := ConfigBlock{} - for _, key := range blocks { - value := raw[key] + // Sorted so a config with two mistakes reports the same one every run. + names := make([]string, 0, len(written)) + for name := range written { + names = append(names, name) + } + sort.Strings(names) - block, nested := asBlock(value) - if !nested { - out[key] = scalar(value) + for _, name := range names { + nested, isNested := asNestedBlock(written[name]) + if !isNested { + flattened[name] = asString(written[name]) continue } - - // A block key carrying a dash would produce a flattened key that reads - // as another field's, so the split on the first dash could no longer - // tell them apart. Participant ids DO carry dashes, which is why the - // dash goes between field and key rather than inside either. - if strings.TrimSpace(key) == "" { - return fmt.Errorf("plugin config: a settings block has an empty name") + if strings.TrimSpace(name) == "" { + return fmt.Errorf("plugin config: a nested block has no name") } - fields := make([]string, 0, len(block)) - for field := range block { - fields = append(fields, field) + settings := make([]string, 0, len(nested)) + for setting := range nested { + settings = append(settings, setting) } - sort.Strings(fields) + sort.Strings(settings) - for _, field := range fields { - if _, deeper := asBlock(block[field]); deeper { + for _, setting := range settings { + if _, deeper := asNestedBlock(nested[setting]); deeper { return fmt.Errorf( - "plugin config: %s.%s nests further; only one level is supported", key, field) + "plugin config: %s.%s nests further; only one level is supported", + name, setting) } - if strings.Contains(field, "-") { + // A setting carrying a dash would flatten to -- + // and resolve to a block that does not exist. The block name + // already says what it belongs to, so inside one a setting is + // named plainly. + if strings.Contains(setting, "-") { return fmt.Errorf( "plugin config: %s.%s carries a dash; inside a block a setting is named "+ - "plainly, and the block already says which one it belongs to", key, field) + "plainly, and the block already says which one it belongs to", + name, setting) } - flat := field + "-" + key - if _, taken := out[flat]; taken { + flatName := setting + "-" + name + if _, alreadySet := flattened[flatName]; alreadySet { return fmt.Errorf( "plugin config: %s is set twice, once as %s.%s and once directly", - flat, key, field) + flatName, name, setting) } - out[flat] = scalar(block[field]) + flattened[flatName] = asString(nested[setting]) } } - *s = out + *block = flattened return nil } -// asBlock recognises a nested mapping. +// asNestedBlock recognises a nested mapping and normalises its keys. // // Two shapes because yaml.v2 decodes a mapping into map[interface{}]interface{} // unless the target says otherwise, so the top level arrives with string keys // and anything under it does not. Normalised here rather than at each use, and // the keys are rendered the same way values are so a non-string key cannot // silently become a different setting. -func asBlock(v any) (map[string]any, bool) { +func asNestedBlock(v any) (map[string]any, bool) { switch typed := v.(type) { case map[string]any: return typed, true case map[any]any: out := make(map[string]any, len(typed)) for key, value := range typed { - out[scalar(key)] = value + out[asString(key)] = value } return out, true } @@ -129,7 +128,7 @@ func asBlock(v any) (map[string]any, bool) { // scalar renders a YAML scalar the way the flat form would have carried it. // %v on an int gives "1048576" rather than an exponent, which a float would; // YAML decodes an integer literal as int, so a byte count survives intact. -func scalar(v any) string { +func asString(v any) string { if v == nil { return "" } @@ -137,18 +136,18 @@ func scalar(v any) string { } type PublisherCfg struct { - ID string `yaml:"id"` - Config Settings `yaml:"config"` + ID string `yaml:"id"` + Config ConfigBlock `yaml:"config"` } type ValidatorCfg struct { - ID string `yaml:"id"` - Config Settings `yaml:"config"` + ID string `yaml:"id"` + Config ConfigBlock `yaml:"config"` } type Config struct { - ID string `yaml:"id"` - Config Settings `yaml:"config"` + ID string `yaml:"id"` + Config ConfigBlock `yaml:"config"` } type ManagerConfig struct { diff --git a/pkg/plugin/config_test.go b/pkg/plugin/config_test.go index 66ce2f6a..623aa78b 100644 --- a/pkg/plugin/config_test.go +++ b/pkg/plugin/config_test.go @@ -14,7 +14,7 @@ import ( // test written against v3 passes while the adapter refuses the same file at // startup. That happened; these tests are the reason it cannot happen quietly // again. -func decode(t *testing.T, body string) (Settings, error) { +func decode(t *testing.T, body string) (ConfigBlock, error) { t.Helper() var c Config err := yaml.Unmarshal([]byte("id: X\nconfig:\n"+body), &c) @@ -33,7 +33,7 @@ func TestSettingsKeepsTheFlatFormUntouched(t *testing.T) { if err != nil { t.Fatalf("decode: %v", err) } - want := Settings{ + want := ConfigBlock{ "bindingKeys": "a|cap,b|cap", "authScheme": "oauth2", "clientIdEnv": "KNOWLEDGE_CLIENT_ID", @@ -175,7 +175,7 @@ func TestSettingsAcceptsNoConfigAtAll(t *testing.T) { } } -// Settings must stay usable exactly where map[string]string was, because every +// ConfigBlock must stay usable exactly where map[string]string was, because every // plugin's New takes one and none of them changed. func TestSettingsPassesWhereAPlainMapIsWanted(t *testing.T) { got, err := decode(t, " authScheme: oauth2\n") @@ -184,10 +184,10 @@ func TestSettingsPassesWhereAPlainMapIsWanted(t *testing.T) { } takesPlainMap := func(m map[string]string) string { return m["authScheme"] } if takesPlainMap(got) != "oauth2" { - t.Error("Settings no longer passes as a map[string]string") + t.Error("ConfigBlock no longer passes as a map[string]string") } var assigned map[string]string = got if assigned["authScheme"] != "oauth2" { - t.Error("Settings no longer assigns to a map[string]string") + t.Error("ConfigBlock no longer assigns to a map[string]string") } } diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin.go b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin.go index b6ee51f4..104a6066 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin.go @@ -52,11 +52,11 @@ func (p knowledgeAdvisoryProvider) parseConfig(config map[string]string) (*Knowl // authScheme- settings. Shared with the other capability // plugins: each used to copy the same field list, so a scheme added in one // had to be remembered in three. - auth, err := upstream.ParseAuth(config) + auth, err := upstream.ParseProviderAuth(config) if err != nil { return nil, err } - cfg.Auth = auth + cfg.AuthByProvider = auth return cfg, nil } diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin_test.go index 438729b6..2f65319b 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin_test.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin_test.go @@ -39,10 +39,10 @@ func TestParseConfig(t *testing.T) { { // Everything absent is left zero: KnowledgeAdvisory.New defaults // it, so the rules are defined in exactly one place. Auth is an - // empty map rather than nil -- ParseAuth always returns one. + // empty map rather than nil -- ParseProviderAuth always returns one. name: "leaves everything unset for New to default", config: map[string]string{}, - expected: &KnowledgeAdvisory.Config{Auth: map[string]*upstream.Auth{}}, + expected: &KnowledgeAdvisory.Config{AuthByProvider: map[string]*upstream.AuthProfile{}}, }, { // The scheme this capability exists to use. Its provider sits @@ -58,7 +58,7 @@ func TestParseConfig(t *testing.T) { }, expected: &KnowledgeAdvisory.Config{ BindingKeys: []string{"knowledge-provider|openagrinet:KnowledgeAdvisory"}, - Auth: map[string]*upstream.Auth{ + AuthByProvider: map[string]*upstream.AuthProfile{ "knowledge-provider": { Provider: "knowledge-provider", Scheme: "oauth2", @@ -90,7 +90,7 @@ func TestParseConfig(t *testing.T) { "knowledge-provider|openagrinet:KnowledgeAdvisory", "vistaar-two|openagrinet:KnowledgeAdvisory", }, - Auth: map[string]*upstream.Auth{ + AuthByProvider: map[string]*upstream.AuthProfile{ "knowledge-provider": { Provider: "knowledge-provider", Scheme: "oauth2", @@ -119,7 +119,7 @@ func TestParseConfig(t *testing.T) { }, expected: &KnowledgeAdvisory.Config{ BindingKeys: []string{"provider.oan.dev|openagrinet:KnowledgeAdvisory"}, - Auth: map[string]*upstream.Auth{ + AuthByProvider: map[string]*upstream.AuthProfile{ "provider.oan.dev": { Provider: "provider.oan.dev", Scheme: "basic", @@ -174,7 +174,7 @@ func TestParseConfig(t *testing.T) { }, expected: &KnowledgeAdvisory.Config{ BindingKeys: []string{"other|capability"}, - Auth: map[string]*upstream.Auth{ + AuthByProvider: map[string]*upstream.AuthProfile{ "other": { Provider: "other", Scheme: "basic", @@ -207,7 +207,7 @@ func TestParseConfig(t *testing.T) { // unset variable produces this, and it should read as "unset". name: "treats an empty response cap as unset", config: map[string]string{"maxResponseBytes": ""}, - expected: &KnowledgeAdvisory.Config{Auth: map[string]*upstream.Auth{}}, + expected: &KnowledgeAdvisory.Config{AuthByProvider: map[string]*upstream.AuthProfile{}}, }, } diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/cmd/shippedconfig_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/shippedconfig_test.go index df4ef08f..958ecccb 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/cmd/shippedconfig_test.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/shippedconfig_test.go @@ -10,7 +10,7 @@ import ( ) // The shipped config and this code move together: the nested per-provider -// block is only valid if pkg/plugin flattens it and ParseAuth reads it back. +// block is only valid if pkg/plugin flattens it and ParseProviderAuth reads it back. // This reads the real file rather than a copy, so the two cannot drift. // // Decoded with yaml.v2, the version cmd/adapter/main.go uses. A test on v3 @@ -68,7 +68,7 @@ func TestShippedConfigParsesIntoAProfilePerProvider(t *testing.T) { break } } - profile, ok := cfg.Auth[provider] + profile, ok := cfg.AuthByProvider[provider] if !ok { t.Errorf("%s serves %q with no auth block", step.ID, provider) continue diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/live_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/live_test.go index 89410ac3..e9f18d49 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/live_test.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/live_test.go @@ -103,7 +103,7 @@ func TestLiveEndToEnd(t *testing.T) { step, closeStep, err := KnowledgeAdvisory.New(context.Background(), registry, mapper, &KnowledgeAdvisory.Config{ BindingKeys: []string{key}, - Auth: map[string]*upstreamstep.Auth{ + AuthByProvider: map[string]*upstreamstep.AuthProfile{ strings.Split(key, "|")[0]: { Scheme: "oauth2", TokenURL: issuer.URL + "/token", diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go index d621a166..3be5aede 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go @@ -212,7 +212,7 @@ func runShippedWith(t *testing.T, request, providerBody string) (map[string]any, BindingKeys: []string{shippedBindingKey}, // Auth is per provider; these upstreams are stubs needing // no credential, and that is declared rather than defaulted. - Auth: map[string]*upstreamstep.Auth{ + AuthByProvider: map[string]*upstreamstep.AuthProfile{ strings.Split(shippedBindingKey, "|")[0]: {Scheme: upstreamstep.AuthSchemeNone}, }, }) diff --git a/pkg/plugin/implementation/MandiPrice/cmd/plugin.go b/pkg/plugin/implementation/MandiPrice/cmd/plugin.go index b033d17a..e3aee941 100644 --- a/pkg/plugin/implementation/MandiPrice/cmd/plugin.go +++ b/pkg/plugin/implementation/MandiPrice/cmd/plugin.go @@ -50,11 +50,11 @@ func (p mandiProvider) parseConfig(config map[string]string) (*MandiPrice.Config // authScheme- settings. Shared with the other capability // plugins: each used to copy the same field list, so a scheme added in one // had to be remembered in three. - auth, err := upstream.ParseAuth(config) + auth, err := upstream.ParseProviderAuth(config) if err != nil { return nil, err } - cfg.Auth = auth + cfg.AuthByProvider = auth return cfg, nil } diff --git a/pkg/plugin/implementation/MandiPrice/cmd/plugin_test.go b/pkg/plugin/implementation/MandiPrice/cmd/plugin_test.go index 7592a0b7..3b5509a7 100644 --- a/pkg/plugin/implementation/MandiPrice/cmd/plugin_test.go +++ b/pkg/plugin/implementation/MandiPrice/cmd/plugin_test.go @@ -41,7 +41,7 @@ func TestParseConfig(t *testing.T) { // rules are defined in exactly one place. name: "leaves everything unset for New to default", config: map[string]string{}, - expected: &MandiPrice.Config{Auth: map[string]*upstream.Auth{}}, + expected: &MandiPrice.Config{AuthByProvider: map[string]*upstream.AuthProfile{}}, }, { // Query auth is why this capability has its own entry rather than @@ -57,7 +57,7 @@ func TestParseConfig(t *testing.T) { }, expected: &MandiPrice.Config{ BindingKeys: []string{"agmarknet|openagrinet:MandiPrice"}, - Auth: map[string]*upstream.Auth{ + AuthByProvider: map[string]*upstream.AuthProfile{ "agmarknet": { Provider: "agmarknet", Scheme: "query", @@ -82,7 +82,7 @@ func TestParseConfig(t *testing.T) { }, expected: &MandiPrice.Config{ BindingKeys: []string{"other|capability"}, - Auth: map[string]*upstream.Auth{ + AuthByProvider: map[string]*upstream.AuthProfile{ "other": { Provider: "other", Scheme: "basic", @@ -113,7 +113,7 @@ func TestParseConfig(t *testing.T) { // as "unset" rather than failing startup. name: "treats an empty response cap as unset", config: map[string]string{"maxResponseBytes": ""}, - expected: &MandiPrice.Config{Auth: map[string]*upstream.Auth{}}, + expected: &MandiPrice.Config{AuthByProvider: map[string]*upstream.AuthProfile{}}, }, } diff --git a/pkg/plugin/implementation/MandiPrice/mappings_test.go b/pkg/plugin/implementation/MandiPrice/mappings_test.go index d7ad6557..20307294 100644 --- a/pkg/plugin/implementation/MandiPrice/mappings_test.go +++ b/pkg/plugin/implementation/MandiPrice/mappings_test.go @@ -197,7 +197,7 @@ func runShippedWith(t *testing.T, request, providerBody string) (url.Values, map BindingKeys: []string{shippedBindingKey}, // Auth is per provider; these upstreams are stubs needing // no credential, and that is declared rather than defaulted. - Auth: map[string]*upstreamstep.Auth{ + AuthByProvider: map[string]*upstreamstep.AuthProfile{ strings.Split(shippedBindingKey, "|")[0]: {Scheme: upstreamstep.AuthSchemeNone}, }, }) @@ -444,7 +444,7 @@ func TestShippedMappingRefusesWhatItCannotServe(t *testing.T) { BindingKeys: []string{shippedBindingKey}, // Auth is per provider; these upstreams are stubs needing // no credential, and that is declared rather than defaulted. - Auth: map[string]*upstreamstep.Auth{ + AuthByProvider: map[string]*upstreamstep.AuthProfile{ strings.Split(shippedBindingKey, "|")[0]: {Scheme: upstreamstep.AuthSchemeNone}, }, }) @@ -760,7 +760,7 @@ func TestShippedMappingRefusesPayloadsItCannotAnswer(t *testing.T) { BindingKeys: []string{shippedBindingKey}, // Auth is per provider; these upstreams are stubs needing // no credential, and that is declared rather than defaulted. - Auth: map[string]*upstreamstep.Auth{ + AuthByProvider: map[string]*upstreamstep.AuthProfile{ strings.Split(shippedBindingKey, "|")[0]: {Scheme: upstreamstep.AuthSchemeNone}, }, }) diff --git a/pkg/plugin/implementation/WeatherObservation/cmd/plugin.go b/pkg/plugin/implementation/WeatherObservation/cmd/plugin.go index fa406f7a..f29512fe 100644 --- a/pkg/plugin/implementation/WeatherObservation/cmd/plugin.go +++ b/pkg/plugin/implementation/WeatherObservation/cmd/plugin.go @@ -46,11 +46,11 @@ func (p weatherProvider) parseConfig(config map[string]string) (*WeatherObservat // authScheme- settings. Shared with the other capability // plugins: each used to copy the same field list, so a scheme added in one // had to be remembered in three. - auth, err := upstream.ParseAuth(config) + auth, err := upstream.ParseProviderAuth(config) if err != nil { return nil, err } - cfg.Auth = auth + cfg.AuthByProvider = auth return cfg, nil } diff --git a/pkg/plugin/implementation/WeatherObservation/cmd/plugin_test.go b/pkg/plugin/implementation/WeatherObservation/cmd/plugin_test.go index 73a8087b..eaa38bcf 100644 --- a/pkg/plugin/implementation/WeatherObservation/cmd/plugin_test.go +++ b/pkg/plugin/implementation/WeatherObservation/cmd/plugin_test.go @@ -41,7 +41,7 @@ func TestParseConfig(t *testing.T) { // rules are defined in exactly one place. name: "leaves everything unset for New to default", config: map[string]string{}, - expected: &WeatherObservation.Config{Auth: map[string]*upstream.Auth{}}, + expected: &WeatherObservation.Config{AuthByProvider: map[string]*upstream.AuthProfile{}}, }, { name: "reads every supported setting", @@ -56,7 +56,7 @@ func TestParseConfig(t *testing.T) { }, expected: &WeatherObservation.Config{ BindingKeys: []string{"other|capability"}, - Auth: map[string]*upstream.Auth{ + AuthByProvider: map[string]*upstream.AuthProfile{ "other": { Provider: "other", Scheme: "basic", diff --git a/pkg/plugin/implementation/WeatherObservation/mappings_test.go b/pkg/plugin/implementation/WeatherObservation/mappings_test.go index 564534cd..6de4ae83 100644 --- a/pkg/plugin/implementation/WeatherObservation/mappings_test.go +++ b/pkg/plugin/implementation/WeatherObservation/mappings_test.go @@ -148,7 +148,7 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { BindingKeys: []string{shippedBindingKey}, // Auth is per provider; these upstreams are stubs needing // no credential, and that is declared rather than defaulted. - Auth: map[string]*upstreamstep.Auth{ + AuthByProvider: map[string]*upstreamstep.AuthProfile{ strings.Split(shippedBindingKey, "|")[0]: {Scheme: upstreamstep.AuthSchemeNone}, }, }) diff --git a/pkg/plugin/implementation/internal/upstream/dispatch_test.go b/pkg/plugin/implementation/internal/upstream/dispatch_test.go index 7509bc8f..27901961 100644 --- a/pkg/plugin/implementation/internal/upstream/dispatch_test.go +++ b/pkg/plugin/implementation/internal/upstream/dispatch_test.go @@ -83,7 +83,7 @@ func TestTwoProviderStepsDispatchByBindingKey(t *testing.T) { BindingKeys: []string{bindingKey}, // Auth is per provider, so each step declares its own even // when neither upstream needs a credential. - Auth: map[string]*upstream.Auth{ + AuthByProvider: map[string]*upstream.AuthProfile{ strings.Split(bindingKey, "|")[0]: {Scheme: upstream.AuthSchemeNone}, }, }) diff --git a/pkg/plugin/implementation/internal/upstream/perprovider_test.go b/pkg/plugin/implementation/internal/upstream/perprovider_test.go index 3bd3e785..6aa58ea3 100644 --- a/pkg/plugin/implementation/internal/upstream/perprovider_test.go +++ b/pkg/plugin/implementation/internal/upstream/perprovider_test.go @@ -79,7 +79,7 @@ func TestTwoProvidersOnOneStepAuthenticateDifferently(t *testing.T) { cfg := &Config{ BindingKeys: []string{"alpha|" + capability, "beta|" + capability}, - Auth: map[string]*Auth{ + AuthByProvider: map[string]*AuthProfile{ "alpha": {Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_PP_QUERY_TOKEN"}, "beta": {Scheme: AuthSchemeHeader, HeaderName: "X-Api-Key", HeaderValueEnv: "TEST_PP_HEADER_TOKEN"}, }, @@ -154,7 +154,7 @@ func TestTwoOAuth2ProvidersDoNotShareAToken(t *testing.T) { cfg := &Config{ BindingKeys: []string{"alpha|" + capability, "beta|" + capability}, - Auth: map[string]*Auth{ + AuthByProvider: map[string]*AuthProfile{ "alpha": {Scheme: AuthSchemeOAuth2, TokenURL: issuerA.URL, ClientIDEnv: "TEST_PP_ID_A", ClientSecretEnv: "TEST_PP_SECRET_A"}, "beta": {Scheme: AuthSchemeOAuth2, TokenURL: issuerB.URL, @@ -213,7 +213,7 @@ func TestRedactionCoversEveryProvidersSecret(t *testing.T) { const capability = "openagrinet:KnowledgeAdvisory" cfg := &Config{ BindingKeys: []string{"alpha|" + capability, "beta|" + capability}, - Auth: map[string]*Auth{ + AuthByProvider: map[string]*AuthProfile{ "alpha": {Scheme: AuthSchemeHeader, HeaderName: "X-A", HeaderValueEnv: "TEST_PP_RED_A"}, "beta": {Scheme: AuthSchemeHeader, HeaderName: "X-B", HeaderValueEnv: "TEST_PP_RED_B"}, }, @@ -246,7 +246,7 @@ func TestRedactionSortsAcrossProvidersNotWithinOne(t *testing.T) { const capability = "openagrinet:KnowledgeAdvisory" cfg := &Config{ BindingKeys: []string{"shorty|" + capability, "longy|" + capability}, - Auth: map[string]*Auth{ + AuthByProvider: map[string]*AuthProfile{ "shorty": {Scheme: AuthSchemeHeader, HeaderName: "X-S", HeaderValueEnv: "TEST_PP_SHORT"}, "longy": {Scheme: AuthSchemeHeader, HeaderName: "X-L", HeaderValueEnv: "TEST_PP_LONG"}, }, diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index 893633da..9844b36b 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -152,7 +152,7 @@ type Config struct { ProviderIDAt string `yaml:"providerIdAt" json:"providerIdAt"` CapabilityCodeAt string `yaml:"capabilityCodeAt" json:"capabilityCodeAt"` - // Auth carries one credential profile PER PROVIDER, keyed by participant + // AuthByProvider carries one credential profile PER PROVIDER, keyed by participant // id -- the left half of a binding key. // // Per provider rather than per step because a step serves several binding @@ -168,17 +168,20 @@ type Config struct { // profile is missing is refused at startup rather than quietly falling // through to sending nothing. // - // Built by ParseAuth from the flattened config, not decoded from YAML + // Built by ParseProviderAuth from the flattened config, not decoded from YAML // directly: what an operator writes is a nested block per provider, which // pkg/plugin flattens on the way in. - Auth map[string]*Auth `yaml:"-" json:"-"` + AuthByProvider map[string]*AuthProfile `yaml:"-" json:"-"` // MaxResponseBytes caps what is read from the provider. MaxResponseBytes int64 `yaml:"maxResponseBytes" json:"maxResponseBytes"` } -// Auth is how one provider's credentials are presented upstream. -type Auth struct { +// AuthProfile is how ONE provider's credentials are presented upstream. +// +// One per provider rather than one per step: a step serves several binding +// keys, and the providers behind them need not authenticate alike. +type AuthProfile struct { // Provider is the participant id this profile belongs to. Held so an error // can name it: with several profiles on one step, "authScheme query // requires queryName" would otherwise leave an operator guessing which @@ -224,7 +227,7 @@ type Auth struct { // authenticating as somebody else -- a failure no test of a single provider // can see. type authenticator struct { - cfg Auth + cfg AuthProfile // Two mechanisms because there are two jobs. tokenMu serialises the // EXCHANGE, so a cold start sends one request to the issuer rather than one @@ -282,9 +285,9 @@ func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper d // Timeout is set per request from the registry's own budget, so the // client carries none of its own. httpClient: &http.Client{}, - auth: make(map[string]*authenticator, len(cfg.Auth)), + auth: make(map[string]*authenticator, len(cfg.AuthByProvider)), } - for provider, profile := range cfg.Auth { + for provider, profile := range cfg.AuthByProvider { step.auth[provider] = &authenticator{cfg: *profile} } @@ -336,8 +339,8 @@ func applyDefaults(cfg *Config) error { if cfg.MaxResponseBytes <= 0 { cfg.MaxResponseBytes = DefaultMaxResponseBytes } - if cfg.Auth == nil { - cfg.Auth = map[string]*Auth{} + if cfg.AuthByProvider == nil { + cfg.AuthByProvider = map[string]*AuthProfile{} } // Both directions, because each catches a different mistake and both are @@ -347,9 +350,9 @@ func applyDefaults(cfg *Config) error { // us. served := map[string]bool{} for _, key := range cfg.BindingKeys { - served[participantOf(key)] = true + served[providerIDFrom(key)] = true } - for provider := range cfg.Auth { + for provider := range cfg.AuthByProvider { if !served[provider] { return fmt.Errorf( "upstream: auth is configured for %q, which is not a provider in bindingKeys (%s)", @@ -357,7 +360,7 @@ func applyDefaults(cfg *Config) error { } } for provider := range served { - profile, ok := cfg.Auth[provider] + profile, ok := cfg.AuthByProvider[provider] if !ok { return fmt.Errorf( "upstream: %q is served but has no auth block; every provider declares its own, "+ @@ -374,7 +377,7 @@ func applyDefaults(cfg *Config) error { // validate refuses a profile whose scheme and fields disagree. Every message // names the provider: with several profiles on one step, the field alone would // leave an operator guessing which block to look at. -func (a *Auth) validate() error { +func (a *AuthProfile) validate() error { switch a.Scheme { case AuthSchemeNone: case AuthSchemeBasic: @@ -409,7 +412,7 @@ func (a *Auth) validate() error { return nil } -// ParseAuth builds one credential profile per provider from a plugin's +// ParseProviderAuth builds one credential profile per provider from a plugin's // flattened settings. // // What an operator writes is a block per provider: @@ -429,8 +432,8 @@ func (a *Auth) validate() error { // The split is on the FIRST dash, which is unambiguous because no setting name // contains one while a participant id routinely does -- knowledge-provider, // provider.oan.dev. So the field is always the part before it. -func ParseAuth(config map[string]string) (map[string]*Auth, error) { - profiles := map[string]*Auth{} +func ParseProviderAuth(config map[string]string) (map[string]*AuthProfile, error) { + profiles := map[string]*AuthProfile{} // Sorted so a config with two mistakes reports the same one every run. keys := make([]string, 0, len(config)) @@ -464,7 +467,7 @@ func ParseAuth(config map[string]string) (map[string]*Auth, error) { profile, seen := profiles[provider] if !seen { - profile = &Auth{Provider: provider} + profile = &AuthProfile{Provider: provider} profiles[provider] = profile } value := config[key] @@ -511,10 +514,10 @@ var authFields = map[string]bool{ "clientSecretEnv": true, } -// participantOf returns the provider half of a binding key. The format is +// providerIDFrom returns the provider half of a binding key. The format is // "|", and a participant id carries dashes and // dots but never a pipe, so the first one separates them. -func participantOf(bindingKey string) string { +func providerIDFrom(bindingKey string) string { provider, _, _ := strings.Cut(bindingKey, "|") return strings.TrimSpace(provider) } @@ -633,7 +636,7 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { // one step serving several providers authenticates each as its own. // Startup guarantees a profile per served provider; this guards the case // where a record arrives for a key the config never declared. - auth, configured := s.auth[participantOf(plan.BindingKey)] + auth, configured := s.auth[providerIDFrom(plan.BindingKey)] if !configured { return fmt.Errorf("upstream: no credential is configured for %s", plan.BindingKey) } diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go index d1479a0b..baba0d2f 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -55,28 +55,28 @@ const testProvider = "mausamgram" // setAuth gives every provider this config serves the same profile. Keyed off // BindingKeys rather than a constant, so a test that serves a different // provider does not also have to remember to move its auth. -func (c *Config) setAuth(a Auth) { - c.Auth = map[string]*Auth{} +func (c *Config) setProviderAuth(a AuthProfile) { + c.AuthByProvider = map[string]*AuthProfile{} for _, key := range c.BindingKeys { - provider := participantOf(key) + provider := providerIDFrom(key) profile := a profile.Provider = provider - c.Auth[provider] = &profile + c.AuthByProvider[provider] = &profile } } -// authFor is setAuth for a Config literal. -func authFor(a Auth) map[string]*Auth { +// authForTestProvider is setProviderAuth for a Config literal. +func authForTestProvider(a AuthProfile) map[string]*AuthProfile { a.Provider = testProvider - return map[string]*Auth{testProvider: &a} + return map[string]*AuthProfile{testProvider: &a} } -// stepWithAuth builds a Step directly, for the handful of tests that exercise +// stepWithProviderAuth builds a Step directly, for the handful of tests that exercise // redaction without a request. New() is the path everything else takes. -func stepWithAuth(a Auth) *Step { +func stepWithProviderAuth(a AuthProfile) *Step { a.Provider = testProvider return &Step{ - config: &Config{BindingKeys: []string{testBindingKey}, Auth: authFor(a)}, + config: &Config{BindingKeys: []string{testBindingKey}, AuthByProvider: authForTestProvider(a)}, auth: map[string]*authenticator{testProvider: {cfg: a}}, } } @@ -146,8 +146,8 @@ func newStep(t *testing.T, registry definition.ProviderRecordLookup, mapper defi // There is no step-wide default, so a served provider without a profile is // refused at startup. Filled in after the tweaks, because a tweak may // replace BindingKeys and the profile has to follow. - if len(cfg.Auth) == 0 { - cfg.setAuth(Auth{Scheme: AuthSchemeNone}) + if len(cfg.AuthByProvider) == 0 { + cfg.setProviderAuth(AuthProfile{Scheme: AuthSchemeNone}) } step, closer, err := New(context.Background(), registry, mapper, nil, cfg) if err != nil { @@ -184,26 +184,26 @@ func TestNewValidatesTheAuthScheme(t *testing.T) { config *Config valid bool }{ - {"none, declared", &Config{Auth: authFor(Auth{Scheme: AuthSchemeNone})}, true}, + {"none, declared", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeNone})}, true}, {"no profile at all for a served provider", &Config{}, false}, - {"an empty scheme", &Config{Auth: authFor(Auth{})}, false}, - {"basic with both variables", &Config{Auth: authFor(Auth{Scheme: AuthSchemeBasic, UsernameEnv: "U", PasswordEnv: "P"})}, true}, - {"basic missing the password variable", &Config{Auth: authFor(Auth{Scheme: AuthSchemeBasic, UsernameEnv: "U"})}, false}, - {"basic missing the username variable", &Config{Auth: authFor(Auth{Scheme: AuthSchemeBasic, PasswordEnv: "P"})}, false}, - {"header with both settings", &Config{Auth: authFor(Auth{Scheme: AuthSchemeHeader, HeaderName: "X-Key", HeaderValueEnv: "V"})}, true}, - {"header missing the value variable", &Config{Auth: authFor(Auth{Scheme: AuthSchemeHeader, HeaderName: "X-Key"})}, false}, - {"query with both settings", &Config{Auth: authFor(Auth{Scheme: AuthSchemeQuery, QueryName: "t", QueryValueEnv: "V"})}, true}, - {"an unknown scheme", &Config{Auth: authFor(Auth{Scheme: "oauth"})}, false}, - {"oauth2 with all three settings", &Config{Auth: authFor(Auth{Scheme: AuthSchemeOAuth2, + {"an empty scheme", &Config{AuthByProvider: authForTestProvider(AuthProfile{})}, false}, + {"basic with both variables", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeBasic, UsernameEnv: "U", PasswordEnv: "P"})}, true}, + {"basic missing the password variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeBasic, UsernameEnv: "U"})}, false}, + {"basic missing the username variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeBasic, PasswordEnv: "P"})}, false}, + {"header with both settings", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeHeader, HeaderName: "X-Key", HeaderValueEnv: "V"})}, true}, + {"header missing the value variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeHeader, HeaderName: "X-Key"})}, false}, + {"query with both settings", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "t", QueryValueEnv: "V"})}, true}, + {"an unknown scheme", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: "oauth"})}, false}, + {"oauth2 with all three settings", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeOAuth2, TokenURL: "https://issuer.invalid/token", ClientIDEnv: "ID", ClientSecretEnv: "SECRET"})}, true}, - {"oauth2 missing the token url", &Config{Auth: authFor(Auth{Scheme: AuthSchemeOAuth2, + {"oauth2 missing the token url", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeOAuth2, ClientIDEnv: "ID", ClientSecretEnv: "SECRET"})}, false}, - {"oauth2 missing the client id variable", &Config{Auth: authFor(Auth{Scheme: AuthSchemeOAuth2, + {"oauth2 missing the client id variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeOAuth2, TokenURL: "https://issuer.invalid/token", ClientSecretEnv: "SECRET"})}, false}, - {"oauth2 missing the client secret variable", &Config{Auth: authFor(Auth{Scheme: AuthSchemeOAuth2, + {"oauth2 missing the client secret variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeOAuth2, TokenURL: "https://issuer.invalid/token", ClientIDEnv: "ID"})}, false}, {"a profile for a provider not in bindingKeys", &Config{ - Auth: map[string]*Auth{"someone-else": {Scheme: AuthSchemeNone}}}, false}, + AuthByProvider: map[string]*AuthProfile{"someone-else": {Scheme: AuthSchemeNone}}}, false}, } for _, tc := range testCases { @@ -337,7 +337,7 @@ func TestRunDoesNotRetryAMissingCredential(t *testing.T) { } mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} step := newStep(t, &stubRegistry{plan: plan}, mapper, func(c *Config) { - c.setAuth(Auth{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_ABSENT_USER_FOR_RETRY", PasswordEnv: "TEST_ABSENT_PASS_FOR_RETRY"}) + c.setProviderAuth(AuthProfile{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_ABSENT_USER_FOR_RETRY", PasswordEnv: "TEST_ABSENT_PASS_FOR_RETRY"}) }) _, err := runStep(t, step, selectBody) @@ -553,7 +553,7 @@ func TestRunRedactsACredentialEchoedInABody(t *testing.T) { mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper, func(c *Config) { - c.setAuth(Auth{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_ECHO_TOKEN"}) + c.setProviderAuth(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_ECHO_TOKEN"}) }) _, err := runStep(t, step, selectBody) @@ -597,7 +597,7 @@ func TestRunSendsTheCredentialAsAQueryParameter(t *testing.T) { mapper := &stubMapper{requestResult: []byte(`{"statecode":"CG"}`), responseResult: []byte(`{}`)} step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper, func(c *Config) { - c.setAuth(Auth{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) + c.setProviderAuth(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) }) if _, err := runStep(t, step, selectBody); err != nil { @@ -627,7 +627,7 @@ func TestRunRedactsAQueryCredentialFromAnError(t *testing.T) { } mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} step := newStep(t, &stubRegistry{plan: plan}, mapper, func(c *Config) { - c.setAuth(Auth{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) + c.setProviderAuth(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) }) _, err := runStep(t, step, selectBody) @@ -649,7 +649,7 @@ func TestRedactStringRemovesTheCredentialFromTheURL(t *testing.T) { // No t.Parallel: t.Setenv forbids it. t.Setenv("TEST_MANDI_TOKEN", "s3cr3t") - step := stepWithAuth(Auth{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) + step := stepWithProviderAuth(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) got := step.redactString("http://host/v1/x?statecode=CG&token=s3cr3t") if strings.Contains(got, "s3cr3t") { t.Errorf("the credential survived redaction: %s", got) @@ -659,7 +659,7 @@ func TestRedactStringRemovesTheCredentialFromTheURL(t *testing.T) { } // Any other scheme has nothing to hide in a URL, so the text is untouched. - plain := stepWithAuth(Auth{Scheme: AuthSchemeNone}) + plain := stepWithProviderAuth(AuthProfile{Scheme: AuthSchemeNone}) if out := plain.redactString("http://host/v1/x?statecode=CG"); out != "http://host/v1/x?statecode=CG" { t.Errorf("a url with no credential must pass through unchanged, got %q", out) } @@ -675,7 +675,7 @@ func TestRedactStringRemovesThePercentEncodedCredential(t *testing.T) { const token = "a+b/c=d e" // every character Encode treats specially t.Setenv("TEST_MANDI_TOKEN", token) - step := stepWithAuth(Auth{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) + step := stepWithProviderAuth(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) // Exactly how the credential appears once authenticate has run: Encode // escapes it, so this is the string a transport error quotes. @@ -701,7 +701,7 @@ func TestRedactStringStillRemovesAnUnescapedCredential(t *testing.T) { // No t.Parallel: t.Setenv forbids it. t.Setenv("TEST_MANDI_TOKEN", "plaintoken123") - step := stepWithAuth(Auth{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) + step := stepWithProviderAuth(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) got := step.redactString("http://host/v1/x?token=plaintoken123") if strings.Contains(got, "plaintoken123") { t.Errorf("the credential survived redaction: %s", got) @@ -718,9 +718,9 @@ func TestNewRefusesAHalfConfiguredQueryScheme(t *testing.T) { cfg *Config }{ {"no queryName", &Config{BindingKeys: []string{testBindingKey}, - Auth: authFor(Auth{Scheme: AuthSchemeQuery, QueryValueEnv: "TEST_MANDI_TOKEN"})}}, + AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeQuery, QueryValueEnv: "TEST_MANDI_TOKEN"})}}, {"no queryValueEnv", &Config{BindingKeys: []string{testBindingKey}, - Auth: authFor(Auth{Scheme: AuthSchemeQuery, QueryName: "token"})}}, + AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token"})}}, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -1157,9 +1157,9 @@ func TestNewRefusesAHalfConfiguredOverride(t *testing.T) { t.Parallel() _, _, err := New(context.Background(), &stubRegistry{}, &stubMapper{}, nil, &Config{ - BindingKeys: []string{testBindingKey}, - ProviderIDAt: "who.provider", - Auth: authFor(Auth{Scheme: AuthSchemeNone}), + BindingKeys: []string{testBindingKey}, + ProviderIDAt: "who.provider", + AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeNone}), }) if err == nil { t.Fatal("expected one path without the other to be refused") @@ -1408,7 +1408,7 @@ func TestRunPresentsBasicCredentialsFromTheEnvironment(t *testing.T) { step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, func(c *Config) { - c.setAuth(Auth{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_MAUSAMGRAM_USER", PasswordEnv: "TEST_MAUSAMGRAM_KEY"}) + c.setProviderAuth(AuthProfile{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_MAUSAMGRAM_USER", PasswordEnv: "TEST_MAUSAMGRAM_KEY"}) }) if _, err := runStep(t, step, selectBody); err != nil { @@ -1432,7 +1432,7 @@ func TestRunPresentsAHeaderCredentialFromTheEnvironment(t *testing.T) { step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, func(c *Config) { - c.setAuth(Auth{Scheme: AuthSchemeHeader, HeaderName: "X-Api-Key", HeaderValueEnv: "TEST_MAUSAMGRAM_TOKEN"}) + c.setProviderAuth(AuthProfile{Scheme: AuthSchemeHeader, HeaderName: "X-Api-Key", HeaderValueEnv: "TEST_MAUSAMGRAM_TOKEN"}) }) if _, err := runStep(t, step, selectBody); err != nil { @@ -1454,7 +1454,7 @@ func TestRunFailsWhenAConfiguredCredentialIsAbsent(t *testing.T) { step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, func(c *Config) { - c.setAuth(Auth{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_MAUSAMGRAM_ABSENT_USER", PasswordEnv: "TEST_MAUSAMGRAM_ABSENT_KEY"}) + c.setProviderAuth(AuthProfile{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_MAUSAMGRAM_ABSENT_USER", PasswordEnv: "TEST_MAUSAMGRAM_ABSENT_KEY"}) }) if _, err := runStep(t, step, selectBody); err == nil { @@ -1758,7 +1758,7 @@ func TestRedactKeepsTheErrorChainMatchable(t *testing.T) { // No t.Parallel: t.Setenv forbids it. t.Setenv("TEST_CHAIN_TOKEN", "s3cr3t") - step := stepWithAuth(Auth{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_CHAIN_TOKEN"}) + step := stepWithProviderAuth(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_CHAIN_TOKEN"}) // The shape net/http produces: the cause wrapped behind text that quotes // the whole URL, credential and all. @@ -1792,7 +1792,7 @@ func TestRedactLeavesAnUnchangedErrorAlone(t *testing.T) { // No t.Parallel: t.Setenv forbids it. t.Setenv("TEST_CHAIN_TOKEN_2", "s3cr3t") - step := stepWithAuth(Auth{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_CHAIN_TOKEN_2"}) + step := stepWithProviderAuth(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_CHAIN_TOKEN_2"}) original := errors.New("nothing sensitive here") if got := step.redact(original); got != original { t.Errorf("redact returned a different error for text it did not change: %v", got) @@ -1967,7 +1967,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { { name: "basic, the wire form a gateway echoes", tweak: func(c *Config) { - c.setAuth(Auth{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_USER", PasswordEnv: "TEST_PASS"}) + c.setProviderAuth(AuthProfile{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_USER", PasswordEnv: "TEST_PASS"}) }, body: `{"error":"invalid Authorization: Basic ` + wire + `"}`, secret: wire, @@ -1975,7 +1975,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { { name: "basic, the password quoted raw", tweak: func(c *Config) { - c.setAuth(Auth{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_USER", PasswordEnv: "TEST_PASS"}) + c.setProviderAuth(AuthProfile{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_USER", PasswordEnv: "TEST_PASS"}) }, body: `{"error":"bad password s3cr3t"}`, secret: "s3cr3t", @@ -1983,7 +1983,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { { name: "header, the value as sent", tweak: func(c *Config) { - c.setAuth(Auth{Scheme: AuthSchemeHeader, HeaderName: "X-API-Key", HeaderValueEnv: "TEST_HDR"}) + c.setProviderAuth(AuthProfile{Scheme: AuthSchemeHeader, HeaderName: "X-API-Key", HeaderValueEnv: "TEST_HDR"}) }, body: `{"error":"bad X-API-Key: hdr-k3y"}`, secret: "hdr-k3y", @@ -1991,7 +1991,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { { name: "query, still covered, raw form", tweak: func(c *Config) { - c.setAuth(Auth{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_QRY"}) + c.setProviderAuth(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_QRY"}) }, body: `{"rejected":"token=a+b/c="}`, secret: "a+b/c=", @@ -1999,7 +1999,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { { name: "query, still covered, percent-encoded form", tweak: func(c *Config) { - c.setAuth(Auth{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_QRY"}) + c.setProviderAuth(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_QRY"}) }, body: `{"rejected":"token=` + url.QueryEscape("a+b/c=") + `"}`, secret: url.QueryEscape("a+b/c="), @@ -2033,7 +2033,7 @@ func TestRedactStringLeavesTheBasicUsername(t *testing.T) { step := newStep(t, &stubRegistry{plan: testPlan("http://provider.invalid", http.MethodGet)}, &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, func(c *Config) { - c.setAuth(Auth{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_USER", PasswordEnv: "TEST_PASS"}) + c.setProviderAuth(AuthProfile{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_USER", PasswordEnv: "TEST_PASS"}) }) got := step.redactString(`user mausam failed to authenticate with s3cr3t`) @@ -2055,7 +2055,7 @@ func TestRedactStringWithNoCredentialConfigured(t *testing.T) { &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, func(c *Config) { // Env vars named but deliberately unset. - c.setAuth(Auth{ + c.setProviderAuth(AuthProfile{ Scheme: scheme, UsernameEnv: "TEST_UNSET_U", PasswordEnv: "TEST_UNSET_P", @@ -2096,7 +2096,7 @@ func TestRunHandsResolvedPrerequisitesToTheMappingAsLocal(t *testing.T) { step, closer, err := New(context.Background(), &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper, prerequisites, &Config{BindingKeys: []string{testBindingKey}, - Auth: authFor(Auth{Scheme: AuthSchemeNone})}) + AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeNone})}) if err != nil { t.Fatalf("New() returned an unexpected error: %v", err) } @@ -2263,7 +2263,7 @@ func oauth2Step(t *testing.T, tokenURL string, seen *[]string) *Step { return newStep(t, &stubRegistry{plan: testPlan(provider.URL, http.MethodGet)}, &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{"a":1}`)}, func(c *Config) { - c.setAuth(Auth{Scheme: AuthSchemeOAuth2, TokenURL: tokenURL, ClientIDEnv: "TEST_OAUTH_ID", ClientSecretEnv: "TEST_OAUTH_SECRET"}) + c.setProviderAuth(AuthProfile{Scheme: AuthSchemeOAuth2, TokenURL: tokenURL, ClientIDEnv: "TEST_OAUTH_ID", ClientSecretEnv: "TEST_OAUTH_SECRET"}) }) } diff --git a/pkg/plugin/implementation/otelsetup/otelsetup_test.go b/pkg/plugin/implementation/otelsetup/otelsetup_test.go index a61a34cb..b5f98551 100644 --- a/pkg/plugin/implementation/otelsetup/otelsetup_test.go +++ b/pkg/plugin/implementation/otelsetup/otelsetup_test.go @@ -204,7 +204,7 @@ func TestToPluginConfig_Success(t *testing.T) { name string cfg *Config expectedID string - expectedConfig plugin.Settings + expectedConfig plugin.ConfigBlock }{ { name: "Valid config with all fields", @@ -221,7 +221,7 @@ func TestToPluginConfig_Success(t *testing.T) { TimeInterval: 5, }, expectedID: "otelsetup", - expectedConfig: plugin.Settings{ + expectedConfig: plugin.ConfigBlock{ "serviceName": "test-service", "serviceVersion": "1.0.0", "environment": "test", @@ -245,7 +245,7 @@ func TestToPluginConfig_Success(t *testing.T) { Environment: "production", }, expectedID: "otelsetup", - expectedConfig: plugin.Settings{ + expectedConfig: plugin.ConfigBlock{ "serviceName": "my-service", "serviceVersion": "2.0.0", "environment": "production", @@ -272,7 +272,7 @@ func TestToPluginConfig_Success(t *testing.T) { OtlpEndpoint: "", }, expectedID: "otelsetup", - expectedConfig: plugin.Settings{ + expectedConfig: plugin.ConfigBlock{ "serviceName": "", "serviceVersion": "", "environment": "", diff --git a/pkg/plugin/manager_test.go b/pkg/plugin/manager_test.go index 655134a4..fcd8c37e 100644 --- a/pkg/plugin/manager_test.go +++ b/pkg/plugin/manager_test.go @@ -2588,7 +2588,7 @@ func TestApplyConstants_UnknownPlugin_NoChange(t *testing.T) { m := makeManager(map[string]map[string]string{"dediregistry": {"url": "https://x.com"}}, nil) c := cfg("cache", map[string]string{"addr": "redis:6379"}) require.NoError(t, m.applyConstants(context.Background(), c)) - assert.Equal(t, Settings{"addr": "redis:6379"}, c.Config) + assert.Equal(t, ConfigBlock{"addr": "redis:6379"}, c.Config) } // --- nil constants --- From 3d49617a33bec9822f54b763bd23f857866f418e Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 17:13:16 +0530 Subject: [PATCH 106/130] docs(upstream): follow the rename in a doc comment [#7] --- pkg/plugin/implementation/internal/upstream/upstream_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/upstream/upstream_test.go index baba0d2f..0f06f8e8 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/upstream/upstream_test.go @@ -52,7 +52,8 @@ const testBindingKey = "mausamgram|openagrinet:WeatherObservation" // provider, so a step config needs a profile under this key. const testProvider = "mausamgram" -// setAuth gives every provider this config serves the same profile. Keyed off +// setProviderAuth gives every provider this config serves the same profile. +// Keyed off // BindingKeys rather than a constant, so a test that serves a different // provider does not also have to remember to move its auth. func (c *Config) setProviderAuth(a AuthProfile) { From 51f4f56114ec9c900985e319321588bacdf50336 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 17:47:22 +0530 Subject: [PATCH 107/130] refactor(upstream): split upstream.go by concern [#7] upstream.go had grown to 1,532 lines and 58 declarations, from config parsing to URL escaping. Split into seven files along the call chain, all in the same package: constant.go 87 the 16 constants, and nothing else upstream.go 225 Config, Step, New, applyDefaults, bindingPaths serve.go 219 Run, serve, resolve, buildRequest -- the pipeline http.go 436 the call, its retry budget, the endpoint, the body auth.go 281 AuthProfile, ParseProviderAuth, authenticate token.go 163 the oauth2 exchange and the token each provider holds redact.go 196 redact, secretForms, longestFirst A PURE MOVE. Nothing renamed, nothing exported, no test touched. Verified three ways rather than by reading: - all 58 declarations present exactly once; the splitter refuses to run if one is placed twice or left behind - every declaration byte-for-byte identical, doc comment included, compared as an unordered set so file boundaries and ordering do not matter - 66 packages green with zero test edits, no races, vet clean http.go is the largest at 436 because it absorbs what could have been three files -- the retry loop, the endpoint construction and the method and body helpers. They are one subject: everything about talking HTTP to a provider. Imports were pruned per file with goimports; constant.go needs only time. authFields stays in auth.go rather than constant.go. It is a var -- Go has no constant maps -- and it belongs beside the parser that reads it. --- .../implementation/internal/upstream/auth.go | 281 ++++ .../internal/upstream/constant.go | 87 ++ .../implementation/internal/upstream/http.go | 436 ++++++ .../internal/upstream/redact.go | 196 +++ .../implementation/internal/upstream/serve.go | 219 +++ .../implementation/internal/upstream/token.go | 163 ++ .../internal/upstream/upstream.go | 1307 ----------------- 7 files changed, 1382 insertions(+), 1307 deletions(-) create mode 100644 pkg/plugin/implementation/internal/upstream/auth.go create mode 100644 pkg/plugin/implementation/internal/upstream/constant.go create mode 100644 pkg/plugin/implementation/internal/upstream/http.go create mode 100644 pkg/plugin/implementation/internal/upstream/redact.go create mode 100644 pkg/plugin/implementation/internal/upstream/serve.go create mode 100644 pkg/plugin/implementation/internal/upstream/token.go diff --git a/pkg/plugin/implementation/internal/upstream/auth.go b/pkg/plugin/implementation/internal/upstream/auth.go new file mode 100644 index 00000000..9a3edc3a --- /dev/null +++ b/pkg/plugin/implementation/internal/upstream/auth.go @@ -0,0 +1,281 @@ +// Credentials: one profile per provider, how a profile is read from config, +// and how it is presented on a request. +package upstream + +import ( + "context" + "fmt" + "net/http" + "os" + "sort" + "strings" + "sync" + "sync/atomic" + + "github.com/beckn-one/beckn-onix/pkg/log" +) + +// AuthProfile is how ONE provider's credentials are presented upstream. +// +// One per provider rather than one per step: a step serves several binding +// keys, and the providers behind them need not authenticate alike. +type AuthProfile struct { + // Provider is the participant id this profile belongs to. Held so an error + // can name it: with several profiles on one step, "authScheme query + // requires queryName" would otherwise leave an operator guessing which + // provider it meant. + Provider string + + // Scheme is one of none, basic, header, query or oauth2. Providers differ + // here, which is why it is configuration and not an assumption. + Scheme string + + // UsernameEnv and PasswordEnv name the environment variables holding basic + // credentials. They are variable NAMES, never the values. + UsernameEnv string + PasswordEnv string + + // HeaderName and HeaderValueEnv configure the header scheme: which header + // to set, and which environment variable holds its value. + HeaderName string + HeaderValueEnv string + + // QueryName and QueryValueEnv configure the query scheme: the parameter + // name to add, and the environment variable holding its value. Named the + // same way as the header pair, for the same reason -- the credential is + // never in this config, only the name of the variable carrying it. + QueryName string + QueryValueEnv string + + // TokenURL is the OAuth2 token endpoint. Not a credential, so it is named + // here rather than through an environment variable -- but it IS + // deployment-specific, so the reference config carries a placeholder. + TokenURL string + // ClientIDEnv and ClientSecretEnv name the variables holding the client + // credentials. The values never appear in config, the registry, or a log. + ClientIDEnv string + ClientSecretEnv string +} + +// authenticator is one provider's profile plus the token it holds. +// +// The token cache lives HERE rather than on the Step, and that is the whole +// reason this type exists. A step-wide cache shared between two oauth2 +// providers would hand the first provider's token to the second, which is +// authenticating as somebody else -- a failure no test of a single provider +// can see. +type authenticator struct { + cfg AuthProfile + + // Two mechanisms because there are two jobs. tokenMu serialises the + // EXCHANGE, so a cold start sends one request to the issuer rather than one + // per concurrent caller. token is atomic so READERS never take that mutex, + // which matters because secretForms is one of them and it is reached from + // inside the exchange -- guarding the value with tokenMu instead deadlocked + // on the first failing exchange, which is how this was found. + tokenMu sync.Mutex + token atomic.Pointer[cachedToken] +} + +// validate refuses a profile whose scheme and fields disagree. Every message +// names the provider: with several profiles on one step, the field alone would +// leave an operator guessing which block to look at. +func (a *AuthProfile) validate() error { + switch a.Scheme { + case AuthSchemeNone: + case AuthSchemeBasic: + if a.UsernameEnv == "" || a.PasswordEnv == "" { + return fmt.Errorf( + "upstream: %s: authScheme basic requires usernameEnv and passwordEnv", a.Provider) + } + case AuthSchemeHeader: + if a.HeaderName == "" || a.HeaderValueEnv == "" { + return fmt.Errorf( + "upstream: %s: authScheme header requires headerName and headerValueEnv", a.Provider) + } + case AuthSchemeQuery: + if a.QueryName == "" || a.QueryValueEnv == "" { + return fmt.Errorf( + "upstream: %s: authScheme query requires queryName and queryValueEnv", a.Provider) + } + case AuthSchemeOAuth2: + if a.TokenURL == "" || a.ClientIDEnv == "" || a.ClientSecretEnv == "" { + return fmt.Errorf( + "upstream: %s: authScheme oauth2 requires tokenUrl, clientIdEnv and clientSecretEnv", + a.Provider) + } + case "": + return fmt.Errorf("upstream: %s: authScheme is required, "+ + "and is none where the upstream needs no credential", a.Provider) + default: + return fmt.Errorf( + "upstream: %s: unknown authScheme %q: must be none, basic, header, query or oauth2", + a.Provider, a.Scheme) + } + return nil +} + +// ParseProviderAuth builds one credential profile per provider from a plugin's +// flattened settings. +// +// What an operator writes is a block per provider: +// +// knowledge-provider: +// authScheme: oauth2 +// tokenUrl: https://issuer.example/token +// +// which pkg/plugin flattens to authScheme-knowledge-provider and +// tokenUrl-knowledge-provider before any plugin sees it. This reads that form +// back into profiles. +// +// Shared rather than repeated in each capability plugin: all of them copied +// the same field list out of the same map, so a scheme added in one place had +// to be remembered in three. +// +// The split is on the FIRST dash, which is unambiguous because no setting name +// contains one while a participant id routinely does -- knowledge-provider, +// provider.oan.dev. So the field is always the part before it. +func ParseProviderAuth(config map[string]string) (map[string]*AuthProfile, error) { + profiles := map[string]*AuthProfile{} + + // Sorted so a config with two mistakes reports the same one every run. + keys := make([]string, 0, len(config)) + for key := range config { + keys = append(keys, key) + } + sort.Strings(keys) + + for _, key := range keys { + field, provider, dashed := strings.Cut(key, "-") + if !dashed { + // A bare auth field is the old step-wide form. Refused rather than + // ignored: silently dropping it leaves every provider on no + // credential at all, which reads as the provider rejecting us. + if authFields[key] { + return nil, fmt.Errorf( + "upstream: %q is set for the whole step; auth is per provider now, "+ + "so it belongs in a block named for the participant id", key) + } + continue + } + if !authFields[field] { + // Not an auth setting, and nothing else on a provider step carries + // a dash -- so this is a misspelled field rather than something to + // pass through. + return nil, fmt.Errorf("upstream: %q is not a credential setting", key) + } + if strings.TrimSpace(provider) == "" { + return nil, fmt.Errorf("upstream: %q names no provider after the dash", key) + } + + profile, seen := profiles[provider] + if !seen { + profile = &AuthProfile{Provider: provider} + profiles[provider] = profile + } + value := config[key] + switch field { + case "authScheme": + profile.Scheme = value + case "usernameEnv": + profile.UsernameEnv = value + case "passwordEnv": + profile.PasswordEnv = value + case "headerName": + profile.HeaderName = value + case "headerValueEnv": + profile.HeaderValueEnv = value + case "queryName": + profile.QueryName = value + case "queryValueEnv": + profile.QueryValueEnv = value + case "tokenUrl": + profile.TokenURL = value + case "clientIdEnv": + profile.ClientIDEnv = value + case "clientSecretEnv": + profile.ClientSecretEnv = value + } + } + return profiles, nil +} + +// authFields is the closed set of per-provider credential settings. Closed on +// purpose: it is what makes the split on the first dash decidable, and what +// turns a misspelled field into a startup error rather than a setting that +// quietly does nothing. +var authFields = map[string]bool{ + "authScheme": true, + "usernameEnv": true, + "passwordEnv": true, + "headerName": true, + "headerValueEnv": true, + "queryName": true, + "queryValueEnv": true, + "tokenUrl": true, + "clientIdEnv": true, + "clientSecretEnv": true, +} + +// providerIDFrom returns the provider half of a binding key. The format is +// "|", and a participant id carries dashes and +// dots but never a pipe, so the first one separates them. +func providerIDFrom(bindingKey string) string { + provider, _, _ := strings.Cut(bindingKey, "|") + return strings.TrimSpace(provider) +} + +// authenticate presents this provider's credentials, read from the environment +// at call time so a rotated secret takes effect without a restart. +// missingCredential reports an unset credential without naming the variable on +// the wire. +// +// The variable name is deployment configuration, and this error is wrapped into +// a 502 that is signed and returned to a network peer. Telling a peer that +// MANDI_TOKEN is what this deployment reads describes the inside of somebody +// else's stack for no benefit to the caller -- the caller cannot set it, and +// the fix is entirely the operator's. So the name goes to the log, where the +// operator is, and the wire gets the scheme that failed. +func (s *Step) missingCredential(ctx context.Context, provider, scheme, envNames string) error { + err := fmt.Errorf("upstream: the %s credential for %s is not configured", scheme, provider) + log.Errorf(ctx, err, "upstream: %s auth is configured for %s but %s is not set", + scheme, provider, envNames) + return err +} + +func (s *Step) authenticate(auth *authenticator, req *http.Request) error { + cfg := auth.cfg + switch cfg.Scheme { + case AuthSchemeBasic: + username, password := os.Getenv(cfg.UsernameEnv), os.Getenv(cfg.PasswordEnv) + if username == "" || password == "" { + return s.missingCredential(req.Context(), cfg.Provider, "basic", + cfg.UsernameEnv+" and "+cfg.PasswordEnv) + } + req.SetBasicAuth(username, password) + case AuthSchemeHeader: + value := os.Getenv(cfg.HeaderValueEnv) + if value == "" { + return s.missingCredential(req.Context(), cfg.Provider, "header", cfg.HeaderValueEnv) + } + req.Header.Set(cfg.HeaderName, value) + case AuthSchemeQuery: + value := os.Getenv(cfg.QueryValueEnv) + if value == "" { + return s.missingCredential(req.Context(), cfg.Provider, "query", cfg.QueryValueEnv) + } + // Set rather than Add: a second copy of the parameter is not a + // credential, it is an ambiguity, and which one an upstream reads is + // its own business. + query := req.URL.Query() + query.Set(cfg.QueryName, value) + req.URL.RawQuery = query.Encode() + case AuthSchemeOAuth2: + token, err := s.bearerToken(req.Context(), auth) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+token) + } + return nil +} diff --git a/pkg/plugin/implementation/internal/upstream/constant.go b/pkg/plugin/implementation/internal/upstream/constant.go new file mode 100644 index 00000000..438a5f13 --- /dev/null +++ b/pkg/plugin/implementation/internal/upstream/constant.go @@ -0,0 +1,87 @@ +// Constants: the budgets a registry row is clamped to, the retry backoff, the +// auth schemes, and the two markers this package puts in text it returns. +package upstream + +import ( + "time" +) + +// Defaults applied when the registry or the operator leaves a setting out. +const ( + // DefaultTimeout and DefaultRetryMax are the registry contract's defaults + // for an action that leaves timeoutMs or retryMax out. Zero retries is + // deliberate: a provider that failed is retried only where the operator + // said so, because a retry on a non-idempotent action is a second booking. + DefaultTimeout = 15 * time.Second + DefaultRetryMax = 0 + // DefaultMaxResponseBytes caps what is read from the provider. The response + // is mapped in memory, so an unbounded one is an unbounded allocation. + DefaultMaxResponseBytes = 4 << 20 // 4 MiB + + // MaxTimeout and MaxRetryMax bound what a registry row may ask for. + // + // Both come from DATA, not from this deployment's config, and neither is + // cheap: an attempt holds a goroutine and the inbound connection for its + // whole timeout, and http.Server's write timeout does not cancel the + // request context. So a row reading retryMax 1000, timeoutMs 60000 pins + // both for roughly seventeen hours, and a handful of such requests is the + // adapter. The registry is trusted to say where a provider is; it is not a + // reason to let one row decide how long this process is busy. + // + // Clamped rather than refused. A row that overreaches is a configuration + // mistake, and failing every request for that capability is a worse answer + // than serving it with a sane budget and saying so in the log. + MaxTimeout = 30 * time.Second + MaxRetryMax = 5 +) + +// Auth schemes this step can present upstream. Credentials themselves are never +// configured here or held in the registry -- config names the environment +// variable to read, so a secret reaches the process through its environment and +// nothing else. +const ( + // RetryBackoffBase is the first wait between attempts, doubling from there + // up to RetryBackoffMax. Short, because the retry budget comes from the + // registry and an operator setting 5 retries did not ask for seconds of + // latency -- only for the provider's brief unavailability to be ridden out. + RetryBackoffBase = 50 * time.Millisecond + RetryBackoffMax = 800 * time.Millisecond + + // redactedMarker stands in for a credential in anything logged or returned. + redactedMarker = "REDACTED" + + AuthSchemeNone = "none" + AuthSchemeBasic = "basic" + AuthSchemeHeader = "header" + // AuthSchemeQuery puts the credential in the query string, which some + // upstreams are built around whatever anyone thinks of it. It is the least + // safe of the four -- a query string is logged by proxies and appears in a + // transport error -- so the value is redacted from anything this package + // logs or returns. See redact. + AuthSchemeQuery = "query" + + // AuthSchemeOAuth2 exchanges a client id and secret for a short-lived + // bearer token, and sends that. For an upstream behind an OAuth2 token + // endpoint -- a Keycloak service account, say -- where a static token is not + // an option: the live one this was written against lives ten hours, so a + // value pasted into an environment variable is wrong twice a day. + // + // Only the client_credentials grant. There is no user to redirect and no + // refresh token in that grant, so the other flows would be dead code. + AuthSchemeOAuth2 = "oauth2" +) + +// codeUpstreamUnavailable reports a provider that could not be reached or +// answered with a failure. It is not this adapter's fault and not the caller's. +const codeUpstreamUnavailable = "NET_DOWNSTREAM_UNAVAILABLE" + +// tokenRefreshSkew is how early an oauth2 token stops being trusted. It has to +// exceed the round trip to the provider, so that a request which passes the +// expiry check cannot arrive after the token has actually died. One extra +// exchange per token lifetime is the whole cost. +const tokenRefreshSkew = 60 * time.Second + +// explainLimit is how much of a failed response is quoted. Enough for a +// provider's own message, short enough not to put a page of HTML in a log line +// or a NACK. +const explainLimit = 300 diff --git a/pkg/plugin/implementation/internal/upstream/http.go b/pkg/plugin/implementation/internal/upstream/http.go new file mode 100644 index 00000000..927b20b3 --- /dev/null +++ b/pkg/plugin/implementation/internal/upstream/http.go @@ -0,0 +1,436 @@ +// Talking HTTP to a provider: building the endpoint and body, the attempt and +// its retry budget, and reading what came back. +package upstream + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/beckn-one/beckn-onix/pkg/log" + "github.com/beckn-one/beckn-onix/pkg/model" +) + +// call makes the upstream request described by the plan, retrying within its +// budget. +// budget resolves how long one attempt may take and how many retries follow +// it, applying the registry's values within this deployment's ceilings. +// +// Pure and separate from call so both bounds can be asserted without a server +// that sleeps for the timeout it is testing. +// +// retryMax counts retries, not attempts, so the call itself is always made +// once. An absent retryMax and an explicit 0 are the same instruction. +func budget(call model.ActionPlan) (time.Duration, int) { + timeout := DefaultTimeout + if call.TimeoutMs > 0 { + timeout = time.Duration(call.TimeoutMs) * time.Millisecond + } + if timeout > MaxTimeout { + timeout = MaxTimeout + } + + retries := DefaultRetryMax + if call.RetryMax > 0 { + retries = call.RetryMax + } + if retries > MaxRetryMax { + retries = MaxRetryMax + } + return timeout, retries +} + +func (s *Step) call(ctx context.Context, auth *authenticator, baseURL string, call model.ActionPlan, mapped []byte) ([]byte, error) { + endpoint, err := buildEndpoint(baseURL, call, mapped) + if err != nil { + return nil, err + } + + timeout, retries := budget(call) + if d := time.Duration(call.TimeoutMs) * time.Millisecond; d > timeout { + log.Warnf(ctx, "upstream: registry asks for a %v timeout; using the %v ceiling", d, timeout) + } + if call.RetryMax > retries { + log.Warnf(ctx, "upstream: registry asks for %d retries; using the %d ceiling", + call.RetryMax, retries) + } + attempts := retries + 1 + + var lastErr error + for attempt := 1; attempt <= attempts; attempt++ { + // A caller that has gone away is not worth another attempt, and neither + // is a budget already spent. Checked before the call rather than after, + // so a cancelled request costs nothing. + if err := ctx.Err(); err != nil { + if lastErr == nil { + lastErr = err + } + break + } + + body, err := s.attempt(ctx, auth, call, endpoint, mapped, timeout) + if err == nil { + return body, nil + } + lastErr = s.redact(err) + log.Warnf(ctx, "upstream: attempt %d/%d failed: %v", attempt, attempts, lastErr) + + // Only some failures are worth repeating. A 4xx, a request this step + // could not build and a credential it could not read will fail + // identically however many times they are tried -- and retrying the + // credential case is the worst of them, because it reports an + // operator's missing environment variable as the provider being down. + if isPermanent(err) { + break + } + if attempt < attempts { + if err := sleep(ctx, backoff(attempt)); err != nil { + break + } + } + } + return nil, model.NewCodedErr(http.StatusBadGateway, codeUpstreamUnavailable, + fmt.Errorf("upstream: provider did not answer after %d attempts: %w", attempts, lastErr)) +} + +// permanentErr marks a failure no retry can fix. Kept unexported and detected +// with errors.As, so a caller of this package sees only the underlying error. +type permanentErr struct{ error } + +func (p permanentErr) Unwrap() error { return p.error } + +// doNotRetry marks err as not worth repeating. +func doNotRetry(err error) error { return permanentErr{err} } + +// isPermanent reports whether err is one no further attempt would change. +func isPermanent(err error) bool { + var permanent permanentErr + return errors.As(err, &permanent) +} + +// backoff is how long to wait before the next attempt. +// +// Exponential from a short base and capped, because the provider being briefly +// busy is the case worth waiting out; anything longer is a timeout's job. With +// no wait at all a retryMax of 5 spends its whole budget inside a couple of +// milliseconds, which is not a retry so much as the same failure six times. +func backoff(attempt int) time.Duration { + if attempt <= 1 { + return RetryBackoffBase + } + // Doubled in a loop that stops at the ceiling rather than shifted and then + // clamped. `RetryBackoffBase << (attempt - 1)` overflows int64 once the + // shift reaches 38 at a 50ms base, and the wrapped value is NEGATIVE -- so + // it passes the `> RetryBackoffMax` check, is returned, and a sleep on a + // negative duration returns immediately. The retry loop then spins as fast + // as the provider can refuse. Stopping at the ceiling cannot overflow, + // because it never doubles a value already at or past it. + wait := RetryBackoffBase + for i := 1; i < attempt && wait < RetryBackoffMax; i++ { + wait *= 2 + } + if wait > RetryBackoffMax { + return RetryBackoffMax + } + return wait +} + +// sleep waits, or reports that the context ended first. +func sleep(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +// attempt makes one upstream request. +func (s *Step) attempt(ctx context.Context, auth *authenticator, call model.ActionPlan, endpoint string, mapped []byte, timeout time.Duration) ([]byte, error) { + attemptCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + method := canonicalMethod(call.Method) + req, err := http.NewRequestWithContext(attemptCtx, method, endpoint, requestBody(method, mapped)) + if err != nil { + return nil, doNotRetry(fmt.Errorf("could not build the request: %w", err)) + } + if hasBody(method) { + req.Header.Set("Content-Type", "application/json") + } + if err := s.authenticate(auth, req); err != nil { + // A missing or unreadable credential is configuration, not weather. + return nil, doNotRetry(err) + } + + // The URL as it actually went on the wire, credential removed. At info + // rather than debug because this is the line that answers "what did we ask, + // and what came back" -- the question every provider problem starts with. + requested := s.redactString(req.URL.String()) + + resp, err := s.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(io.LimitReader(resp.Body, s.config.MaxResponseBytes+1)) + if err != nil { + return nil, fmt.Errorf("could not read the response: %w", err) + } + log.Infof(ctx, "upstream: %s %s -> %s, %d bytes", method, requested, resp.Status, len(body)) + if int64(len(body)) > s.config.MaxResponseBytes { + // Asking again will not make the answer smaller. + return nil, doNotRetry(fmt.Errorf("response exceeds the %d byte limit", s.config.MaxResponseBytes)) + } + // Any 2xx, not 200 alone. A provider is entitled to answer 202 for work it + // accepted, 204 for nothing to report, or 201 for something it created, and + // treating those as failures would refuse a perfectly good exchange. 3xx + // does not reach here: the client follows redirects. + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + // The body is logged, not returned. It goes into a 502 that is signed + // and sent to the network caller, and what a provider puts in a failure + // body is its own business -- a stack trace, an internal hostname, a + // database error. The status is the caller's business and stays; the + // body is the operator's, and the log is where the operator looks. + // Redacted on the way to the log too. A provider that rejects a + // request often quotes it back, credential and all -- so the body is + // exactly where a query-string token turns up, and moving it from the + // error to the log would only move the leak. + log.Warnf(ctx, "upstream: provider returned %s for %s %s: %s", + resp.Status, method, requested, s.redactString(explain(body))) + err := fmt.Errorf("provider returned %s", resp.Status) + // 5xx and 429 are the provider asking to be tried again. Every other + // 4xx is a statement about the request, which will not improve. + if resp.StatusCode < http.StatusInternalServerError && resp.StatusCode != http.StatusTooManyRequests { + return nil, doNotRetry(err) + } + return nil, err + } + return body, nil +} + +// explain renders a failed response body for a human. +// +// The body was already read and then thrown away, so a provider's own account +// of what was wrong -- Agmarknet says "no data" in the body of a 400 -- never +// reached anyone. The status alone says a call failed and nothing about why, +// which is the first thing an operator needs and the thing that makes a real +// provider's behaviour observable at all. +func explain(body []byte) string { + text := strings.TrimSpace(string(body)) + if text == "" { + return "(no body)" + } + // Collapse whitespace: a provider that answers with indented JSON or an + // HTML error page should not spread one failure over forty log lines. + text = strings.Join(strings.Fields(text), " ") + if len(text) > explainLimit { + return text[:explainLimit] + "... (truncated)" + } + return text +} + +// buildEndpoint joins the plan's base URL and path, carrying the mapped request +// as query parameters when the method takes no body. +func buildEndpoint(baseURL string, call model.ActionPlan, mapped []byte) (string, error) { + if err := verifyBaseURL(baseURL); err != nil { + return "", err + } + if err := verifyPath(call.Path); err != nil { + return "", err + } + + // baseUrl cannot end in a slash and path must begin with one, so exactly one + // separator appears between them. The trim is belt and braces: the registry + // refuses a trailing slash on baseUrl, and this keeps a row that predates + // that from producing a doubled one. + endpoint := strings.TrimSuffix(baseURL, "/") + call.Path + if hasBody(call.Method) { + return endpoint, nil + } + + query, err := asQuery(mapped) + if err != nil { + return "", err + } + if query == "" { + return endpoint, nil + } + if strings.Contains(endpoint, "?") { + return endpoint + "&" + query, nil + } + return endpoint + "?" + query, nil +} + +// verifyPath refuses a published path nobody could have meant. +// +// The registry constrains this, but it is a separate deployable that may not be +// updated in step, so a row that slipped through has to fail here with something +// an operator can act on rather than as a provider's 404 three hops away. +// +// An empty segment is the case worth catching: "//get-daily" is never +// deliberate, and plenty of servers answer it differently from "/get-daily". A +// trailing slash is deliberately left alone -- "/api/" and "/api" are a +// distinction some APIs genuinely make, so stripping it would silently change +// the URL the operator published. +func verifyPath(path string) error { + if path == "" { + return model.NewBadReqErr("", errors.New("upstream: the registry publishes no path for this action")) + } + if !strings.HasPrefix(path, "/") { + return model.NewBadReqErr("", fmt.Errorf( + "upstream: path %q does not begin with a slash, so it cannot be joined to a base url", path)) + } + if strings.Contains(path, "//") { + return model.NewBadReqErr("", fmt.Errorf( + "upstream: path %q has an empty segment; write it with single slashes", path)) + } + // A dot segment is refused rather than resolved. The registry says which + // path answers an action, and a row that climbs out of it is either a + // mistake or an attempt to reach something the row does not name -- and + // net/url would quietly resolve it either way, so the request that left + // would not be the request the row described. + for _, segment := range strings.Split(path, "/") { + if segment == ".." || segment == "." { + return model.NewBadReqErr("", fmt.Errorf( + "upstream: path %q contains the %q segment; publish the path it resolves to instead", + path, segment)) + } + } + // A fragment is never sent, so a row carrying one describes a request that + // cannot be made. Refused here rather than silently dropped by the + // transport, which would make the row look honoured. + if strings.Contains(path, "#") { + return model.NewBadReqErr("", fmt.Errorf( + "upstream: path %q contains a fragment, which is never sent to a server", path)) + } + return nil +} + +// verifyBaseURL checks the participant's base url before it is joined to a +// path, so a row that cannot produce a request says so as a bad request rather +// than as the provider being unreachable. +// +// Without this, `baseUrl: "registry:8081"` -- a scheme left off -- failed +// inside http.NewRequestWithContext and arrived as a 502 "provider did not +// answer after 1 attempts: could not build the request". That names the +// provider for an error in the row describing it, and it is retried on the way +// there. jsonmapper has always checked its own reference this way; this is the +// same check on the other url the registry publishes. +func verifyBaseURL(baseURL string) error { + if baseURL == "" { + return model.NewBadReqErr("", errors.New("upstream: the registry publishes no base url for this provider")) + } + parsed, err := url.Parse(baseURL) + if err != nil { + return model.NewBadReqErr("", fmt.Errorf("upstream: invalid base url %q: %w", baseURL, err)) + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return model.NewBadReqErr("", fmt.Errorf( + "upstream: base url %q must be http or https", baseURL)) + } + if parsed.Host == "" { + return model.NewBadReqErr("", fmt.Errorf("upstream: base url %q names no host", baseURL)) + } + return nil +} + +// asQuery renders a mapped request as query parameters. +// +// A method with no body still needs the mapping's output somewhere, and the +// query string is the only place it can go. Only scalars are carried: a nested +// value has no single obvious encoding, and inventing one here would put a +// convention in Go that belongs in the mapping. +func asQuery(mapped []byte) (string, error) { + if len(bytes.TrimSpace(mapped)) == 0 { + return "", nil + } + var fields map[string]any + if err := json.Unmarshal(mapped, &fields); err != nil { + return "", fmt.Errorf("upstream: mapped request is not an object, so it cannot become a query: %w", err) + } + + values := url.Values{} + for name, value := range fields { + rendered, ok := renderScalar(value) + if !ok { + return "", fmt.Errorf("upstream: mapped field %q is not a scalar and cannot become a query parameter", name) + } + values.Set(name, rendered) + } + return values.Encode(), nil +} + +// renderScalar renders a JSON scalar as a query parameter value. +func renderScalar(value any) (string, bool) { + switch typed := value.(type) { + case string: + return typed, true + case bool: + return strconv.FormatBool(typed), true + case float64: + // 'g' with -1 precision round-trips without inventing trailing zeros, so + // 19.9975 stays 19.9975 rather than becoming 19.997500. + return strconv.FormatFloat(typed, 'g', -1, 64), true + default: + return "", false + } +} + +// requestBody returns the body to send, which is none for methods that take none. +func requestBody(method string, mapped []byte) io.Reader { + if !hasBody(method) { + return nil + } + return bytes.NewReader(mapped) +} + +// hasBody reports whether a method carries a request body. +func hasBody(method string) bool { + switch canonicalMethod(method) { + case http.MethodGet, http.MethodHead, http.MethodDelete, "": + return false + default: + return true + } +} + +// canonicalMethod returns a known HTTP method in the spelling the RFC gives +// it, and anything else unchanged. +// +// hasBody used to upper-case privately, which made the method look +// case-insensitive when it is not: NewRequestWithContext transmits it verbatim, +// so a registry row reading `method: "post"` sent `post /path HTTP/1.1`. The +// body was attached correctly -- hasBody had normalised -- but nginx and most +// gateways answer 405 to a lowercase method, which classifies permanent and +// surfaces as a 502 "provider did not answer". A row that is right in every +// respect but its capitalisation is a bad way to spend an afternoon. +// +// Only known methods are rewritten. Upper-casing everything would be a new +// restriction on an upstream entitled to a method this list has not heard of, +// and net/http already refuses one that is not a valid token. +// +// An empty method is left empty: net/http documents "" as GET and substitutes +// it, and hasBody agrees that it carries no body, so the two are already +// consistent and inventing a value here would only hide where it comes from. +func canonicalMethod(method string) string { + upper := strings.ToUpper(method) + switch upper { + case http.MethodGet, http.MethodHead, http.MethodPost, http.MethodPut, + http.MethodPatch, http.MethodDelete, http.MethodConnect, + http.MethodOptions, http.MethodTrace: + return upper + } + return method +} diff --git a/pkg/plugin/implementation/internal/upstream/redact.go b/pkg/plugin/implementation/internal/upstream/redact.go new file mode 100644 index 00000000..85cda7b4 --- /dev/null +++ b/pkg/plugin/implementation/internal/upstream/redact.go @@ -0,0 +1,196 @@ +// Keeping credentials out of anything this package logs or returns. +package upstream + +import ( + "encoding/base64" + "net/url" + "os" + "sort" + "strings" +) + +// redact removes a query-string credential from an error's text. +// +// Go's transport errors quote the whole URL -- `Get "http://host/p?token=..." +// dial tcp: ...` -- so without this, one unreachable host writes the credential +// into the log at warn level. Nothing else in this package puts a URL in a +// message, which is why this is the only place it is needed. +// +// A plain string replacement, because the value is what leaks and the value is +// what we hold. Parsing the error to find it would assume a shape net/http does +// not promise. +func (s *Step) redact(err error) error { + if err == nil { + return nil + } + text := s.redactString(err.Error()) + if text == err.Error() { + return err + } + return redactedErr{text: text, err: err} +} + +// redactedErr reports a redacted message while keeping the original reachable +// for errors.Is and errors.As. +// +// errors.New(text) was the obvious thing and it broke the chain: the redacted +// value is what gets %w-wrapped into the final 502, so under a query-string +// scheme -- and only then, since nothing else redacts -- errors.Is(err, +// context.DeadlineExceeded) silently stopped matching. Retry classification +// was never affected, because isPermanent tests the error before redaction, +// which is why nothing failed visibly. +// +// fmt.Errorf("%s: %w", text, err) would have restored the chain and undone the +// redaction with it: %w formats the original, credential included. Reporting +// the redacted text from Error() and the original from Unwrap() keeps both. +// +// The original's text is reachable through errors.Unwrap, which is a +// deliberate act by a caller who wants the cause -- and %v, %s and %w on the +// value itself all go through Error() and stay redacted. +type redactedErr struct { + text string + err error +} + +func (e redactedErr) Error() string { return e.text } + +func (e redactedErr) Unwrap() error { return e.err } + +// redactString removes the configured credential from any text about to be +// logged or returned -- an error, a provider's response body, or the URL that +// was requested. +// +// Logging those is deliberate: they say what was asked of whom and what came +// back, which is the first thing anyone wants when a provider misbehaves. This +// is what makes that safe to do at info and warn level. +// +// EVERY scheme, not just query. This used to return early unless the scheme was +// query, on the reasoning that only a query credential reaches a URL -- true of +// the URL, and wrong about the body. A provider quoting the request it rejected +// is the ordinary shape of a 401 or 403 body, an API gateway echoing the +// Authorization header is routine, and a wrong-credential 4xx is not retried, +// so it lands in the log once per request for as long as the credential is +// wrong. basic is the scheme the reference config ships. +func (s *Step) redactString(text string) string { + for _, secret := range s.secretForms() { + text = strings.ReplaceAll(text, secret, redactedMarker) + } + return text +} + +// secretForms returns every form the configured credential can appear in, +// longest first so a value that contains another is replaced before its +// substring turns the longer one into a partial redaction. +// +// Per scheme, because the schemes leak differently and redacting the value we +// hold is not enough on its own: +// +// - basic wraps the pair: SetBasicAuth sends base64(user:pass), so the +// password alone does not appear on the wire and replacing it misses the +// echoed header entirely. +// - query escapes: authenticate goes through url.Values.Encode, so a base64 +// token carrying "+", "/" or "=" appears as "a%2Bb%2Fc%3D". Escaping what +// we hold is exact -- same function Encode used, so the two agree by +// construction rather than by a guess about which characters matter. +// - header sends the value as-is. +// +// The raw form is kept alongside the wrapped one in both cases: an error built +// from the config rather than from the request still quotes the credential +// unwrapped. +func (s *Step) secretForms() []string { + // EVERY profile, not the one being served. + // + // Redaction is about what could appear in a piece of text, not about which + // provider a request happened to be for. An error or a log line built while + // serving one provider can quote another's credential -- a shared client + // echoing a header, an issuer naming the wrong caller -- and scoping this + // to the active profile would let that reach the log unredacted. There are + // a handful of profiles and this runs on a failure path, so walking all of + // them costs nothing worth measuring. + var forms []string + for _, auth := range s.auth { + forms = append(forms, auth.secretForms()...) + } + // Sorted across the merged set rather than per profile: one provider's + // short token can be a substring of another's, and replacing the short one + // first would leave the longer half-redacted. + return longestFirst(forms) +} + +// secretForms returns every form this profile's credential can appear in. +// +// Per scheme, because the schemes leak differently and redacting the value we +// hold is not enough on its own: +// +// - basic wraps the pair: SetBasicAuth sends base64(user:pass), so the +// password alone does not appear on the wire and replacing it misses the +// echoed header entirely. +// - query escapes: authenticate goes through url.Values.Encode, so a base64 +// token carrying "+", "/" or "=" appears as "a%2Bb%2Fc%3D". Escaping what +// we hold is exact -- same function Encode used, so the two agree by +// construction rather than by a guess about which characters matter. +// - header sends the value as-is. +// +// The raw form is kept alongside the wrapped one in both cases: an error built +// from the config rather than from the request still quotes the credential +// unwrapped. +func (a *authenticator) secretForms() []string { + switch a.cfg.Scheme { + case AuthSchemeBasic: + username, password := os.Getenv(a.cfg.UsernameEnv), os.Getenv(a.cfg.PasswordEnv) + if password == "" { + return nil + } + forms := []string{password} + if username != "" { + // The wire form, which is what a gateway echoes back. + forms = append(forms, + base64.StdEncoding.EncodeToString([]byte(username+":"+password))) + } + // The username is deliberately NOT redacted. It identifies rather than + // authenticates, and it is routinely a short common word -- redacting + // "user" or "admin" would eat unrelated text and cost the operator the + // log line they came for. The pair and the password are the secrets. + return forms + case AuthSchemeHeader: + value := os.Getenv(a.cfg.HeaderValueEnv) + if value == "" { + return nil + } + return []string{value} + case AuthSchemeOAuth2: + // Both halves: the client secret we send to the issuer, and the token + // it gave back. The token is the one that reaches the provider, so it + // is the one an echoing 401 body quotes. + var forms []string + if secret := os.Getenv(a.cfg.ClientSecretEnv); secret != "" { + forms = append(forms, secret) + } + // Read without tokenMu: this is reached from inside the exchange, which + // holds it. + if held := a.token.Load(); held != nil && held.value != "" { + forms = append(forms, held.value) + } + // The client id is deliberately NOT redacted: it identifies, it does + // not authenticate, and it is what makes a log line useful. + return forms + case AuthSchemeQuery: + value := os.Getenv(a.cfg.QueryValueEnv) + if value == "" { + return nil + } + forms := []string{value} + if encoded := url.QueryEscape(value); encoded != value { + forms = append(forms, encoded) + } + return forms + } + return nil +} + +// longestFirst orders replacement candidates so a longer form is substituted +// before any shorter one it contains. +func longestFirst(forms []string) []string { + sort.Slice(forms, func(i, j int) bool { return len(forms[i]) > len(forms[j]) }) + return forms +} diff --git a/pkg/plugin/implementation/internal/upstream/serve.go b/pkg/plugin/implementation/internal/upstream/serve.go new file mode 100644 index 00000000..305e5f35 --- /dev/null +++ b/pkg/plugin/implementation/internal/upstream/serve.go @@ -0,0 +1,219 @@ +// The request pipeline: recognise the capability, resolve its call plan, +// translate out, call, translate back. +package upstream + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + + "github.com/beckn-one/beckn-onix/pkg/log" + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/capabilitybinding" +) + +// Run serves the request when it is for this step's capability, and does +// nothing when it is not. +// +// Doing nothing is the dispatch mechanism: several provider steps sit in one +// pipeline and each recognises its own work, so adding a provider is one more +// entry rather than a change to a routing table. +func (s *Step) Run(ctx *model.StepContext) error { + binding, err := capabilitybinding.From(s.paths, ctx.Body) + if errors.Is(err, capabilitybinding.ErrNoBinding) { + return nil + } + if err != nil { + // Everything From refuses is a statement about the payload: unreadable + // JSON, or a request naming more than one call. Unclassified it becomes + // a 500, which says this adapter broke and leaves the reason in a log + // the caller cannot read. + return model.NewBadReqErr("", err) + } + if !s.serves(binding.Key()) { + log.Debugf(ctx, "upstream: %s is not one of this step's capabilities, passing through", binding.Key()) + return nil + } + + plan, err := s.registry.ProviderRecord(ctx, binding.Key()) + if err != nil { + // A definite "no such binding" is the caller naming something that is + // not there, so 404 -- the same reasoning the no-route path uses to + // refuse an unrecognised capability rather than ACK it. A registry that + // could not be consulted is different and stays a 500: unclassified, + // because it is this adapter that failed. + if errors.Is(err, definition.ErrProviderRecordNotFound) { + // %w, not %v: the sentinel has to stay unwrappable, or anything + // upstream testing errors.Is against it silently stops matching. + return model.NewNotFoundErr("", fmt.Errorf( + "upstream: the registry publishes no active binding for %s: %w", binding.Key(), err)) + } + return fmt.Errorf("upstream: no call plan for %s: %w", binding.Key(), err) + } + + return s.serve(ctx, plan) +} + +// resolve runs whatever prerequisite work this capability needs, and returns the +// values for the mapping to read under _local. +// +// Empty rather than nil when there is nothing: a mapping referring to _local on +// a capability that resolves nothing should read a missing field, not fail. +func (s *Step) resolve(ctx context.Context, bindingKey string, beckn any) (map[string]any, error) { + prerequisite, needed := s.prerequisites[bindingKey] + if !needed { + return map[string]any{}, nil + } + local, err := prerequisite(ctx, beckn) + if err != nil { + return nil, fmt.Errorf("upstream: %s could not resolve what it needs before the call: %w", bindingKey, err) + } + if local == nil { + return map[string]any{}, nil + } + return local, nil +} + +// serves reports whether a binding key is one this step answers to. +// +// A slice rather than a set: a step serves a handful of capabilities at most, so +// the scan costs less than the map would, and the config order is preserved in +// the log line above. +func (s *Step) serves(key string) bool { + return slices.Contains(s.config.BindingKeys, key) +} + +// serve runs the exchange this step exists for: resolve, map out, call, map back. +func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { + action := extractAction(ctx.Body) + call, served := plan.Actions[action] + if !served { + // The capability publishes no endpoint for this action, so it does not + // serve it. Refused here rather than after a call to whichever endpoint + // happened to be on the record -- naming what it does serve turns a + // registry mistake into a one-line fix. + return model.NewBadReqErr("", fmt.Errorf( + "upstream: %s does not serve action %q; it serves %s", + plan.BindingKey, action, strings.Join(plan.ServedActions(), ", "))) + } + + beckn, err := decodeBody(ctx.Body) + if err != nil { + return err + } + + // What this provider requires of a payload is declared by its mapping, not + // by this step. A capability with a different rule is a different mapping + // file rather than a different build -- and the rule sits beside the + // extraction it guards. + if err := s.mapper.Verify(ctx, call.Mappings, map[string]any{"beckn": beckn}); err != nil { + return err + } + + // Whatever this capability needs that its payload does not carry. Empty for + // most: the mapping reads the payload directly and needs nothing resolved. + local, err := s.resolve(ctx, plan.BindingKey, beckn) + if err != nil { + return err + } + + upstreamRequest, err := s.buildRequest(ctx, call, beckn, local) + if err != nil { + return err + } + + // Which credential this provider takes. Resolved from the binding key, so + // one step serving several providers authenticates each as its own. + // Startup guarantees a profile per served provider; this guards the case + // where a record arrives for a key the config never declared. + auth, configured := s.auth[providerIDFrom(plan.BindingKey)] + if !configured { + return fmt.Errorf("upstream: no credential is configured for %s", plan.BindingKey) + } + + upstreamResponse, err := s.call(ctx, auth, plan.BaseURL, call, upstreamRequest) + if err != nil { + return err + } + + answer, err := decodeBody(upstreamResponse) + if err != nil { + return fmt.Errorf("upstream: provider answered with something that is not JSON: %w", err) + } + + // The same mapping reference as the request, other half: one file carries + // both directions for this action. + // + // The mapping is handed what each party sent, plus whatever prerequisites + // resolved, under _local. Empty when there are none. + becknResponse, err := s.mapper.Transform(ctx, call.Mappings, definition.DirectionResponse, map[string]any{ + "beckn": beckn, + "_local": local, + "response": answer, + }) + if err != nil { + return err + } + if len(becknResponse) == 0 { + // Either the file has no response half, or its transform matched nothing + // in this answer. Both leave no Beckn response to return, and returning + // the provider's own shape instead would be worse than failing. The + // message says what was observed rather than guessing which it was. + return fmt.Errorf("upstream: the response half of %s produced nothing, so %s cannot be answered", + call.Mappings, plan.BindingKey) + } + + ctx.ResponseBody = becknResponse + log.Infof(ctx, "upstream: served %s in %d bytes", plan.BindingKey, len(becknResponse)) + return nil +} + +// buildRequest produces what the provider is sent. +// +// Whatever the mapping produces IS the request: a body for a method that takes +// one, query parameters for a method that does not. Nothing is substituted when +// it produces nothing, so an empty request half means an empty request. +// +// This step used to extract a point from the payload and fall back to sending +// that. It meant the choice of which payload fields reach the provider lived in +// Go, so adding a parameter -- a date range, say -- was a rebuild. Now it is a +// mapping edit and nothing else. +func (s *Step) buildRequest(ctx context.Context, call model.ActionPlan, beckn any, local map[string]any) ([]byte, error) { + mapped, err := s.mapper.Transform(ctx, call.Mappings, definition.DirectionRequest, map[string]any{ + "beckn": beckn, + "_local": local, + }) + if err != nil { + return nil, err + } + if len(mapped) == 0 { + log.Debugf(ctx, "upstream: the request half of %s produced nothing; sending an empty request", call.Mappings) + } + return mapped, nil +} + +// extractAction reads the Beckn action a request is for. +func extractAction(body []byte) string { + var payload struct { + Context struct { + Action string `json:"action"` + } `json:"context"` + } + if err := json.Unmarshal(body, &payload); err != nil { + return "" + } + return payload.Context.Action +} + +// decodeBody turns raw JSON into the generic value a mapping reads. +func decodeBody(body []byte) (any, error) { + var decoded any + if err := json.Unmarshal(body, &decoded); err != nil { + return nil, fmt.Errorf("upstream: could not read JSON: %w", err) + } + return decoded, nil +} diff --git a/pkg/plugin/implementation/internal/upstream/token.go b/pkg/plugin/implementation/internal/upstream/token.go new file mode 100644 index 00000000..624dd813 --- /dev/null +++ b/pkg/plugin/implementation/internal/upstream/token.go @@ -0,0 +1,163 @@ +// The oauth2 client_credentials exchange and the token each provider holds. +package upstream + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" + + "github.com/beckn-one/beckn-onix/pkg/log" + "github.com/beckn-one/beckn-onix/pkg/model" +) + +// bearerToken returns a token to send, exchanging one if what we hold has +// expired or if we hold none. +// +// The lock spans the fetch on purpose. Without it a cold start sends every +// concurrent request to the token endpoint, and the provider's issuer sees a +// burst of identical exchanges. Serialising them costs one wait per token +// lifetime and nothing after that, which is a better trade than a second +// caching layer. +// +// NOTHING IS CACHED ON FAILURE. A transient outage at the token endpoint must +// not leave this step holding a failure for the lifetime it never obtained -- +// the next request tries again. +func (s *Step) bearerToken(ctx context.Context, auth *authenticator) (string, error) { + if held := auth.token.Load(); held != nil && time.Now().Before(held.expiry) { + return held.value, nil + } + + auth.tokenMu.Lock() + defer auth.tokenMu.Unlock() + + // Re-checked after acquiring: while this caller waited, whoever held the + // mutex may already have exchanged a fresh token. + if held := auth.token.Load(); held != nil && time.Now().Before(held.expiry) { + return held.value, nil + } + + token, lifetime, err := s.exchangeToken(ctx, auth) + if err != nil { + return "", err + } + auth.token.Store(&cachedToken{value: token, expiry: time.Now().Add(lifetime)}) + return token, nil +} + +// cachedToken is a token and the moment it stops being trusted. +type cachedToken struct { + value string + expiry time.Time +} + +// tokenResponse is the part of an OAuth2 token response this step reads. +type tokenResponse struct { + AccessToken string `json:"access_token"` + ExpiresIn int `json:"expires_in"` +} + +// exchangeToken performs the client_credentials grant. +// +// Every failure here is the UPSTREAM exchange failing, so all of them carry +// 502. Unclassified they would surface as a 500, which tells a network peer +// this adapter broke when in fact the provider's issuer did. +func (s *Step) exchangeToken(ctx context.Context, auth *authenticator) (string, time.Duration, error) { + cfg := auth.cfg + clientID, clientSecret := os.Getenv(cfg.ClientIDEnv), os.Getenv(cfg.ClientSecretEnv) + if clientID == "" || clientSecret == "" { + return "", 0, s.missingCredential(ctx, cfg.Provider, "oauth2", + cfg.ClientIDEnv+" and "+cfg.ClientSecretEnv) + } + + form := url.Values{ + "grant_type": {"client_credentials"}, + "client_id": {clientID}, + "client_secret": {clientSecret}, + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, cfg.TokenURL, + strings.NewReader(form.Encode())) + if err != nil { + return "", 0, s.tokenErr(fmt.Errorf("token request could not be built: %w", err)) + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + resp, err := s.httpClient.Do(req) + if err != nil { + return "", 0, s.tokenErr(fmt.Errorf("token endpoint %s could not be reached: %w", + cfg.TokenURL, err)) + } + defer resp.Body.Close() + + // Bounded like any other upstream read: a token response is small, and an + // unbounded read here would be a hole in the same wall. + body, err := io.ReadAll(io.LimitReader(resp.Body, s.config.MaxResponseBytes+1)) + if err != nil { + return "", 0, s.tokenErr(fmt.Errorf("token response could not be read: %w", err)) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + // The status, not the body: what an issuer puts in a failure body is its + // own business, and it routinely quotes the request back. + log.Warnf(ctx, "upstream: token endpoint %s returned %s: %s", + cfg.TokenURL, resp.Status, s.redactString(explain(body))) + return "", 0, s.tokenErr(fmt.Errorf("token endpoint returned %s", resp.Status)) + } + + var parsed tokenResponse + if err := json.Unmarshal(body, &parsed); err != nil { + return "", 0, s.tokenErr(fmt.Errorf("token response from %s is not JSON: %w", + cfg.TokenURL, err)) + } + if parsed.AccessToken == "" { + return "", 0, s.tokenErr(fmt.Errorf("token response from %s carries no access_token", + cfg.TokenURL)) + } + lifetime, ok := tokenLifetime(parsed.ExpiresIn, tokenRefreshSkew) + if !ok { + return "", 0, s.tokenErr(fmt.Errorf( + "token response from %s carries no usable expires_in, so its lifetime is unknown", + cfg.TokenURL)) + } + return parsed.AccessToken, lifetime, nil +} + +// tokenErr classifies a failed exchange. Always 502: the exchange is with the +// provider's issuer, so its failure is upstream's, never the caller's. +func (s *Step) tokenErr(err error) error { + return model.NewCodedErr(http.StatusBadGateway, codeUpstreamUnavailable, + fmt.Errorf("upstream: oauth2 token exchange failed: %w", err)) +} + +// tokenLifetime turns a token response's expires_in into how long we may hold +// that token, or reports that we may not hold it at all. +// +// The issuer owns this number, so it is read from the response and never from +// our config: a configured copy is a second version of the same fact, and it +// is wrong the moment a realm's token lifetime is retuned -- silently, with +// every request 401ing until someone edits a file. +// +// skew is subtracted so a request that passes the check cannot arrive at the +// provider after expiry. It has to exceed the round trip, and costs one extra +// refresh per token lifetime. +// +// Two answers rather than one duration, because "we were not told" is not a +// lifetime. A response with no expires_in is refused rather than cached for a +// guessed period or re-fetched on every single request. +func tokenLifetime(expiresIn int, skew time.Duration) (time.Duration, bool) { + if expiresIn <= 0 { + return 0, false + } + lifetime := time.Duration(expiresIn) * time.Second + // A lifetime at or under the skew would put the expiry in the past, and + // then every request fetches a fresh token. Half is still early enough to + // refresh before the real expiry. + if lifetime <= skew { + return lifetime / 2, true + } + return lifetime - skew, true +} diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/upstream/upstream.go index 9844b36b..17a678e1 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/upstream/upstream.go @@ -14,105 +14,17 @@ package upstream import ( - "bytes" "context" - "encoding/base64" - "encoding/json" "errors" "fmt" - "io" "net/http" - "net/url" - "os" - "slices" - "sort" - "strconv" "strings" - "sync" - "sync/atomic" - "time" "github.com/beckn-one/beckn-onix/pkg/log" - "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/capabilitybinding" ) -// Defaults applied when the registry or the operator leaves a setting out. -const ( - // DefaultTimeout and DefaultRetryMax are the registry contract's defaults - // for an action that leaves timeoutMs or retryMax out. Zero retries is - // deliberate: a provider that failed is retried only where the operator - // said so, because a retry on a non-idempotent action is a second booking. - DefaultTimeout = 15 * time.Second - DefaultRetryMax = 0 - // DefaultMaxResponseBytes caps what is read from the provider. The response - // is mapped in memory, so an unbounded one is an unbounded allocation. - DefaultMaxResponseBytes = 4 << 20 // 4 MiB - - // MaxTimeout and MaxRetryMax bound what a registry row may ask for. - // - // Both come from DATA, not from this deployment's config, and neither is - // cheap: an attempt holds a goroutine and the inbound connection for its - // whole timeout, and http.Server's write timeout does not cancel the - // request context. So a row reading retryMax 1000, timeoutMs 60000 pins - // both for roughly seventeen hours, and a handful of such requests is the - // adapter. The registry is trusted to say where a provider is; it is not a - // reason to let one row decide how long this process is busy. - // - // Clamped rather than refused. A row that overreaches is a configuration - // mistake, and failing every request for that capability is a worse answer - // than serving it with a sane budget and saying so in the log. - MaxTimeout = 30 * time.Second - MaxRetryMax = 5 -) - -// Auth schemes this step can present upstream. Credentials themselves are never -// configured here or held in the registry -- config names the environment -// variable to read, so a secret reaches the process through its environment and -// nothing else. -const ( - // RetryBackoffBase is the first wait between attempts, doubling from there - // up to RetryBackoffMax. Short, because the retry budget comes from the - // registry and an operator setting 5 retries did not ask for seconds of - // latency -- only for the provider's brief unavailability to be ridden out. - RetryBackoffBase = 50 * time.Millisecond - RetryBackoffMax = 800 * time.Millisecond - - // redactedMarker stands in for a credential in anything logged or returned. - redactedMarker = "REDACTED" - - AuthSchemeNone = "none" - AuthSchemeBasic = "basic" - AuthSchemeHeader = "header" - // AuthSchemeQuery puts the credential in the query string, which some - // upstreams are built around whatever anyone thinks of it. It is the least - // safe of the four -- a query string is logged by proxies and appears in a - // transport error -- so the value is redacted from anything this package - // logs or returns. See redact. - AuthSchemeQuery = "query" - - // AuthSchemeOAuth2 exchanges a client id and secret for a short-lived - // bearer token, and sends that. For an upstream behind an OAuth2 token - // endpoint -- a Keycloak service account, say -- where a static token is not - // an option: the live one this was written against lives ten hours, so a - // value pasted into an environment variable is wrong twice a day. - // - // Only the client_credentials grant. There is no user to redirect and no - // refresh token in that grant, so the other flows would be dead code. - AuthSchemeOAuth2 = "oauth2" -) - -// codeUpstreamUnavailable reports a provider that could not be reached or -// answered with a failure. It is not this adapter's fault and not the caller's. -const codeUpstreamUnavailable = "NET_DOWNSTREAM_UNAVAILABLE" - -// tokenRefreshSkew is how early an oauth2 token stops being trusted. It has to -// exceed the round trip to the provider, so that a request which passes the -// expiry check cannot arrive after the token has actually died. One extra -// exchange per token lifetime is the whole cost. -const tokenRefreshSkew = 60 * time.Second - // Prerequisites are the values a capability needs that its payload does not // carry, keyed by binding key. // @@ -177,68 +89,6 @@ type Config struct { MaxResponseBytes int64 `yaml:"maxResponseBytes" json:"maxResponseBytes"` } -// AuthProfile is how ONE provider's credentials are presented upstream. -// -// One per provider rather than one per step: a step serves several binding -// keys, and the providers behind them need not authenticate alike. -type AuthProfile struct { - // Provider is the participant id this profile belongs to. Held so an error - // can name it: with several profiles on one step, "authScheme query - // requires queryName" would otherwise leave an operator guessing which - // provider it meant. - Provider string - - // Scheme is one of none, basic, header, query or oauth2. Providers differ - // here, which is why it is configuration and not an assumption. - Scheme string - - // UsernameEnv and PasswordEnv name the environment variables holding basic - // credentials. They are variable NAMES, never the values. - UsernameEnv string - PasswordEnv string - - // HeaderName and HeaderValueEnv configure the header scheme: which header - // to set, and which environment variable holds its value. - HeaderName string - HeaderValueEnv string - - // QueryName and QueryValueEnv configure the query scheme: the parameter - // name to add, and the environment variable holding its value. Named the - // same way as the header pair, for the same reason -- the credential is - // never in this config, only the name of the variable carrying it. - QueryName string - QueryValueEnv string - - // TokenURL is the OAuth2 token endpoint. Not a credential, so it is named - // here rather than through an environment variable -- but it IS - // deployment-specific, so the reference config carries a placeholder. - TokenURL string - // ClientIDEnv and ClientSecretEnv name the variables holding the client - // credentials. The values never appear in config, the registry, or a log. - ClientIDEnv string - ClientSecretEnv string -} - -// authenticator is one provider's profile plus the token it holds. -// -// The token cache lives HERE rather than on the Step, and that is the whole -// reason this type exists. A step-wide cache shared between two oauth2 -// providers would hand the first provider's token to the second, which is -// authenticating as somebody else -- a failure no test of a single provider -// can see. -type authenticator struct { - cfg AuthProfile - - // Two mechanisms because there are two jobs. tokenMu serialises the - // EXCHANGE, so a cold start sends one request to the issuer rather than one - // per concurrent caller. token is atomic so READERS never take that mutex, - // which matters because secretForms is one of them and it is reached from - // inside the exchange -- guarding the value with tokenMu instead deadlocked - // on the first failing exchange, which is how this was found. - tokenMu sync.Mutex - token atomic.Pointer[cachedToken] -} - // Step serves whatever capabilities a domain package configures it for. It is // safe for concurrent use. type Step struct { @@ -373,1160 +223,3 @@ func applyDefaults(cfg *Config) error { } return nil } - -// validate refuses a profile whose scheme and fields disagree. Every message -// names the provider: with several profiles on one step, the field alone would -// leave an operator guessing which block to look at. -func (a *AuthProfile) validate() error { - switch a.Scheme { - case AuthSchemeNone: - case AuthSchemeBasic: - if a.UsernameEnv == "" || a.PasswordEnv == "" { - return fmt.Errorf( - "upstream: %s: authScheme basic requires usernameEnv and passwordEnv", a.Provider) - } - case AuthSchemeHeader: - if a.HeaderName == "" || a.HeaderValueEnv == "" { - return fmt.Errorf( - "upstream: %s: authScheme header requires headerName and headerValueEnv", a.Provider) - } - case AuthSchemeQuery: - if a.QueryName == "" || a.QueryValueEnv == "" { - return fmt.Errorf( - "upstream: %s: authScheme query requires queryName and queryValueEnv", a.Provider) - } - case AuthSchemeOAuth2: - if a.TokenURL == "" || a.ClientIDEnv == "" || a.ClientSecretEnv == "" { - return fmt.Errorf( - "upstream: %s: authScheme oauth2 requires tokenUrl, clientIdEnv and clientSecretEnv", - a.Provider) - } - case "": - return fmt.Errorf("upstream: %s: authScheme is required, "+ - "and is none where the upstream needs no credential", a.Provider) - default: - return fmt.Errorf( - "upstream: %s: unknown authScheme %q: must be none, basic, header, query or oauth2", - a.Provider, a.Scheme) - } - return nil -} - -// ParseProviderAuth builds one credential profile per provider from a plugin's -// flattened settings. -// -// What an operator writes is a block per provider: -// -// knowledge-provider: -// authScheme: oauth2 -// tokenUrl: https://issuer.example/token -// -// which pkg/plugin flattens to authScheme-knowledge-provider and -// tokenUrl-knowledge-provider before any plugin sees it. This reads that form -// back into profiles. -// -// Shared rather than repeated in each capability plugin: all of them copied -// the same field list out of the same map, so a scheme added in one place had -// to be remembered in three. -// -// The split is on the FIRST dash, which is unambiguous because no setting name -// contains one while a participant id routinely does -- knowledge-provider, -// provider.oan.dev. So the field is always the part before it. -func ParseProviderAuth(config map[string]string) (map[string]*AuthProfile, error) { - profiles := map[string]*AuthProfile{} - - // Sorted so a config with two mistakes reports the same one every run. - keys := make([]string, 0, len(config)) - for key := range config { - keys = append(keys, key) - } - sort.Strings(keys) - - for _, key := range keys { - field, provider, dashed := strings.Cut(key, "-") - if !dashed { - // A bare auth field is the old step-wide form. Refused rather than - // ignored: silently dropping it leaves every provider on no - // credential at all, which reads as the provider rejecting us. - if authFields[key] { - return nil, fmt.Errorf( - "upstream: %q is set for the whole step; auth is per provider now, "+ - "so it belongs in a block named for the participant id", key) - } - continue - } - if !authFields[field] { - // Not an auth setting, and nothing else on a provider step carries - // a dash -- so this is a misspelled field rather than something to - // pass through. - return nil, fmt.Errorf("upstream: %q is not a credential setting", key) - } - if strings.TrimSpace(provider) == "" { - return nil, fmt.Errorf("upstream: %q names no provider after the dash", key) - } - - profile, seen := profiles[provider] - if !seen { - profile = &AuthProfile{Provider: provider} - profiles[provider] = profile - } - value := config[key] - switch field { - case "authScheme": - profile.Scheme = value - case "usernameEnv": - profile.UsernameEnv = value - case "passwordEnv": - profile.PasswordEnv = value - case "headerName": - profile.HeaderName = value - case "headerValueEnv": - profile.HeaderValueEnv = value - case "queryName": - profile.QueryName = value - case "queryValueEnv": - profile.QueryValueEnv = value - case "tokenUrl": - profile.TokenURL = value - case "clientIdEnv": - profile.ClientIDEnv = value - case "clientSecretEnv": - profile.ClientSecretEnv = value - } - } - return profiles, nil -} - -// authFields is the closed set of per-provider credential settings. Closed on -// purpose: it is what makes the split on the first dash decidable, and what -// turns a misspelled field into a startup error rather than a setting that -// quietly does nothing. -var authFields = map[string]bool{ - "authScheme": true, - "usernameEnv": true, - "passwordEnv": true, - "headerName": true, - "headerValueEnv": true, - "queryName": true, - "queryValueEnv": true, - "tokenUrl": true, - "clientIdEnv": true, - "clientSecretEnv": true, -} - -// providerIDFrom returns the provider half of a binding key. The format is -// "|", and a participant id carries dashes and -// dots but never a pipe, so the first one separates them. -func providerIDFrom(bindingKey string) string { - provider, _, _ := strings.Cut(bindingKey, "|") - return strings.TrimSpace(provider) -} - -// Run serves the request when it is for this step's capability, and does -// nothing when it is not. -// -// Doing nothing is the dispatch mechanism: several provider steps sit in one -// pipeline and each recognises its own work, so adding a provider is one more -// entry rather than a change to a routing table. -func (s *Step) Run(ctx *model.StepContext) error { - binding, err := capabilitybinding.From(s.paths, ctx.Body) - if errors.Is(err, capabilitybinding.ErrNoBinding) { - return nil - } - if err != nil { - // Everything From refuses is a statement about the payload: unreadable - // JSON, or a request naming more than one call. Unclassified it becomes - // a 500, which says this adapter broke and leaves the reason in a log - // the caller cannot read. - return model.NewBadReqErr("", err) - } - if !s.serves(binding.Key()) { - log.Debugf(ctx, "upstream: %s is not one of this step's capabilities, passing through", binding.Key()) - return nil - } - - plan, err := s.registry.ProviderRecord(ctx, binding.Key()) - if err != nil { - // A definite "no such binding" is the caller naming something that is - // not there, so 404 -- the same reasoning the no-route path uses to - // refuse an unrecognised capability rather than ACK it. A registry that - // could not be consulted is different and stays a 500: unclassified, - // because it is this adapter that failed. - if errors.Is(err, definition.ErrProviderRecordNotFound) { - // %w, not %v: the sentinel has to stay unwrappable, or anything - // upstream testing errors.Is against it silently stops matching. - return model.NewNotFoundErr("", fmt.Errorf( - "upstream: the registry publishes no active binding for %s: %w", binding.Key(), err)) - } - return fmt.Errorf("upstream: no call plan for %s: %w", binding.Key(), err) - } - - return s.serve(ctx, plan) -} - -// resolve runs whatever prerequisite work this capability needs, and returns the -// values for the mapping to read under _local. -// -// Empty rather than nil when there is nothing: a mapping referring to _local on -// a capability that resolves nothing should read a missing field, not fail. -func (s *Step) resolve(ctx context.Context, bindingKey string, beckn any) (map[string]any, error) { - prerequisite, needed := s.prerequisites[bindingKey] - if !needed { - return map[string]any{}, nil - } - local, err := prerequisite(ctx, beckn) - if err != nil { - return nil, fmt.Errorf("upstream: %s could not resolve what it needs before the call: %w", bindingKey, err) - } - if local == nil { - return map[string]any{}, nil - } - return local, nil -} - -// serves reports whether a binding key is one this step answers to. -// -// A slice rather than a set: a step serves a handful of capabilities at most, so -// the scan costs less than the map would, and the config order is preserved in -// the log line above. -func (s *Step) serves(key string) bool { - return slices.Contains(s.config.BindingKeys, key) -} - -// serve runs the exchange this step exists for: resolve, map out, call, map back. -func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { - action := extractAction(ctx.Body) - call, served := plan.Actions[action] - if !served { - // The capability publishes no endpoint for this action, so it does not - // serve it. Refused here rather than after a call to whichever endpoint - // happened to be on the record -- naming what it does serve turns a - // registry mistake into a one-line fix. - return model.NewBadReqErr("", fmt.Errorf( - "upstream: %s does not serve action %q; it serves %s", - plan.BindingKey, action, strings.Join(plan.ServedActions(), ", "))) - } - - beckn, err := decodeBody(ctx.Body) - if err != nil { - return err - } - - // What this provider requires of a payload is declared by its mapping, not - // by this step. A capability with a different rule is a different mapping - // file rather than a different build -- and the rule sits beside the - // extraction it guards. - if err := s.mapper.Verify(ctx, call.Mappings, map[string]any{"beckn": beckn}); err != nil { - return err - } - - // Whatever this capability needs that its payload does not carry. Empty for - // most: the mapping reads the payload directly and needs nothing resolved. - local, err := s.resolve(ctx, plan.BindingKey, beckn) - if err != nil { - return err - } - - upstreamRequest, err := s.buildRequest(ctx, call, beckn, local) - if err != nil { - return err - } - - // Which credential this provider takes. Resolved from the binding key, so - // one step serving several providers authenticates each as its own. - // Startup guarantees a profile per served provider; this guards the case - // where a record arrives for a key the config never declared. - auth, configured := s.auth[providerIDFrom(plan.BindingKey)] - if !configured { - return fmt.Errorf("upstream: no credential is configured for %s", plan.BindingKey) - } - - upstreamResponse, err := s.call(ctx, auth, plan.BaseURL, call, upstreamRequest) - if err != nil { - return err - } - - answer, err := decodeBody(upstreamResponse) - if err != nil { - return fmt.Errorf("upstream: provider answered with something that is not JSON: %w", err) - } - - // The same mapping reference as the request, other half: one file carries - // both directions for this action. - // - // The mapping is handed what each party sent, plus whatever prerequisites - // resolved, under _local. Empty when there are none. - becknResponse, err := s.mapper.Transform(ctx, call.Mappings, definition.DirectionResponse, map[string]any{ - "beckn": beckn, - "_local": local, - "response": answer, - }) - if err != nil { - return err - } - if len(becknResponse) == 0 { - // Either the file has no response half, or its transform matched nothing - // in this answer. Both leave no Beckn response to return, and returning - // the provider's own shape instead would be worse than failing. The - // message says what was observed rather than guessing which it was. - return fmt.Errorf("upstream: the response half of %s produced nothing, so %s cannot be answered", - call.Mappings, plan.BindingKey) - } - - ctx.ResponseBody = becknResponse - log.Infof(ctx, "upstream: served %s in %d bytes", plan.BindingKey, len(becknResponse)) - return nil -} - -// buildRequest produces what the provider is sent. -// -// Whatever the mapping produces IS the request: a body for a method that takes -// one, query parameters for a method that does not. Nothing is substituted when -// it produces nothing, so an empty request half means an empty request. -// -// This step used to extract a point from the payload and fall back to sending -// that. It meant the choice of which payload fields reach the provider lived in -// Go, so adding a parameter -- a date range, say -- was a rebuild. Now it is a -// mapping edit and nothing else. -func (s *Step) buildRequest(ctx context.Context, call model.ActionPlan, beckn any, local map[string]any) ([]byte, error) { - mapped, err := s.mapper.Transform(ctx, call.Mappings, definition.DirectionRequest, map[string]any{ - "beckn": beckn, - "_local": local, - }) - if err != nil { - return nil, err - } - if len(mapped) == 0 { - log.Debugf(ctx, "upstream: the request half of %s produced nothing; sending an empty request", call.Mappings) - } - return mapped, nil -} - -// extractAction reads the Beckn action a request is for. -func extractAction(body []byte) string { - var payload struct { - Context struct { - Action string `json:"action"` - } `json:"context"` - } - if err := json.Unmarshal(body, &payload); err != nil { - return "" - } - return payload.Context.Action -} - -// decodeBody turns raw JSON into the generic value a mapping reads. -func decodeBody(body []byte) (any, error) { - var decoded any - if err := json.Unmarshal(body, &decoded); err != nil { - return nil, fmt.Errorf("upstream: could not read JSON: %w", err) - } - return decoded, nil -} - -// call makes the upstream request described by the plan, retrying within its -// budget. -// budget resolves how long one attempt may take and how many retries follow -// it, applying the registry's values within this deployment's ceilings. -// -// Pure and separate from call so both bounds can be asserted without a server -// that sleeps for the timeout it is testing. -// -// retryMax counts retries, not attempts, so the call itself is always made -// once. An absent retryMax and an explicit 0 are the same instruction. -func budget(call model.ActionPlan) (time.Duration, int) { - timeout := DefaultTimeout - if call.TimeoutMs > 0 { - timeout = time.Duration(call.TimeoutMs) * time.Millisecond - } - if timeout > MaxTimeout { - timeout = MaxTimeout - } - - retries := DefaultRetryMax - if call.RetryMax > 0 { - retries = call.RetryMax - } - if retries > MaxRetryMax { - retries = MaxRetryMax - } - return timeout, retries -} - -func (s *Step) call(ctx context.Context, auth *authenticator, baseURL string, call model.ActionPlan, mapped []byte) ([]byte, error) { - endpoint, err := buildEndpoint(baseURL, call, mapped) - if err != nil { - return nil, err - } - - timeout, retries := budget(call) - if d := time.Duration(call.TimeoutMs) * time.Millisecond; d > timeout { - log.Warnf(ctx, "upstream: registry asks for a %v timeout; using the %v ceiling", d, timeout) - } - if call.RetryMax > retries { - log.Warnf(ctx, "upstream: registry asks for %d retries; using the %d ceiling", - call.RetryMax, retries) - } - attempts := retries + 1 - - var lastErr error - for attempt := 1; attempt <= attempts; attempt++ { - // A caller that has gone away is not worth another attempt, and neither - // is a budget already spent. Checked before the call rather than after, - // so a cancelled request costs nothing. - if err := ctx.Err(); err != nil { - if lastErr == nil { - lastErr = err - } - break - } - - body, err := s.attempt(ctx, auth, call, endpoint, mapped, timeout) - if err == nil { - return body, nil - } - lastErr = s.redact(err) - log.Warnf(ctx, "upstream: attempt %d/%d failed: %v", attempt, attempts, lastErr) - - // Only some failures are worth repeating. A 4xx, a request this step - // could not build and a credential it could not read will fail - // identically however many times they are tried -- and retrying the - // credential case is the worst of them, because it reports an - // operator's missing environment variable as the provider being down. - if isPermanent(err) { - break - } - if attempt < attempts { - if err := sleep(ctx, backoff(attempt)); err != nil { - break - } - } - } - return nil, model.NewCodedErr(http.StatusBadGateway, codeUpstreamUnavailable, - fmt.Errorf("upstream: provider did not answer after %d attempts: %w", attempts, lastErr)) -} - -// permanentErr marks a failure no retry can fix. Kept unexported and detected -// with errors.As, so a caller of this package sees only the underlying error. -type permanentErr struct{ error } - -func (p permanentErr) Unwrap() error { return p.error } - -// doNotRetry marks err as not worth repeating. -func doNotRetry(err error) error { return permanentErr{err} } - -// isPermanent reports whether err is one no further attempt would change. -func isPermanent(err error) bool { - var permanent permanentErr - return errors.As(err, &permanent) -} - -// backoff is how long to wait before the next attempt. -// -// Exponential from a short base and capped, because the provider being briefly -// busy is the case worth waiting out; anything longer is a timeout's job. With -// no wait at all a retryMax of 5 spends its whole budget inside a couple of -// milliseconds, which is not a retry so much as the same failure six times. -func backoff(attempt int) time.Duration { - if attempt <= 1 { - return RetryBackoffBase - } - // Doubled in a loop that stops at the ceiling rather than shifted and then - // clamped. `RetryBackoffBase << (attempt - 1)` overflows int64 once the - // shift reaches 38 at a 50ms base, and the wrapped value is NEGATIVE -- so - // it passes the `> RetryBackoffMax` check, is returned, and a sleep on a - // negative duration returns immediately. The retry loop then spins as fast - // as the provider can refuse. Stopping at the ceiling cannot overflow, - // because it never doubles a value already at or past it. - wait := RetryBackoffBase - for i := 1; i < attempt && wait < RetryBackoffMax; i++ { - wait *= 2 - } - if wait > RetryBackoffMax { - return RetryBackoffMax - } - return wait -} - -// sleep waits, or reports that the context ended first. -func sleep(ctx context.Context, d time.Duration) error { - timer := time.NewTimer(d) - defer timer.Stop() - select { - case <-ctx.Done(): - return ctx.Err() - case <-timer.C: - return nil - } -} - -// attempt makes one upstream request. -func (s *Step) attempt(ctx context.Context, auth *authenticator, call model.ActionPlan, endpoint string, mapped []byte, timeout time.Duration) ([]byte, error) { - attemptCtx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - method := canonicalMethod(call.Method) - req, err := http.NewRequestWithContext(attemptCtx, method, endpoint, requestBody(method, mapped)) - if err != nil { - return nil, doNotRetry(fmt.Errorf("could not build the request: %w", err)) - } - if hasBody(method) { - req.Header.Set("Content-Type", "application/json") - } - if err := s.authenticate(auth, req); err != nil { - // A missing or unreadable credential is configuration, not weather. - return nil, doNotRetry(err) - } - - // The URL as it actually went on the wire, credential removed. At info - // rather than debug because this is the line that answers "what did we ask, - // and what came back" -- the question every provider problem starts with. - requested := s.redactString(req.URL.String()) - - resp, err := s.httpClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(io.LimitReader(resp.Body, s.config.MaxResponseBytes+1)) - if err != nil { - return nil, fmt.Errorf("could not read the response: %w", err) - } - log.Infof(ctx, "upstream: %s %s -> %s, %d bytes", method, requested, resp.Status, len(body)) - if int64(len(body)) > s.config.MaxResponseBytes { - // Asking again will not make the answer smaller. - return nil, doNotRetry(fmt.Errorf("response exceeds the %d byte limit", s.config.MaxResponseBytes)) - } - // Any 2xx, not 200 alone. A provider is entitled to answer 202 for work it - // accepted, 204 for nothing to report, or 201 for something it created, and - // treating those as failures would refuse a perfectly good exchange. 3xx - // does not reach here: the client follows redirects. - if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { - // The body is logged, not returned. It goes into a 502 that is signed - // and sent to the network caller, and what a provider puts in a failure - // body is its own business -- a stack trace, an internal hostname, a - // database error. The status is the caller's business and stays; the - // body is the operator's, and the log is where the operator looks. - // Redacted on the way to the log too. A provider that rejects a - // request often quotes it back, credential and all -- so the body is - // exactly where a query-string token turns up, and moving it from the - // error to the log would only move the leak. - log.Warnf(ctx, "upstream: provider returned %s for %s %s: %s", - resp.Status, method, requested, s.redactString(explain(body))) - err := fmt.Errorf("provider returned %s", resp.Status) - // 5xx and 429 are the provider asking to be tried again. Every other - // 4xx is a statement about the request, which will not improve. - if resp.StatusCode < http.StatusInternalServerError && resp.StatusCode != http.StatusTooManyRequests { - return nil, doNotRetry(err) - } - return nil, err - } - return body, nil -} - -// explainLimit is how much of a failed response is quoted. Enough for a -// provider's own message, short enough not to put a page of HTML in a log line -// or a NACK. -const explainLimit = 300 - -// explain renders a failed response body for a human. -// -// The body was already read and then thrown away, so a provider's own account -// of what was wrong -- Agmarknet says "no data" in the body of a 400 -- never -// reached anyone. The status alone says a call failed and nothing about why, -// which is the first thing an operator needs and the thing that makes a real -// provider's behaviour observable at all. -func explain(body []byte) string { - text := strings.TrimSpace(string(body)) - if text == "" { - return "(no body)" - } - // Collapse whitespace: a provider that answers with indented JSON or an - // HTML error page should not spread one failure over forty log lines. - text = strings.Join(strings.Fields(text), " ") - if len(text) > explainLimit { - return text[:explainLimit] + "... (truncated)" - } - return text -} - -// authenticate presents this provider's credentials, read from the environment -// at call time so a rotated secret takes effect without a restart. -// missingCredential reports an unset credential without naming the variable on -// the wire. -// -// The variable name is deployment configuration, and this error is wrapped into -// a 502 that is signed and returned to a network peer. Telling a peer that -// MANDI_TOKEN is what this deployment reads describes the inside of somebody -// else's stack for no benefit to the caller -- the caller cannot set it, and -// the fix is entirely the operator's. So the name goes to the log, where the -// operator is, and the wire gets the scheme that failed. -func (s *Step) missingCredential(ctx context.Context, provider, scheme, envNames string) error { - err := fmt.Errorf("upstream: the %s credential for %s is not configured", scheme, provider) - log.Errorf(ctx, err, "upstream: %s auth is configured for %s but %s is not set", - scheme, provider, envNames) - return err -} - -func (s *Step) authenticate(auth *authenticator, req *http.Request) error { - cfg := auth.cfg - switch cfg.Scheme { - case AuthSchemeBasic: - username, password := os.Getenv(cfg.UsernameEnv), os.Getenv(cfg.PasswordEnv) - if username == "" || password == "" { - return s.missingCredential(req.Context(), cfg.Provider, "basic", - cfg.UsernameEnv+" and "+cfg.PasswordEnv) - } - req.SetBasicAuth(username, password) - case AuthSchemeHeader: - value := os.Getenv(cfg.HeaderValueEnv) - if value == "" { - return s.missingCredential(req.Context(), cfg.Provider, "header", cfg.HeaderValueEnv) - } - req.Header.Set(cfg.HeaderName, value) - case AuthSchemeQuery: - value := os.Getenv(cfg.QueryValueEnv) - if value == "" { - return s.missingCredential(req.Context(), cfg.Provider, "query", cfg.QueryValueEnv) - } - // Set rather than Add: a second copy of the parameter is not a - // credential, it is an ambiguity, and which one an upstream reads is - // its own business. - query := req.URL.Query() - query.Set(cfg.QueryName, value) - req.URL.RawQuery = query.Encode() - case AuthSchemeOAuth2: - token, err := s.bearerToken(req.Context(), auth) - if err != nil { - return err - } - req.Header.Set("Authorization", "Bearer "+token) - } - return nil -} - -// redact removes a query-string credential from an error's text. -// -// Go's transport errors quote the whole URL -- `Get "http://host/p?token=..." -// dial tcp: ...` -- so without this, one unreachable host writes the credential -// into the log at warn level. Nothing else in this package puts a URL in a -// message, which is why this is the only place it is needed. -// -// A plain string replacement, because the value is what leaks and the value is -// what we hold. Parsing the error to find it would assume a shape net/http does -// not promise. -func (s *Step) redact(err error) error { - if err == nil { - return nil - } - text := s.redactString(err.Error()) - if text == err.Error() { - return err - } - return redactedErr{text: text, err: err} -} - -// redactedErr reports a redacted message while keeping the original reachable -// for errors.Is and errors.As. -// -// errors.New(text) was the obvious thing and it broke the chain: the redacted -// value is what gets %w-wrapped into the final 502, so under a query-string -// scheme -- and only then, since nothing else redacts -- errors.Is(err, -// context.DeadlineExceeded) silently stopped matching. Retry classification -// was never affected, because isPermanent tests the error before redaction, -// which is why nothing failed visibly. -// -// fmt.Errorf("%s: %w", text, err) would have restored the chain and undone the -// redaction with it: %w formats the original, credential included. Reporting -// the redacted text from Error() and the original from Unwrap() keeps both. -// -// The original's text is reachable through errors.Unwrap, which is a -// deliberate act by a caller who wants the cause -- and %v, %s and %w on the -// value itself all go through Error() and stay redacted. -type redactedErr struct { - text string - err error -} - -func (e redactedErr) Error() string { return e.text } -func (e redactedErr) Unwrap() error { return e.err } - -// bearerToken returns a token to send, exchanging one if what we hold has -// expired or if we hold none. -// -// The lock spans the fetch on purpose. Without it a cold start sends every -// concurrent request to the token endpoint, and the provider's issuer sees a -// burst of identical exchanges. Serialising them costs one wait per token -// lifetime and nothing after that, which is a better trade than a second -// caching layer. -// -// NOTHING IS CACHED ON FAILURE. A transient outage at the token endpoint must -// not leave this step holding a failure for the lifetime it never obtained -- -// the next request tries again. -func (s *Step) bearerToken(ctx context.Context, auth *authenticator) (string, error) { - if held := auth.token.Load(); held != nil && time.Now().Before(held.expiry) { - return held.value, nil - } - - auth.tokenMu.Lock() - defer auth.tokenMu.Unlock() - - // Re-checked after acquiring: while this caller waited, whoever held the - // mutex may already have exchanged a fresh token. - if held := auth.token.Load(); held != nil && time.Now().Before(held.expiry) { - return held.value, nil - } - - token, lifetime, err := s.exchangeToken(ctx, auth) - if err != nil { - return "", err - } - auth.token.Store(&cachedToken{value: token, expiry: time.Now().Add(lifetime)}) - return token, nil -} - -// cachedToken is a token and the moment it stops being trusted. -type cachedToken struct { - value string - expiry time.Time -} - -// tokenResponse is the part of an OAuth2 token response this step reads. -type tokenResponse struct { - AccessToken string `json:"access_token"` - ExpiresIn int `json:"expires_in"` -} - -// exchangeToken performs the client_credentials grant. -// -// Every failure here is the UPSTREAM exchange failing, so all of them carry -// 502. Unclassified they would surface as a 500, which tells a network peer -// this adapter broke when in fact the provider's issuer did. -func (s *Step) exchangeToken(ctx context.Context, auth *authenticator) (string, time.Duration, error) { - cfg := auth.cfg - clientID, clientSecret := os.Getenv(cfg.ClientIDEnv), os.Getenv(cfg.ClientSecretEnv) - if clientID == "" || clientSecret == "" { - return "", 0, s.missingCredential(ctx, cfg.Provider, "oauth2", - cfg.ClientIDEnv+" and "+cfg.ClientSecretEnv) - } - - form := url.Values{ - "grant_type": {"client_credentials"}, - "client_id": {clientID}, - "client_secret": {clientSecret}, - } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, cfg.TokenURL, - strings.NewReader(form.Encode())) - if err != nil { - return "", 0, s.tokenErr(fmt.Errorf("token request could not be built: %w", err)) - } - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - - resp, err := s.httpClient.Do(req) - if err != nil { - return "", 0, s.tokenErr(fmt.Errorf("token endpoint %s could not be reached: %w", - cfg.TokenURL, err)) - } - defer resp.Body.Close() - - // Bounded like any other upstream read: a token response is small, and an - // unbounded read here would be a hole in the same wall. - body, err := io.ReadAll(io.LimitReader(resp.Body, s.config.MaxResponseBytes+1)) - if err != nil { - return "", 0, s.tokenErr(fmt.Errorf("token response could not be read: %w", err)) - } - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - // The status, not the body: what an issuer puts in a failure body is its - // own business, and it routinely quotes the request back. - log.Warnf(ctx, "upstream: token endpoint %s returned %s: %s", - cfg.TokenURL, resp.Status, s.redactString(explain(body))) - return "", 0, s.tokenErr(fmt.Errorf("token endpoint returned %s", resp.Status)) - } - - var parsed tokenResponse - if err := json.Unmarshal(body, &parsed); err != nil { - return "", 0, s.tokenErr(fmt.Errorf("token response from %s is not JSON: %w", - cfg.TokenURL, err)) - } - if parsed.AccessToken == "" { - return "", 0, s.tokenErr(fmt.Errorf("token response from %s carries no access_token", - cfg.TokenURL)) - } - lifetime, ok := tokenLifetime(parsed.ExpiresIn, tokenRefreshSkew) - if !ok { - return "", 0, s.tokenErr(fmt.Errorf( - "token response from %s carries no usable expires_in, so its lifetime is unknown", - cfg.TokenURL)) - } - return parsed.AccessToken, lifetime, nil -} - -// tokenErr classifies a failed exchange. Always 502: the exchange is with the -// provider's issuer, so its failure is upstream's, never the caller's. -func (s *Step) tokenErr(err error) error { - return model.NewCodedErr(http.StatusBadGateway, codeUpstreamUnavailable, - fmt.Errorf("upstream: oauth2 token exchange failed: %w", err)) -} - -// tokenLifetime turns a token response's expires_in into how long we may hold -// that token, or reports that we may not hold it at all. -// -// The issuer owns this number, so it is read from the response and never from -// our config: a configured copy is a second version of the same fact, and it -// is wrong the moment a realm's token lifetime is retuned -- silently, with -// every request 401ing until someone edits a file. -// -// skew is subtracted so a request that passes the check cannot arrive at the -// provider after expiry. It has to exceed the round trip, and costs one extra -// refresh per token lifetime. -// -// Two answers rather than one duration, because "we were not told" is not a -// lifetime. A response with no expires_in is refused rather than cached for a -// guessed period or re-fetched on every single request. -func tokenLifetime(expiresIn int, skew time.Duration) (time.Duration, bool) { - if expiresIn <= 0 { - return 0, false - } - lifetime := time.Duration(expiresIn) * time.Second - // A lifetime at or under the skew would put the expiry in the past, and - // then every request fetches a fresh token. Half is still early enough to - // refresh before the real expiry. - if lifetime <= skew { - return lifetime / 2, true - } - return lifetime - skew, true -} - -// redactString removes the configured credential from any text about to be -// logged or returned -- an error, a provider's response body, or the URL that -// was requested. -// -// Logging those is deliberate: they say what was asked of whom and what came -// back, which is the first thing anyone wants when a provider misbehaves. This -// is what makes that safe to do at info and warn level. -// -// EVERY scheme, not just query. This used to return early unless the scheme was -// query, on the reasoning that only a query credential reaches a URL -- true of -// the URL, and wrong about the body. A provider quoting the request it rejected -// is the ordinary shape of a 401 or 403 body, an API gateway echoing the -// Authorization header is routine, and a wrong-credential 4xx is not retried, -// so it lands in the log once per request for as long as the credential is -// wrong. basic is the scheme the reference config ships. -func (s *Step) redactString(text string) string { - for _, secret := range s.secretForms() { - text = strings.ReplaceAll(text, secret, redactedMarker) - } - return text -} - -// secretForms returns every form the configured credential can appear in, -// longest first so a value that contains another is replaced before its -// substring turns the longer one into a partial redaction. -// -// Per scheme, because the schemes leak differently and redacting the value we -// hold is not enough on its own: -// -// - basic wraps the pair: SetBasicAuth sends base64(user:pass), so the -// password alone does not appear on the wire and replacing it misses the -// echoed header entirely. -// - query escapes: authenticate goes through url.Values.Encode, so a base64 -// token carrying "+", "/" or "=" appears as "a%2Bb%2Fc%3D". Escaping what -// we hold is exact -- same function Encode used, so the two agree by -// construction rather than by a guess about which characters matter. -// - header sends the value as-is. -// -// The raw form is kept alongside the wrapped one in both cases: an error built -// from the config rather than from the request still quotes the credential -// unwrapped. -func (s *Step) secretForms() []string { - // EVERY profile, not the one being served. - // - // Redaction is about what could appear in a piece of text, not about which - // provider a request happened to be for. An error or a log line built while - // serving one provider can quote another's credential -- a shared client - // echoing a header, an issuer naming the wrong caller -- and scoping this - // to the active profile would let that reach the log unredacted. There are - // a handful of profiles and this runs on a failure path, so walking all of - // them costs nothing worth measuring. - var forms []string - for _, auth := range s.auth { - forms = append(forms, auth.secretForms()...) - } - // Sorted across the merged set rather than per profile: one provider's - // short token can be a substring of another's, and replacing the short one - // first would leave the longer half-redacted. - return longestFirst(forms) -} - -// secretForms returns every form this profile's credential can appear in. -// -// Per scheme, because the schemes leak differently and redacting the value we -// hold is not enough on its own: -// -// - basic wraps the pair: SetBasicAuth sends base64(user:pass), so the -// password alone does not appear on the wire and replacing it misses the -// echoed header entirely. -// - query escapes: authenticate goes through url.Values.Encode, so a base64 -// token carrying "+", "/" or "=" appears as "a%2Bb%2Fc%3D". Escaping what -// we hold is exact -- same function Encode used, so the two agree by -// construction rather than by a guess about which characters matter. -// - header sends the value as-is. -// -// The raw form is kept alongside the wrapped one in both cases: an error built -// from the config rather than from the request still quotes the credential -// unwrapped. -func (a *authenticator) secretForms() []string { - switch a.cfg.Scheme { - case AuthSchemeBasic: - username, password := os.Getenv(a.cfg.UsernameEnv), os.Getenv(a.cfg.PasswordEnv) - if password == "" { - return nil - } - forms := []string{password} - if username != "" { - // The wire form, which is what a gateway echoes back. - forms = append(forms, - base64.StdEncoding.EncodeToString([]byte(username+":"+password))) - } - // The username is deliberately NOT redacted. It identifies rather than - // authenticates, and it is routinely a short common word -- redacting - // "user" or "admin" would eat unrelated text and cost the operator the - // log line they came for. The pair and the password are the secrets. - return forms - case AuthSchemeHeader: - value := os.Getenv(a.cfg.HeaderValueEnv) - if value == "" { - return nil - } - return []string{value} - case AuthSchemeOAuth2: - // Both halves: the client secret we send to the issuer, and the token - // it gave back. The token is the one that reaches the provider, so it - // is the one an echoing 401 body quotes. - var forms []string - if secret := os.Getenv(a.cfg.ClientSecretEnv); secret != "" { - forms = append(forms, secret) - } - // Read without tokenMu: this is reached from inside the exchange, which - // holds it. - if held := a.token.Load(); held != nil && held.value != "" { - forms = append(forms, held.value) - } - // The client id is deliberately NOT redacted: it identifies, it does - // not authenticate, and it is what makes a log line useful. - return forms - case AuthSchemeQuery: - value := os.Getenv(a.cfg.QueryValueEnv) - if value == "" { - return nil - } - forms := []string{value} - if encoded := url.QueryEscape(value); encoded != value { - forms = append(forms, encoded) - } - return forms - } - return nil -} - -// longestFirst orders replacement candidates so a longer form is substituted -// before any shorter one it contains. -func longestFirst(forms []string) []string { - sort.Slice(forms, func(i, j int) bool { return len(forms[i]) > len(forms[j]) }) - return forms -} - -// buildEndpoint joins the plan's base URL and path, carrying the mapped request -// as query parameters when the method takes no body. -func buildEndpoint(baseURL string, call model.ActionPlan, mapped []byte) (string, error) { - if err := verifyBaseURL(baseURL); err != nil { - return "", err - } - if err := verifyPath(call.Path); err != nil { - return "", err - } - - // baseUrl cannot end in a slash and path must begin with one, so exactly one - // separator appears between them. The trim is belt and braces: the registry - // refuses a trailing slash on baseUrl, and this keeps a row that predates - // that from producing a doubled one. - endpoint := strings.TrimSuffix(baseURL, "/") + call.Path - if hasBody(call.Method) { - return endpoint, nil - } - - query, err := asQuery(mapped) - if err != nil { - return "", err - } - if query == "" { - return endpoint, nil - } - if strings.Contains(endpoint, "?") { - return endpoint + "&" + query, nil - } - return endpoint + "?" + query, nil -} - -// verifyPath refuses a published path nobody could have meant. -// -// The registry constrains this, but it is a separate deployable that may not be -// updated in step, so a row that slipped through has to fail here with something -// an operator can act on rather than as a provider's 404 three hops away. -// -// An empty segment is the case worth catching: "//get-daily" is never -// deliberate, and plenty of servers answer it differently from "/get-daily". A -// trailing slash is deliberately left alone -- "/api/" and "/api" are a -// distinction some APIs genuinely make, so stripping it would silently change -// the URL the operator published. -func verifyPath(path string) error { - if path == "" { - return model.NewBadReqErr("", errors.New("upstream: the registry publishes no path for this action")) - } - if !strings.HasPrefix(path, "/") { - return model.NewBadReqErr("", fmt.Errorf( - "upstream: path %q does not begin with a slash, so it cannot be joined to a base url", path)) - } - if strings.Contains(path, "//") { - return model.NewBadReqErr("", fmt.Errorf( - "upstream: path %q has an empty segment; write it with single slashes", path)) - } - // A dot segment is refused rather than resolved. The registry says which - // path answers an action, and a row that climbs out of it is either a - // mistake or an attempt to reach something the row does not name -- and - // net/url would quietly resolve it either way, so the request that left - // would not be the request the row described. - for _, segment := range strings.Split(path, "/") { - if segment == ".." || segment == "." { - return model.NewBadReqErr("", fmt.Errorf( - "upstream: path %q contains the %q segment; publish the path it resolves to instead", - path, segment)) - } - } - // A fragment is never sent, so a row carrying one describes a request that - // cannot be made. Refused here rather than silently dropped by the - // transport, which would make the row look honoured. - if strings.Contains(path, "#") { - return model.NewBadReqErr("", fmt.Errorf( - "upstream: path %q contains a fragment, which is never sent to a server", path)) - } - return nil -} - -// verifyBaseURL checks the participant's base url before it is joined to a -// path, so a row that cannot produce a request says so as a bad request rather -// than as the provider being unreachable. -// -// Without this, `baseUrl: "registry:8081"` -- a scheme left off -- failed -// inside http.NewRequestWithContext and arrived as a 502 "provider did not -// answer after 1 attempts: could not build the request". That names the -// provider for an error in the row describing it, and it is retried on the way -// there. jsonmapper has always checked its own reference this way; this is the -// same check on the other url the registry publishes. -func verifyBaseURL(baseURL string) error { - if baseURL == "" { - return model.NewBadReqErr("", errors.New("upstream: the registry publishes no base url for this provider")) - } - parsed, err := url.Parse(baseURL) - if err != nil { - return model.NewBadReqErr("", fmt.Errorf("upstream: invalid base url %q: %w", baseURL, err)) - } - if parsed.Scheme != "http" && parsed.Scheme != "https" { - return model.NewBadReqErr("", fmt.Errorf( - "upstream: base url %q must be http or https", baseURL)) - } - if parsed.Host == "" { - return model.NewBadReqErr("", fmt.Errorf("upstream: base url %q names no host", baseURL)) - } - return nil -} - -// asQuery renders a mapped request as query parameters. -// -// A method with no body still needs the mapping's output somewhere, and the -// query string is the only place it can go. Only scalars are carried: a nested -// value has no single obvious encoding, and inventing one here would put a -// convention in Go that belongs in the mapping. -func asQuery(mapped []byte) (string, error) { - if len(bytes.TrimSpace(mapped)) == 0 { - return "", nil - } - var fields map[string]any - if err := json.Unmarshal(mapped, &fields); err != nil { - return "", fmt.Errorf("upstream: mapped request is not an object, so it cannot become a query: %w", err) - } - - values := url.Values{} - for name, value := range fields { - rendered, ok := renderScalar(value) - if !ok { - return "", fmt.Errorf("upstream: mapped field %q is not a scalar and cannot become a query parameter", name) - } - values.Set(name, rendered) - } - return values.Encode(), nil -} - -// renderScalar renders a JSON scalar as a query parameter value. -func renderScalar(value any) (string, bool) { - switch typed := value.(type) { - case string: - return typed, true - case bool: - return strconv.FormatBool(typed), true - case float64: - // 'g' with -1 precision round-trips without inventing trailing zeros, so - // 19.9975 stays 19.9975 rather than becoming 19.997500. - return strconv.FormatFloat(typed, 'g', -1, 64), true - default: - return "", false - } -} - -// requestBody returns the body to send, which is none for methods that take none. -func requestBody(method string, mapped []byte) io.Reader { - if !hasBody(method) { - return nil - } - return bytes.NewReader(mapped) -} - -// hasBody reports whether a method carries a request body. -func hasBody(method string) bool { - switch canonicalMethod(method) { - case http.MethodGet, http.MethodHead, http.MethodDelete, "": - return false - default: - return true - } -} - -// canonicalMethod returns a known HTTP method in the spelling the RFC gives -// it, and anything else unchanged. -// -// hasBody used to upper-case privately, which made the method look -// case-insensitive when it is not: NewRequestWithContext transmits it verbatim, -// so a registry row reading `method: "post"` sent `post /path HTTP/1.1`. The -// body was attached correctly -- hasBody had normalised -- but nginx and most -// gateways answer 405 to a lowercase method, which classifies permanent and -// surfaces as a 502 "provider did not answer". A row that is right in every -// respect but its capitalisation is a bad way to spend an afternoon. -// -// Only known methods are rewritten. Upper-casing everything would be a new -// restriction on an upstream entitled to a method this list has not heard of, -// and net/http already refuses one that is not a valid token. -// -// An empty method is left empty: net/http documents "" as GET and substitutes -// it, and hasBody agrees that it carries no body, so the two are already -// consistent and inventing a value here would only hide where it comes from. -func canonicalMethod(method string) string { - upper := strings.ToUpper(method) - switch upper { - case http.MethodGet, http.MethodHead, http.MethodPost, http.MethodPut, - http.MethodPatch, http.MethodDelete, http.MethodConnect, - http.MethodOptions, http.MethodTrace: - return upper - } - return method -} From d4e1e8db270755122aec60de2903220789b928c9 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 17:49:22 +0530 Subject: [PATCH 108/130] docs(upstream): group constant.go by subject [#7] One block was headed "Auth schemes this step can present upstream" while also holding RetryBackoffBase, RetryBackoffMax and redactedMarker. Three subjects under one heading, which the split carried across unchanged rather than fixing mid-move. Now three: the retry waits, the redaction marker on its own, and the auth schemes under the comment that was always about them. Comments only -- all 16 constants present with identical values, verified name by name against the file before the split. 66 packages green, no races. The file doc also says why authFields is not here: it is a var, Go has no constant maps, and it belongs beside the parser in auth.go that reads it. --- .../internal/upstream/constant.go | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/pkg/plugin/implementation/internal/upstream/constant.go b/pkg/plugin/implementation/internal/upstream/constant.go index 438a5f13..1262d607 100644 --- a/pkg/plugin/implementation/internal/upstream/constant.go +++ b/pkg/plugin/implementation/internal/upstream/constant.go @@ -1,5 +1,8 @@ -// Constants: the budgets a registry row is clamped to, the retry backoff, the -// auth schemes, and the two markers this package puts in text it returns. +// Constants only: the budgets a registry row is clamped to, the retry backoff, +// the auth schemes, and the markers this package puts in text it returns. +// +// authFields is deliberately NOT here. It is a var -- Go has no constant maps -- +// and it belongs beside the parser in auth.go that reads it. package upstream import ( @@ -35,10 +38,7 @@ const ( MaxRetryMax = 5 ) -// Auth schemes this step can present upstream. Credentials themselves are never -// configured here or held in the registry -- config names the environment -// variable to read, so a secret reaches the process through its environment and -// nothing else. +// How long this step waits between attempts. const ( // RetryBackoffBase is the first wait between attempts, doubling from there // up to RetryBackoffMax. Short, because the retry budget comes from the @@ -46,10 +46,16 @@ const ( // latency -- only for the provider's brief unavailability to be ridden out. RetryBackoffBase = 50 * time.Millisecond RetryBackoffMax = 800 * time.Millisecond +) - // redactedMarker stands in for a credential in anything logged or returned. - redactedMarker = "REDACTED" +// redactedMarker stands in for a credential in anything logged or returned. +const redactedMarker = "REDACTED" +// Auth schemes this step can present upstream. Credentials themselves are never +// configured here or held in the registry -- config names the environment +// variable to read, so a secret reaches the process through its environment and +// nothing else. +const ( AuthSchemeNone = "none" AuthSchemeBasic = "basic" AuthSchemeHeader = "header" From 53b4bd7e1bd090f6045213abf9457981ae77298d Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 18:20:18 +0530 Subject: [PATCH 109/130] refactor: rename internal/upstream to internal/common [#7] The package holds what all three capability plugins share, and the name now says so: WeatherObservation, MandiPrice and KnowledgeAdvisory are each a name, a set of binding keys and a credential profile over this machinery. internal/upstream -> internal/common package upstream -> package common upstream.go -> common.go upstream_test.go -> common_test.go 17 files and 45 qualified references. Four of them had to ALIAS the import as upstreamstep, because `upstream` was already the name of the httptest server in those tests -- that alias is gone, since nothing collides with `common`. THE WORD "UPSTREAM" STAYS where it means the API being called. It is the registry's own -- a Participant of type upstream, as against a node that speaks Beckn -- so it remains in the 51 error and log prefixes, in upstreamRequest and upstreamResponse, and in the comments. An operator reading "upstream: provider did not answer" is being told which leg of the call failed; "common:" would tell them nothing. The package doc now records that split explicitly. The doc also records what the name must not become. `common` is a name that attracts unrelated code, so it says: everything here is one subject, calling an upstream on a capability's behalf, and something shared by fewer than all three plugins belongs in its own package rather than here on the strength of the name. 66 packages green, vet clean, no races. One thing the compiler caught rather than review: the first pass rewrote `upstream.URL` and `upstream.Close` on the httptest server, because the regex matched a capitalised field the same way it matched an exported identifier. 82 of those, reverted -- the four legitimate reference shapes are Config, New, AuthProfile and ParseProviderAuth. --- .../AgricultureFacility/v0.1/attributes.yaml | 149 +++++++++++++ .../v0.1/examples/custom-hiring-centre.json | 30 +++ .../v0.1/examples/krishi-vigyan-kendra.json | 34 +++ .../on-demand-facility-discovery.json | 23 ++ .../v0.1/examples/soil-testing-facility.json | 28 +++ .../v0.1/examples/warehouse.json | 40 ++++ .../AgricultureResource/v0.1/attributes.yaml | 197 ++++++++++++++++++ .../Address/v2.0/attributes.yaml | 45 ++++ .../Contact/v2.0/attributes.yaml | 36 ++++ .../Descriptor/v2.1/attributes.yaml | 52 +++++ .../GeoJSONGeometry/v2.0/attributes.yaml | 57 +++++ .../Location/v2.0/attributes.yaml | 26 +++ .../Address/v2.0/attributes.yaml | 45 ++++ .../Document/v2.0/attributes.yaml | 56 +++++ .../GeoJSONGeometry/v2.0/attributes.yaml | 57 +++++ .../MediaFile/v2.0/attributes.yaml | 32 +++ .../KnowledgeAdvisory/KnowledgeAdvisory.go | 8 +- .../KnowledgeAdvisory/cmd/plugin.go | 6 +- .../KnowledgeAdvisory/cmd/plugin_test.go | 14 +- .../KnowledgeAdvisory/live_test.go | 4 +- .../KnowledgeAdvisory/mappings_test.go | 6 +- .../KnowledgeAdvisory/prerequisites.go | 4 +- .../implementation/MandiPrice/MandiPrice.go | 8 +- .../implementation/MandiPrice/cmd/plugin.go | 6 +- .../MandiPrice/cmd/plugin_test.go | 10 +- .../MandiPrice/mappings_test.go | 14 +- .../MandiPrice/prerequisites.go | 4 +- .../WeatherObservation/WeatherObservation.go | 8 +- .../WeatherObservation/cmd/plugin.go | 6 +- .../WeatherObservation/cmd/plugin_test.go | 6 +- .../WeatherObservation/mappings_test.go | 6 +- .../WeatherObservation/prerequisites.go | 4 +- .../internal/{upstream => common}/auth.go | 2 +- .../upstream.go => common/common.go} | 24 ++- .../common_test.go} | 2 +- .../internal/{upstream => common}/constant.go | 2 +- .../{upstream => common}/dispatch_test.go | 12 +- .../internal/{upstream => common}/http.go | 2 +- .../{upstream => common}/perprovider_test.go | 2 +- .../internal/{upstream => common}/redact.go | 2 +- .../internal/{upstream => common}/serve.go | 2 +- .../internal/{upstream => common}/token.go | 2 +- 42 files changed, 996 insertions(+), 77 deletions(-) create mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/attributes.yaml create mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/custom-hiring-centre.json create mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/krishi-vigyan-kendra.json create mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/on-demand-facility-discovery.json create mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/soil-testing-facility.json create mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/warehouse.json create mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureResource/v0.1/attributes.yaml create mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Address/v2.0/attributes.yaml create mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Contact/v2.0/attributes.yaml create mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Descriptor/v2.1/attributes.yaml create mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/GeoJSONGeometry/v2.0/attributes.yaml create mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Location/v2.0/attributes.yaml create mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/Address/v2.0/attributes.yaml create mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/Document/v2.0/attributes.yaml create mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/GeoJSONGeometry/v2.0/attributes.yaml create mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/MediaFile/v2.0/attributes.yaml rename pkg/plugin/implementation/internal/{upstream => common}/auth.go (99%) rename pkg/plugin/implementation/internal/{upstream/upstream.go => common/common.go} (88%) rename pkg/plugin/implementation/internal/{upstream/upstream_test.go => common/common_test.go} (99%) rename pkg/plugin/implementation/internal/{upstream => common}/constant.go (99%) rename pkg/plugin/implementation/internal/{upstream => common}/dispatch_test.go (94%) rename pkg/plugin/implementation/internal/{upstream => common}/http.go (99%) rename pkg/plugin/implementation/internal/{upstream => common}/perprovider_test.go (99%) rename pkg/plugin/implementation/internal/{upstream => common}/redact.go (99%) rename pkg/plugin/implementation/internal/{upstream => common}/serve.go (99%) rename pkg/plugin/implementation/internal/{upstream => common}/token.go (99%) diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/attributes.yaml new file mode 100644 index 00000000..d4b063a8 --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/attributes.yaml @@ -0,0 +1,149 @@ +openapi: 3.1.1 +info: + title: OpenAgriNet - Agriculture Facility Attributes + version: 0.1.0 + description: > + Agricultural facilities that may be discovered through a Provider or + supplied directly. Combines with AgricultureResource through allOf and + attaches through resourceAttributes. + +components: + schemas: + AgricultureFacility: + type: object + x-beckn-container: resourceAttributes + x-jsonld: + "@context": "https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld" + "@type": openagrinet:AgricultureFacility + allOf: + - $ref: "../../AgricultureResource/v0.1/attributes.yaml#/components/schemas/AgricultureResource" + - type: object + required: ["@type"] + properties: + "@type": + oneOf: + - type: string + const: openagrinet:AgricultureFacility + - type: array + minItems: 2 + uniqueItems: true + contains: + const: openagrinet:AgricultureFacility + items: + oneOf: + - const: openagrinet:AgricultureFacility + - type: string + minLength: 1 + not: + pattern: "^openagrinet:" + description: Canonical OAN type, optionally accompanied by Provider-defined JSON-LD types. + supportedFacilityTypes: + type: array + minItems: 1 + uniqueItems: true + description: Facility types the Provider can return on demand. + items: + $ref: "#/components/schemas/FacilityType" + x-jsonld-id: "openagrinet:supportedFacilityTypes" + facilityType: + allOf: + - $ref: "#/components/schemas/FacilityType" + description: Governed type of the facility represented by a Direct Resource. + x-jsonld-id: "openagrinet:facilityType" + location: + allOf: + - $ref: "../../AgricultureResource/v0.1/attributes.yaml#/components/schemas/CompleteLocation" + description: Verified facility geometry and optional address. Do not populate it with the search origin or another inferred point. + x-jsonld-id: "openagrinet:location" + address: + allOf: + - $ref: "../../AgricultureResource/v0.1/attributes.yaml#/components/schemas/NonEmptyAddress" + description: Human-readable facility address when verified geometry is unavailable. + x-jsonld-id: "openagrinet:address" + services: + type: array + minItems: 1 + uniqueItems: true + description: Services publicly offered at the facility. + items: + $ref: "../../AgricultureResource/v0.1/attributes.yaml#/components/schemas/IdentifiedDescriptor" + x-jsonld-id: "openagrinet:services" + capacity: + allOf: + - $ref: "#/components/schemas/FacilityCapacity" + description: Published operating or storage capacity when applicable. + x-jsonld-id: "openagrinet:capacity" + publicContact: + allOf: + - $ref: "https://schema.beckn.io/Contact/v2.0/attributes.yaml#/components/schemas/Contact" + description: Public organizational contact explicitly approved by the Provider for catalog publication. Do not use for private personal contact data. + x-jsonld-id: "openagrinet:publicContact" + website: + type: string + format: uri + description: Public facility or service website. + x-jsonld-id: "openagrinet:website" + source: + allOf: + - $ref: "../../AgricultureResource/v0.1/attributes.yaml#/components/schemas/SourceReference" + description: Provider or authoritative source of the facility record. + x-jsonld-id: "openagrinet:source" + lastUpdatedAt: + type: string + format: date-time + description: Time at which the Provider last updated or verified the facility record. + x-jsonld-id: "openagrinet:lastUpdatedAt" + - if: + properties: + informationMode: + const: OnDemand + required: [informationMode] + then: + required: [supportedFacilityTypes] + not: + anyOf: + - required: [facilityType] + - required: [location] + - required: [address] + - required: [services] + - required: [capacity] + - required: [publicContact] + - required: [website] + - required: [source] + - required: [lastUpdatedAt] + - if: + properties: + informationMode: + const: Direct + required: [informationMode] + then: + required: [facilityType, source] + anyOf: + - required: [location] + - required: [address] + + FacilityType: + type: string + description: Governed agriculture facility category. SoilTestingFacility covers a Soil Health Centre or Soil Testing Laboratory that provides soil-testing services. + enum: [CustomHiringCentre, KrishiVigyanKendra, Warehouse, SoilTestingFacility] + + FacilityCapacity: + type: object + required: [value, unit] + properties: + value: + type: number + minimum: 0 + description: Numeric capacity value. + x-jsonld-id: "openagrinet:value" + unit: + type: string + minLength: 1 + description: Governed or source unit, such as metric tonne. + x-jsonld-id: "openagrinet:unit" + basis: + type: string + minLength: 1 + description: Capacity basis when required, such as total storage or daily throughput. + x-jsonld-id: "openagrinet:capacityBasis" + additionalProperties: false diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/custom-hiring-centre.json b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/custom-hiring-centre.json new file mode 100644 index 00000000..e97b9fcf --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/custom-hiring-centre.json @@ -0,0 +1,30 @@ +{ + "@context": "https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld", + "@type": "openagrinet:AgricultureFacility", + "informationMode": "Direct", + "languages": [ + "en", + "mr" + ], + "facilityType": "CustomHiringCentre", + "address": { + "streetAddress": "Krushisafalya Farmer Producer Company Limited", + "addressLocality": "Gogalgaon", + "addressRegion": "Maharashtra", + "postalCode": "423107", + "addressCountry": "IN", + "extendedAddress": "Rahta, Ahmednagar district" + }, + "services": [ + { + "code": "FARM_MACHINERY_HIRE", + "name": "Farm machinery hire", + "shortDesc": "Access to agricultural machinery and implements." + } + ], + "source": { + "sourceId": "provider:agriculture-common-services", + "sourceName": "Agriculture Common Services Provider" + }, + "lastUpdatedAt": "2026-08-29T00:00:00Z" +} diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/krishi-vigyan-kendra.json b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/krishi-vigyan-kendra.json new file mode 100644 index 00000000..234646e9 --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/krishi-vigyan-kendra.json @@ -0,0 +1,34 @@ +{ + "@context": "https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld", + "@type": "openagrinet:AgricultureFacility", + "informationMode": "Direct", + "languages": [ + "en", + "hi", + "mr" + ], + "facilityType": "KrishiVigyanKendra", + "address": { + "streetAddress": "Krishi Vigyan Kendra, PO Babhleshwar", + "addressLocality": "Babhleshwar", + "addressRegion": "Maharashtra", + "postalCode": "413737", + "addressCountry": "IN", + "extendedAddress": "Rahata, Ahmednagar district" + }, + "services": [ + { + "code": "FARM_ADVISORY", + "name": "Farm advisory" + }, + { + "code": "FARMER_TRAINING", + "name": "Farmer training" + } + ], + "source": { + "sourceId": "provider:agriculture-common-services", + "sourceName": "Agriculture Common Services Provider" + }, + "lastUpdatedAt": "2026-08-29T00:00:00Z" +} diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/on-demand-facility-discovery.json b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/on-demand-facility-discovery.json new file mode 100644 index 00000000..fb57f3ee --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/on-demand-facility-discovery.json @@ -0,0 +1,23 @@ +{ + "@context": "https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld", + "@type": "openagrinet:AgricultureFacility", + "informationMode": "OnDemand", + "languages": [ + "en", + "mr" + ], + "supportedFacilityTypes": [ + "CustomHiringCentre", + "KrishiVigyanKendra", + "Warehouse", + "SoilTestingFacility" + ], + "coverageAreas": [ + { + "codeScheme": "ISO-3166-2", + "areaCode": "IN-MH", + "areaLevel": "State", + "areaName": "Maharashtra" + } + ] +} diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/soil-testing-facility.json b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/soil-testing-facility.json new file mode 100644 index 00000000..3ff39abb --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/soil-testing-facility.json @@ -0,0 +1,28 @@ +{ + "@context": "https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld", + "@type": "openagrinet:AgricultureFacility", + "informationMode": "Direct", + "languages": [ + "en", + "mr" + ], + "facilityType": "SoilTestingFacility", + "address": { + "streetAddress": "District Soil Survey and Soil Testing Laboratory, Pumping Station Road, Bhutkarwadi", + "addressLocality": "Ahilyanagar", + "addressRegion": "Maharashtra", + "addressCountry": "IN" + }, + "services": [ + { + "code": "SOIL_TESTING", + "name": "Soil sample testing", + "shortDesc": "Laboratory testing of soil samples for agricultural use." + } + ], + "source": { + "sourceId": "provider:agriculture-common-services", + "sourceName": "Agriculture Common Services Provider" + }, + "lastUpdatedAt": "2026-08-29T00:00:00Z" +} diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/warehouse.json b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/warehouse.json new file mode 100644 index 00000000..47d1f76f --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/warehouse.json @@ -0,0 +1,40 @@ +{ + "@context": "https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld", + "@type": "openagrinet:AgricultureFacility", + "informationMode": "Direct", + "languages": [ + "en", + "mr" + ], + "facilityType": "Warehouse", + "address": { + "streetAddress": "Survey No. 142/2, Musalgaon MIDC Area", + "addressLocality": "Musalgaon", + "addressRegion": "Maharashtra", + "postalCode": "422112", + "addressCountry": "IN", + "extendedAddress": "Sinnar, Nashik district" + }, + "services": [ + { + "code": "GENERAL_STORAGE", + "name": "General agricultural storage" + } + ], + "capacity": { + "value": 500, + "unit": "metric tonne", + "basis": "total storage" + }, + "publicContact": { + "company": "Example Warehouse Services", + "phone": "+910000000000", + "email": "warehouse@example.org" + }, + "website": "https://warehouse.example.org/facilities/musalgaon", + "source": { + "sourceId": "provider:warehouse-services", + "sourceName": "Warehouse Services Provider" + }, + "lastUpdatedAt": "2026-08-29T00:00:00Z" +} diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureResource/v0.1/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureResource/v0.1/attributes.yaml new file mode 100644 index 00000000..3fd3984d --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureResource/v0.1/attributes.yaml @@ -0,0 +1,197 @@ +openapi: 3.1.1 +info: + title: OpenAgriNet - Agriculture Resource Attributes + version: 0.1.0 + description: > + Shared agriculture attributes for information published on OpenAgriNet. + Attaches to beckn:Resource through resourceAttributes. + +components: + schemas: + AgricultureResource: + type: object + x-beckn-container: resourceAttributes + x-jsonld: + "@context": "https://openagrinet.github.io/network-specs/schema/AgricultureResource/v0.1/context.jsonld" + "@type": openagrinet:AgricultureResource + required: [informationMode] + properties: + informationMode: + type: string + enum: [OnDemand, Direct] + description: Whether obtaining specific information requires a Provider invocation or the Resource already contains or directly references that information. + x-jsonld-id: "openagrinet:informationMode" + subjectCategories: + type: array + minItems: 1 + uniqueItems: true + description: Optional broad agricultural subjects used for discovery. The Resource type is expressed by @type, not repeated here. + items: + type: string + enum: [Crop, Livestock, Weather, Market, Scheme, Practice] + x-jsonld-id: "openagrinet:subjectCategories" + agricultureSubjects: + type: array + minItems: 1 + uniqueItems: true + description: Optional governed subjects such as a named crop, livestock species, pest, or scheme. Absence means the Resource applies broadly within subjectCategories. + items: + $ref: "#/components/schemas/AgricultureSubjectReference" + x-jsonld-id: "openagrinet:agricultureSubjects" + languages: + type: array + minItems: 1 + uniqueItems: true + description: BCP 47 language tags supported by or present in the Resource. + items: + type: string + minLength: 2 + x-jsonld-id: "openagrinet:languages" + coverageAreas: + type: array + minItems: 1 + uniqueItems: true + description: Geographic areas within which the Resource applies. Each item is either an administrative area reference or a GeoJSON geometry. + items: + oneOf: + - $ref: "#/components/schemas/AdministrativeAreaReference" + - $ref: "#/components/schemas/CompleteGeoJSONGeometry" + x-jsonld-id: "openagrinet:coverageAreas" + + AdministrativeAreaReference: + type: object + description: A governed administrative or postal area identified within a stated code system. Use a GeoJSON geometry instead when coverage is defined by coordinates. + required: [codeScheme, areaCode] + properties: + codeScheme: + type: string + minLength: 1 + description: Geographic code system, such as ISO-3166-1, ISO-3166-2, LGD, or IN-PIN. + x-jsonld-id: "openagrinet:areaCodeScheme" + areaCode: + type: string + minLength: 1 + description: Area identifier in the stated scheme. + x-jsonld-id: "openagrinet:areaCode" + areaLevel: + type: string + enum: [Country, State, District, Block, Village, PostalCode, Other] + description: Administrative or postal level represented by the code when known. + x-jsonld-id: "openagrinet:areaLevel" + areaName: + type: string + minLength: 1 + description: Optional human-readable area name. + x-jsonld-id: "openagrinet:areaName" + additionalProperties: false + + AgricultureSubjectReference: + type: object + description: Stable reference to a governed agriculture subject and its human-readable Beckn descriptor. + required: [subjectId, subjectType, descriptor] + properties: + subjectId: + type: string + format: uri + description: Stable identifier from a governed agriculture taxonomy. + x-jsonld-id: "openagrinet:subjectId" + subjectType: + type: string + enum: [Crop, Livestock, Commodity, Pest, Disease, Practice, Scheme, Market] + description: Kind of agricultural subject referenced by subjectId. + x-jsonld-id: "openagrinet:subjectType" + descriptor: + $ref: "#/components/schemas/IdentifiedDescriptor" + description: Beckn descriptor carrying the governed code and display name. + x-jsonld-id: "https://schema.beckn.io/descriptor" + additionalProperties: false + + SourceReference: + type: object + description: Reference to the authoritative upstream source of information. + required: [sourceId] + properties: + sourceId: + type: string + minLength: 1 + description: Stable identifier of the upstream source. + x-jsonld-id: "openagrinet:sourceId" + sourceName: + type: string + minLength: 1 + description: Human-readable source name. + x-jsonld-id: "openagrinet:sourceName" + sourceUri: + type: string + format: uri + description: Authoritative public or Provider-supplied source URI. + x-jsonld-id: "openagrinet:sourceUri" + additionalProperties: false + + TimePeriod: + type: object + description: Inclusive validity or applicability period. + minProperties: 1 + properties: + startsAt: + type: string + format: date-time + description: Start of the period. + x-jsonld-id: "openagrinet:startsAt" + endsAt: + type: string + format: date-time + description: End of the period. + x-jsonld-id: "openagrinet:endsAt" + additionalProperties: false + + ClosedTimePeriod: + allOf: + - $ref: "#/components/schemas/TimePeriod" + - type: object + required: [startsAt, endsAt] + description: A bounded applicability period with both start and end timestamps. Conformance also verifies that startsAt is not after endsAt. + + IdentifiedDescriptor: + allOf: + - $ref: "https://schema.beckn.io/Descriptor/v2.1/attributes.yaml#/components/schemas/Descriptor" + - type: object + anyOf: + - required: [code] + - required: [name] + description: A Beckn Descriptor with at least a machine-readable code or human-readable name. + + CompleteGeoJSONGeometry: + allOf: + - $ref: "https://schema.beckn.io/GeoJSONGeometry/v2.0/attributes.yaml#/components/schemas/GeoJSONGeometry" + - oneOf: + - type: object + properties: + type: + const: GeometryCollection + required: [type, geometries] + - type: object + properties: + type: + enum: [Point, LineString, Polygon, MultiPoint, MultiLineString, MultiPolygon] + required: [type, coordinates] + description: A GeoJSON geometry that includes coordinates, or geometries for a GeometryCollection. + + NonEmptyAddress: + allOf: + - $ref: "https://schema.beckn.io/Address/v2.0/attributes.yaml#/components/schemas/Address" + - type: object + minProperties: 1 + description: A Beckn postal address containing at least one address field. + + CompleteLocation: + allOf: + - $ref: "https://schema.beckn.io/Location/v2.0/attributes.yaml#/components/schemas/Location" + - type: object + required: [geo] + properties: + geo: + $ref: "#/components/schemas/CompleteGeoJSONGeometry" + address: + $ref: "#/components/schemas/NonEmptyAddress" + description: A Beckn Location with a complete GeoJSON geometry and an optional non-empty address. diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Address/v2.0/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Address/v2.0/attributes.yaml new file mode 100644 index 00000000..e6517b90 --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Address/v2.0/attributes.yaml @@ -0,0 +1,45 @@ +openapi: 3.1.1 +info: + title: Address + version: 2.0.0 + description: "Container schemas fetched from beckn.yaml. This cannot be extended as it is a reserved schema in beckn protocol. Any additional properties added to this schema can only be made using its *Attributes property" + license: + name: CC-BY-NC-SA 4.0 International + contact: + name: Beckn Labs + url: https://beckn.io +components: + schemas: + Address: + x-iri: https://schema.nfh.global/Address/v2.0 + description: '**Postal address** aligned with schema.org `PostalAddress`. Use for human-readable addresses. Geometry lives in `Location.geo` as GeoJSON.' + title: Address + type: object + properties: + addressCountry: + description: Country name or ISO-3166-1 alpha-2 code. + type: string + example: IN + addressLocality: + description: City/locality. + type: string + example: Bengaluru + addressRegion: + description: State/region/province. + type: string + example: Karnataka + extendedAddress: + description: Address extension (apt/suite/floor, C/O). + type: string + example: Apt 4B + postalCode: + description: Postal/ZIP code. + type: string + example: '560001' + streetAddress: + description: Street address (building name/number and street). + type: string + example: 123 Tech Street + additionalProperties: false + x-tags: + - common diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Contact/v2.0/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Contact/v2.0/attributes.yaml new file mode 100644 index 00000000..f2564773 --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Contact/v2.0/attributes.yaml @@ -0,0 +1,36 @@ +openapi: 3.1.1 +info: + title: Contact + version: 2.0.0 + description: Schema for Contact in Beckn Logistics domain. + license: + name: CC-BY-NC-SA 4.0 International + contact: + name: Beckn Labs + url: https://beckn.io +components: + schemas: + Contact: + x-iri: https://schema.nfh.global/Contact/v2.0 + type: object + title: Contact + description: Contact information for sender, receiver, driver, or operator. + properties: + name: + type: string + example: Priya Mehta + phone: + type: string + example: '+919876543210' + email: + type: string + format: email + example: priya.mehta@email.com + alternatePhone: + type: string + company: + type: string + description: Company name if B2B contact + gstNumber: + type: string + description: GST number for B2B invoicing diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Descriptor/v2.1/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Descriptor/v2.1/attributes.yaml new file mode 100644 index 00000000..0d53150d --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Descriptor/v2.1/attributes.yaml @@ -0,0 +1,52 @@ +openapi: 3.1.1 +info: + title: Descriptor + version: 2.1.0 + description: "Container schemas fetched from beckn.yaml. This cannot be extended as it is a reserved schema in beckn protocol. Any additional properties added to this schema can only be made using its *Attributes property" + license: + name: CC-BY-NC-SA 4.0 International + contact: + name: Beckn Labs + url: https://beckn.io +components: + schemas: + Descriptor: + x-iri: https://schema.nfh.global/Descriptor/v2.1 + description: Schema definition for Descriptor in the Beckn Protocol v2.0.1 + title: Descriptor + type: object + properties: + code: + description: A machine-readable code identifying the state or type of the + entity being described. The valid values for this field are defined by + the context in which the Descriptor is used (e.g. DRAFT, ACTIVE, CANCELLED, + COMPLETE for a Contract status). + type: string + longDesc: + description: Detailed description of the item + type: string + example: Powerful gaming laptop with NVIDIA RTX graphics, fast SSD storage, + and high-refresh display + shortDesc: + description: Short description of the item + type: string + example: High-performance gaming laptop with RTX graphics + name: + description: Name of the entity being described + type: string + example: Premium Gaming Laptop Pro + thumbnailImage: + description: Name of the entity being described + type: string + format: uri + docs: + description: Links to downloadable business/supporting documents + type: array + items: + $ref: https://schema.nfh.global/Document/v2.0/attributes.yaml#/components/schemas/Document + mediaFile: + description: Links to multimedia files and images + type: array + items: + $ref: https://schema.nfh.global/MediaFile/v2.0/attributes.yaml#/components/schemas/MediaFile + additionalProperties: false diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/GeoJSONGeometry/v2.0/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/GeoJSONGeometry/v2.0/attributes.yaml new file mode 100644 index 00000000..d5045971 --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/GeoJSONGeometry/v2.0/attributes.yaml @@ -0,0 +1,57 @@ +openapi: 3.1.1 +info: + title: GeoJSONGeometry + version: 2.0.0 + description: "Container schemas fetched from beckn.yaml. This cannot be extended as it is a reserved schema in beckn protocol. Any additional properties added to this schema can only be made using its *Attributes property" + license: + name: CC-BY-NC-SA 4.0 International + contact: + name: Beckn Labs + url: https://beckn.io +components: + schemas: + GeoJSONGeometry: + x-iri: https://schema.nfh.global/GeoJSONGeometry/v2.0 + description: "**GeoJSON geometry** per RFC 7946. Coordinates are in **EPSG:4326\ + \ (WGS-84)** and MUST follow **[longitude, latitude, (altitude?)]** order. \ + Supported types: - Point, LineString, Polygon - MultiPoint, MultiLineString,\ + \ MultiPolygon - GeometryCollection (uses `geometries` instead of `coordinates`) \ + Notes: - For rectangles, use a Polygon with a single linear ring where the\ + \ first and last positions are identical. - Circles are **not native**\ + \ to GeoJSON. For circular searches, use `SpatialConstraint` with `op: s_dwithin`\ + \ and a Point + `distanceMeters`, or approximate the circle as a Polygon. \ + - Optional `bbox` is `[west, south, east, north]` in degrees." + title: GeoJSONGeometry + type: object + properties: + bbox: + description: Optional bounding box `[west, south, east, north]` in degrees. + type: array + minItems: 4 + maxItems: 4 + items: + type: number + coordinates: + description: Coordinates per RFC 7946 for all types **except** GeometryCollection. + Order is **[lon, lat, (alt)]**. For Polygons, this is an array of linear + rings; each ring is an array of positions. + type: array + items: {} + geometries: + description: Member geometries when `type` is **GeometryCollection**. + type: array + items: + $ref: https://schema.nfh.global/GeoJSONGeometry/v2.0/attributes.yaml#/components/schemas/GeoJSONGeometry + type: + type: string + enum: + - Point + - LineString + - Polygon + - MultiPoint + - MultiLineString + - MultiPolygon + - GeometryCollection + required: + - type + additionalProperties: true diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Location/v2.0/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Location/v2.0/attributes.yaml new file mode 100644 index 00000000..c7636123 --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Location/v2.0/attributes.yaml @@ -0,0 +1,26 @@ +openapi: 3.1.1 +info: + title: Location + version: 2.0.0 + description: "Container schemas fetched from beckn.yaml. This cannot be extended as it is a reserved schema in beckn protocol. Any additional properties added to this schema can only be made using its *Attributes property" + license: + name: CC-BY-NC-SA 4.0 International + contact: + name: Beckn Labs + url: https://beckn.io +components: + schemas: + Location: + x-iri: https://schema.nfh.global/Location/v2.0 + description: 'A place represented by GeoJSON geometry and optional address. + + Source: main/schema/core/v2/attributes.yaml#Location' + type: object + required: + - geo + additionalProperties: false + properties: + geo: + $ref: https://schema.nfh.global/GeoJSONGeometry/v2.0/attributes.yaml#/components/schemas/GeoJSONGeometry + address: + $ref: https://schema.nfh.global/Address/v2.0/attributes.yaml#/components/schemas/Address diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/Address/v2.0/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/Address/v2.0/attributes.yaml new file mode 100644 index 00000000..e6517b90 --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/Address/v2.0/attributes.yaml @@ -0,0 +1,45 @@ +openapi: 3.1.1 +info: + title: Address + version: 2.0.0 + description: "Container schemas fetched from beckn.yaml. This cannot be extended as it is a reserved schema in beckn protocol. Any additional properties added to this schema can only be made using its *Attributes property" + license: + name: CC-BY-NC-SA 4.0 International + contact: + name: Beckn Labs + url: https://beckn.io +components: + schemas: + Address: + x-iri: https://schema.nfh.global/Address/v2.0 + description: '**Postal address** aligned with schema.org `PostalAddress`. Use for human-readable addresses. Geometry lives in `Location.geo` as GeoJSON.' + title: Address + type: object + properties: + addressCountry: + description: Country name or ISO-3166-1 alpha-2 code. + type: string + example: IN + addressLocality: + description: City/locality. + type: string + example: Bengaluru + addressRegion: + description: State/region/province. + type: string + example: Karnataka + extendedAddress: + description: Address extension (apt/suite/floor, C/O). + type: string + example: Apt 4B + postalCode: + description: Postal/ZIP code. + type: string + example: '560001' + streetAddress: + description: Street address (building name/number and street). + type: string + example: 123 Tech Street + additionalProperties: false + x-tags: + - common diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/Document/v2.0/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/Document/v2.0/attributes.yaml new file mode 100644 index 00000000..ef0f1e88 --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/Document/v2.0/attributes.yaml @@ -0,0 +1,56 @@ +openapi: 3.1.1 +info: + title: Document + version: 2.0.0 + description: "Container schemas fetched from beckn.yaml. This cannot be extended as it is a reserved schema in beckn protocol. Any additional properties added to this schema can only be made using its *Attributes property" + license: + name: CC-BY-NC-SA 4.0 International + contact: + name: Beckn Labs + url: https://beckn.io +components: + schemas: + Document: + x-iri: https://schema.nfh.global/Document/v2.0 + description: A document, that can be parsed, printed, download or displayed. This has intentionally been kept separate from MediaFile as they may contain additional attributes like signature, schema etc. + title: Document + x-tags: + - common + type: object + properties: + label: + description: The display name of the media file + type: string + mimeType: + description: MIME type if 'data' is provided (application/pdf, image/png, application/json). + type: string + standard: + description: Describes the schema type in case the document follows a standard schema like a verifiableCredential + type: string + url: + description: URL to the document + type: string + format: uri + security: + type: object + properties: + checksum: + type: string + checksumAlgorithm: + type: string + enum: + - SHA256 + - SHA512 + - OTHER + signature: + type: string + signatureAlgorithm: + type: string + description: e.g., Ed25519, ES256 + keyId: + type: string + required: + - label + - url + - mimeType + additionalProperties: false diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/GeoJSONGeometry/v2.0/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/GeoJSONGeometry/v2.0/attributes.yaml new file mode 100644 index 00000000..d5045971 --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/GeoJSONGeometry/v2.0/attributes.yaml @@ -0,0 +1,57 @@ +openapi: 3.1.1 +info: + title: GeoJSONGeometry + version: 2.0.0 + description: "Container schemas fetched from beckn.yaml. This cannot be extended as it is a reserved schema in beckn protocol. Any additional properties added to this schema can only be made using its *Attributes property" + license: + name: CC-BY-NC-SA 4.0 International + contact: + name: Beckn Labs + url: https://beckn.io +components: + schemas: + GeoJSONGeometry: + x-iri: https://schema.nfh.global/GeoJSONGeometry/v2.0 + description: "**GeoJSON geometry** per RFC 7946. Coordinates are in **EPSG:4326\ + \ (WGS-84)** and MUST follow **[longitude, latitude, (altitude?)]** order. \ + Supported types: - Point, LineString, Polygon - MultiPoint, MultiLineString,\ + \ MultiPolygon - GeometryCollection (uses `geometries` instead of `coordinates`) \ + Notes: - For rectangles, use a Polygon with a single linear ring where the\ + \ first and last positions are identical. - Circles are **not native**\ + \ to GeoJSON. For circular searches, use `SpatialConstraint` with `op: s_dwithin`\ + \ and a Point + `distanceMeters`, or approximate the circle as a Polygon. \ + - Optional `bbox` is `[west, south, east, north]` in degrees." + title: GeoJSONGeometry + type: object + properties: + bbox: + description: Optional bounding box `[west, south, east, north]` in degrees. + type: array + minItems: 4 + maxItems: 4 + items: + type: number + coordinates: + description: Coordinates per RFC 7946 for all types **except** GeometryCollection. + Order is **[lon, lat, (alt)]**. For Polygons, this is an array of linear + rings; each ring is an array of positions. + type: array + items: {} + geometries: + description: Member geometries when `type` is **GeometryCollection**. + type: array + items: + $ref: https://schema.nfh.global/GeoJSONGeometry/v2.0/attributes.yaml#/components/schemas/GeoJSONGeometry + type: + type: string + enum: + - Point + - LineString + - Polygon + - MultiPoint + - MultiLineString + - MultiPolygon + - GeometryCollection + required: + - type + additionalProperties: true diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/MediaFile/v2.0/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/MediaFile/v2.0/attributes.yaml new file mode 100644 index 00000000..7906ca96 --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/MediaFile/v2.0/attributes.yaml @@ -0,0 +1,32 @@ +openapi: 3.1.1 +info: + title: MediaFile + version: 2.0.0 + description: "Container schemas fetched from beckn.yaml. This cannot be extended as it is a reserved schema in beckn protocol. Any additional properties added to this schema can only be made using its *Attributes property" + license: + name: CC-BY-NC-SA 4.0 International + contact: + name: Beckn Labs + url: https://beckn.io +components: + schemas: + MediaFile: + x-iri: https://schema.nfh.global/MediaFile/v2.0 + description: A image, audio, or video typically intended for display purposes + title: MediaFile + x-tags: + - common + type: object + properties: + label: + description: The display name of the media file + type: string + mimeType: + description: MIME type if 'data' is provided (application/pdf, image/png). + type: string + example: image/png + uri: + description: URL to the document + type: string + format: uri + additionalProperties: false diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/KnowledgeAdvisory.go b/pkg/plugin/implementation/KnowledgeAdvisory/KnowledgeAdvisory.go index 1f27849d..e114da95 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/KnowledgeAdvisory.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/KnowledgeAdvisory.go @@ -6,7 +6,7 @@ // // Almost nothing lives here, and that is the point. Recognising a capability, // resolving the call plan, authenticating, calling with the registry's budget -// and translating in both directions are all internal/upstream's, because none +// and translating in both directions are all internal/common's, because none // of them differ by domain. What this package owns is its name. // // The upstream this was written against is a retrieval service: it takes a @@ -20,12 +20,12 @@ import ( "context" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" ) // Config is upstream's, unchanged. Aliased here so a domain plugin's cmd // package need not know where the machinery lives. -type Config = upstream.Config +type Config = common.Config // New creates the knowledge advisory step. // @@ -33,5 +33,5 @@ type Config = upstream.Config // serving a family cannot guess which of them a deployment has providers for. func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper definition.Mapper, cfg *Config) (definition.Step, func() error, error) { - return upstream.New(ctx, registry, mapper, prerequisites, cfg) + return common.New(ctx, registry, mapper, prerequisites, cfg) } diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin.go b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin.go index 104a6066..9683969b 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin.go @@ -16,7 +16,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/KnowledgeAdvisory" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" ) // knowledgeAdvisoryProvider implements definition.ProviderStepProvider. @@ -31,7 +31,7 @@ var newStepFunc = KnowledgeAdvisory.New func (p knowledgeAdvisoryProvider) parseConfig(config map[string]string) (*KnowledgeAdvisory.Config, error) { cfg := &KnowledgeAdvisory.Config{ BindingKeys: splitList(config["bindingKeys"]), - // Absent means the Beckn v2 convention. See upstream.Config for why + // Absent means the Beckn v2 convention. See common.Config for why // this is a default rather than something to set. ProviderIDAt: config["providerIdAt"], CapabilityCodeAt: config["capabilityCodeAt"], @@ -52,7 +52,7 @@ func (p knowledgeAdvisoryProvider) parseConfig(config map[string]string) (*Knowl // authScheme- settings. Shared with the other capability // plugins: each used to copy the same field list, so a scheme added in one // had to be remembered in three. - auth, err := upstream.ParseProviderAuth(config) + auth, err := common.ParseProviderAuth(config) if err != nil { return nil, err } diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin_test.go index 2f65319b..8a0443cf 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin_test.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/cmd/plugin_test.go @@ -10,7 +10,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/KnowledgeAdvisory" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" ) type stubRegistry struct{} @@ -42,7 +42,7 @@ func TestParseConfig(t *testing.T) { // empty map rather than nil -- ParseProviderAuth always returns one. name: "leaves everything unset for New to default", config: map[string]string{}, - expected: &KnowledgeAdvisory.Config{AuthByProvider: map[string]*upstream.AuthProfile{}}, + expected: &KnowledgeAdvisory.Config{AuthByProvider: map[string]*common.AuthProfile{}}, }, { // The scheme this capability exists to use. Its provider sits @@ -58,7 +58,7 @@ func TestParseConfig(t *testing.T) { }, expected: &KnowledgeAdvisory.Config{ BindingKeys: []string{"knowledge-provider|openagrinet:KnowledgeAdvisory"}, - AuthByProvider: map[string]*upstream.AuthProfile{ + AuthByProvider: map[string]*common.AuthProfile{ "knowledge-provider": { Provider: "knowledge-provider", Scheme: "oauth2", @@ -90,7 +90,7 @@ func TestParseConfig(t *testing.T) { "knowledge-provider|openagrinet:KnowledgeAdvisory", "vistaar-two|openagrinet:KnowledgeAdvisory", }, - AuthByProvider: map[string]*upstream.AuthProfile{ + AuthByProvider: map[string]*common.AuthProfile{ "knowledge-provider": { Provider: "knowledge-provider", Scheme: "oauth2", @@ -119,7 +119,7 @@ func TestParseConfig(t *testing.T) { }, expected: &KnowledgeAdvisory.Config{ BindingKeys: []string{"provider.oan.dev|openagrinet:KnowledgeAdvisory"}, - AuthByProvider: map[string]*upstream.AuthProfile{ + AuthByProvider: map[string]*common.AuthProfile{ "provider.oan.dev": { Provider: "provider.oan.dev", Scheme: "basic", @@ -174,7 +174,7 @@ func TestParseConfig(t *testing.T) { }, expected: &KnowledgeAdvisory.Config{ BindingKeys: []string{"other|capability"}, - AuthByProvider: map[string]*upstream.AuthProfile{ + AuthByProvider: map[string]*common.AuthProfile{ "other": { Provider: "other", Scheme: "basic", @@ -207,7 +207,7 @@ func TestParseConfig(t *testing.T) { // unset variable produces this, and it should read as "unset". name: "treats an empty response cap as unset", config: map[string]string{"maxResponseBytes": ""}, - expected: &KnowledgeAdvisory.Config{AuthByProvider: map[string]*upstream.AuthProfile{}}, + expected: &KnowledgeAdvisory.Config{AuthByProvider: map[string]*common.AuthProfile{}}, }, } diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/live_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/live_test.go index e9f18d49..70bf2f25 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/live_test.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/live_test.go @@ -15,7 +15,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/KnowledgeAdvisory" - upstreamstep "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" ) @@ -103,7 +103,7 @@ func TestLiveEndToEnd(t *testing.T) { step, closeStep, err := KnowledgeAdvisory.New(context.Background(), registry, mapper, &KnowledgeAdvisory.Config{ BindingKeys: []string{key}, - AuthByProvider: map[string]*upstreamstep.AuthProfile{ + AuthByProvider: map[string]*common.AuthProfile{ strings.Split(key, "|")[0]: { Scheme: "oauth2", TokenURL: issuer.URL + "/token", diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go index 3be5aede..9c06c18e 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go @@ -22,7 +22,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/KnowledgeAdvisory" - upstreamstep "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" ) @@ -212,8 +212,8 @@ func runShippedWith(t *testing.T, request, providerBody string) (map[string]any, BindingKeys: []string{shippedBindingKey}, // Auth is per provider; these upstreams are stubs needing // no credential, and that is declared rather than defaulted. - AuthByProvider: map[string]*upstreamstep.AuthProfile{ - strings.Split(shippedBindingKey, "|")[0]: {Scheme: upstreamstep.AuthSchemeNone}, + AuthByProvider: map[string]*common.AuthProfile{ + strings.Split(shippedBindingKey, "|")[0]: {Scheme: common.AuthSchemeNone}, }, }) if err != nil { diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/prerequisites.go b/pkg/plugin/implementation/KnowledgeAdvisory/prerequisites.go index 553f921e..f99241a5 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/prerequisites.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/prerequisites.go @@ -1,6 +1,6 @@ package KnowledgeAdvisory -import "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" +import "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" // prerequisites is what a knowledge advisory capability needs that its payload // does not carry, keyed by binding key. @@ -19,4 +19,4 @@ import "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstr // mapping cannot obtain: a free-text topic resolved to a governed taxonomy id, // say. The corpus index is not one -- it is a deployment constant, and the // mapping file is itself per-deployment, named by the registry row. -var prerequisites = upstream.Prerequisites{} +var prerequisites = common.Prerequisites{} diff --git a/pkg/plugin/implementation/MandiPrice/MandiPrice.go b/pkg/plugin/implementation/MandiPrice/MandiPrice.go index 38fff611..d1996f99 100644 --- a/pkg/plugin/implementation/MandiPrice/MandiPrice.go +++ b/pkg/plugin/implementation/MandiPrice/MandiPrice.go @@ -6,7 +6,7 @@ // // Almost nothing lives here, and that is the point. Recognising a capability, // resolving the call plan, authenticating, calling with the registry's budget -// and translating in both directions are all internal/upstream's, because none +// and translating in both directions are all internal/common's, because none // of them differ by domain. What this package owns is its name, and // prerequisites -- the work a mapping cannot express, which is domain knowledge // by definition. @@ -21,12 +21,12 @@ import ( "context" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" ) // Config is upstream's, unchanged. Aliased here so a domain plugin's cmd package // need not know where the machinery lives. -type Config = upstream.Config +type Config = common.Config // New creates the mandi step. // @@ -34,5 +34,5 @@ type Config = upstream.Config // serving a family cannot guess which of them a deployment has providers for. func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper definition.Mapper, cfg *Config) (definition.Step, func() error, error) { - return upstream.New(ctx, registry, mapper, prerequisites, cfg) + return common.New(ctx, registry, mapper, prerequisites, cfg) } diff --git a/pkg/plugin/implementation/MandiPrice/cmd/plugin.go b/pkg/plugin/implementation/MandiPrice/cmd/plugin.go index e3aee941..865c3053 100644 --- a/pkg/plugin/implementation/MandiPrice/cmd/plugin.go +++ b/pkg/plugin/implementation/MandiPrice/cmd/plugin.go @@ -14,7 +14,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/MandiPrice" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" ) // mandiProvider implements definition.ProviderStepProvider. @@ -29,7 +29,7 @@ var newStepFunc = MandiPrice.New func (p mandiProvider) parseConfig(config map[string]string) (*MandiPrice.Config, error) { cfg := &MandiPrice.Config{ BindingKeys: splitList(config["bindingKeys"]), - // Absent means the Beckn v2 convention. See upstream.Config for why + // Absent means the Beckn v2 convention. See common.Config for why // this is a default rather than something to set. ProviderIDAt: config["providerIdAt"], CapabilityCodeAt: config["capabilityCodeAt"], @@ -50,7 +50,7 @@ func (p mandiProvider) parseConfig(config map[string]string) (*MandiPrice.Config // authScheme- settings. Shared with the other capability // plugins: each used to copy the same field list, so a scheme added in one // had to be remembered in three. - auth, err := upstream.ParseProviderAuth(config) + auth, err := common.ParseProviderAuth(config) if err != nil { return nil, err } diff --git a/pkg/plugin/implementation/MandiPrice/cmd/plugin_test.go b/pkg/plugin/implementation/MandiPrice/cmd/plugin_test.go index 3b5509a7..9c72e126 100644 --- a/pkg/plugin/implementation/MandiPrice/cmd/plugin_test.go +++ b/pkg/plugin/implementation/MandiPrice/cmd/plugin_test.go @@ -10,7 +10,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/MandiPrice" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" ) type stubRegistry struct{} @@ -41,7 +41,7 @@ func TestParseConfig(t *testing.T) { // rules are defined in exactly one place. name: "leaves everything unset for New to default", config: map[string]string{}, - expected: &MandiPrice.Config{AuthByProvider: map[string]*upstream.AuthProfile{}}, + expected: &MandiPrice.Config{AuthByProvider: map[string]*common.AuthProfile{}}, }, { // Query auth is why this capability has its own entry rather than @@ -57,7 +57,7 @@ func TestParseConfig(t *testing.T) { }, expected: &MandiPrice.Config{ BindingKeys: []string{"agmarknet|openagrinet:MandiPrice"}, - AuthByProvider: map[string]*upstream.AuthProfile{ + AuthByProvider: map[string]*common.AuthProfile{ "agmarknet": { Provider: "agmarknet", Scheme: "query", @@ -82,7 +82,7 @@ func TestParseConfig(t *testing.T) { }, expected: &MandiPrice.Config{ BindingKeys: []string{"other|capability"}, - AuthByProvider: map[string]*upstream.AuthProfile{ + AuthByProvider: map[string]*common.AuthProfile{ "other": { Provider: "other", Scheme: "basic", @@ -113,7 +113,7 @@ func TestParseConfig(t *testing.T) { // as "unset" rather than failing startup. name: "treats an empty response cap as unset", config: map[string]string{"maxResponseBytes": ""}, - expected: &MandiPrice.Config{AuthByProvider: map[string]*upstream.AuthProfile{}}, + expected: &MandiPrice.Config{AuthByProvider: map[string]*common.AuthProfile{}}, }, } diff --git a/pkg/plugin/implementation/MandiPrice/mappings_test.go b/pkg/plugin/implementation/MandiPrice/mappings_test.go index 20307294..75299068 100644 --- a/pkg/plugin/implementation/MandiPrice/mappings_test.go +++ b/pkg/plugin/implementation/MandiPrice/mappings_test.go @@ -23,7 +23,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/MandiPrice" - upstreamstep "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" ) @@ -197,8 +197,8 @@ func runShippedWith(t *testing.T, request, providerBody string) (url.Values, map BindingKeys: []string{shippedBindingKey}, // Auth is per provider; these upstreams are stubs needing // no credential, and that is declared rather than defaulted. - AuthByProvider: map[string]*upstreamstep.AuthProfile{ - strings.Split(shippedBindingKey, "|")[0]: {Scheme: upstreamstep.AuthSchemeNone}, + AuthByProvider: map[string]*common.AuthProfile{ + strings.Split(shippedBindingKey, "|")[0]: {Scheme: common.AuthSchemeNone}, }, }) if err != nil { @@ -444,8 +444,8 @@ func TestShippedMappingRefusesWhatItCannotServe(t *testing.T) { BindingKeys: []string{shippedBindingKey}, // Auth is per provider; these upstreams are stubs needing // no credential, and that is declared rather than defaulted. - AuthByProvider: map[string]*upstreamstep.AuthProfile{ - strings.Split(shippedBindingKey, "|")[0]: {Scheme: upstreamstep.AuthSchemeNone}, + AuthByProvider: map[string]*common.AuthProfile{ + strings.Split(shippedBindingKey, "|")[0]: {Scheme: common.AuthSchemeNone}, }, }) if err != nil { @@ -760,8 +760,8 @@ func TestShippedMappingRefusesPayloadsItCannotAnswer(t *testing.T) { BindingKeys: []string{shippedBindingKey}, // Auth is per provider; these upstreams are stubs needing // no credential, and that is declared rather than defaulted. - AuthByProvider: map[string]*upstreamstep.AuthProfile{ - strings.Split(shippedBindingKey, "|")[0]: {Scheme: upstreamstep.AuthSchemeNone}, + AuthByProvider: map[string]*common.AuthProfile{ + strings.Split(shippedBindingKey, "|")[0]: {Scheme: common.AuthSchemeNone}, }, }) if err != nil { diff --git a/pkg/plugin/implementation/MandiPrice/prerequisites.go b/pkg/plugin/implementation/MandiPrice/prerequisites.go index d7c1f8aa..3c365a39 100644 --- a/pkg/plugin/implementation/MandiPrice/prerequisites.go +++ b/pkg/plugin/implementation/MandiPrice/prerequisites.go @@ -1,6 +1,6 @@ package MandiPrice -import "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" +import "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" // prerequisites is what a mandi capability needs that its payload does not // carry, keyed by binding key. @@ -17,4 +17,4 @@ import "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstr // bring the question of where the provider-to-function binding belongs -- see // the note in WeatherObservation/prerequisites.go and prefer keeping the payload explicit // over adding an entry here. -var prerequisites = upstream.Prerequisites{} +var prerequisites = common.Prerequisites{} diff --git a/pkg/plugin/implementation/WeatherObservation/WeatherObservation.go b/pkg/plugin/implementation/WeatherObservation/WeatherObservation.go index 65820e84..5ba8ca80 100644 --- a/pkg/plugin/implementation/WeatherObservation/WeatherObservation.go +++ b/pkg/plugin/implementation/WeatherObservation/WeatherObservation.go @@ -9,7 +9,7 @@ // // Almost nothing lives here. Recognising a capability, resolving the call plan, // authenticating, calling with the registry's budget and translating in both -// directions are all internal/upstream's, because none of them differ by domain. +// directions are all internal/common's, because none of them differ by domain. // What this package owns is its name, and prerequisites -- the work a mapping // cannot express, which is domain knowledge by definition. package WeatherObservation @@ -18,12 +18,12 @@ import ( "context" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" ) // Config is upstream's, unchanged. Aliased here so a domain plugin's cmd package // need not know where the machinery lives. -type Config = upstream.Config +type Config = common.Config // New creates the weather step. // @@ -31,5 +31,5 @@ type Config = upstream.Config // serving a family cannot guess which of them a deployment has providers for. func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper definition.Mapper, cfg *Config) (definition.Step, func() error, error) { - return upstream.New(ctx, registry, mapper, prerequisites, cfg) + return common.New(ctx, registry, mapper, prerequisites, cfg) } diff --git a/pkg/plugin/implementation/WeatherObservation/cmd/plugin.go b/pkg/plugin/implementation/WeatherObservation/cmd/plugin.go index f29512fe..ddad2ff1 100644 --- a/pkg/plugin/implementation/WeatherObservation/cmd/plugin.go +++ b/pkg/plugin/implementation/WeatherObservation/cmd/plugin.go @@ -10,7 +10,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/WeatherObservation" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" ) // weatherProvider implements definition.ProviderStepProvider. @@ -25,7 +25,7 @@ var newStepFunc = WeatherObservation.New func (p weatherProvider) parseConfig(config map[string]string) (*WeatherObservation.Config, error) { cfg := &WeatherObservation.Config{ BindingKeys: splitList(config["bindingKeys"]), - // Absent means the Beckn v2 convention. See upstream.Config for why + // Absent means the Beckn v2 convention. See common.Config for why // this is a default rather than something to set. ProviderIDAt: config["providerIdAt"], CapabilityCodeAt: config["capabilityCodeAt"], @@ -46,7 +46,7 @@ func (p weatherProvider) parseConfig(config map[string]string) (*WeatherObservat // authScheme- settings. Shared with the other capability // plugins: each used to copy the same field list, so a scheme added in one // had to be remembered in three. - auth, err := upstream.ParseProviderAuth(config) + auth, err := common.ParseProviderAuth(config) if err != nil { return nil, err } diff --git a/pkg/plugin/implementation/WeatherObservation/cmd/plugin_test.go b/pkg/plugin/implementation/WeatherObservation/cmd/plugin_test.go index eaa38bcf..be757b81 100644 --- a/pkg/plugin/implementation/WeatherObservation/cmd/plugin_test.go +++ b/pkg/plugin/implementation/WeatherObservation/cmd/plugin_test.go @@ -10,7 +10,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/WeatherObservation" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" ) type stubRegistry struct{} @@ -41,7 +41,7 @@ func TestParseConfig(t *testing.T) { // rules are defined in exactly one place. name: "leaves everything unset for New to default", config: map[string]string{}, - expected: &WeatherObservation.Config{AuthByProvider: map[string]*upstream.AuthProfile{}}, + expected: &WeatherObservation.Config{AuthByProvider: map[string]*common.AuthProfile{}}, }, { name: "reads every supported setting", @@ -56,7 +56,7 @@ func TestParseConfig(t *testing.T) { }, expected: &WeatherObservation.Config{ BindingKeys: []string{"other|capability"}, - AuthByProvider: map[string]*upstream.AuthProfile{ + AuthByProvider: map[string]*common.AuthProfile{ "other": { Provider: "other", Scheme: "basic", diff --git a/pkg/plugin/implementation/WeatherObservation/mappings_test.go b/pkg/plugin/implementation/WeatherObservation/mappings_test.go index 6de4ae83..0f98413f 100644 --- a/pkg/plugin/implementation/WeatherObservation/mappings_test.go +++ b/pkg/plugin/implementation/WeatherObservation/mappings_test.go @@ -23,7 +23,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/WeatherObservation" - upstreamstep "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" ) @@ -148,8 +148,8 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { BindingKeys: []string{shippedBindingKey}, // Auth is per provider; these upstreams are stubs needing // no credential, and that is declared rather than defaulted. - AuthByProvider: map[string]*upstreamstep.AuthProfile{ - strings.Split(shippedBindingKey, "|")[0]: {Scheme: upstreamstep.AuthSchemeNone}, + AuthByProvider: map[string]*common.AuthProfile{ + strings.Split(shippedBindingKey, "|")[0]: {Scheme: common.AuthSchemeNone}, }, }) if err != nil { diff --git a/pkg/plugin/implementation/WeatherObservation/prerequisites.go b/pkg/plugin/implementation/WeatherObservation/prerequisites.go index 3492d577..d3b4c480 100644 --- a/pkg/plugin/implementation/WeatherObservation/prerequisites.go +++ b/pkg/plugin/implementation/WeatherObservation/prerequisites.go @@ -1,6 +1,6 @@ package WeatherObservation -import "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" +import "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" // prerequisites is what a weather capability needs that its payload does not // carry, keyed by binding key. @@ -21,4 +21,4 @@ import "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstr // // request: | // { "station": _local.stationId } -var prerequisites = upstream.Prerequisites{} +var prerequisites = common.Prerequisites{} diff --git a/pkg/plugin/implementation/internal/upstream/auth.go b/pkg/plugin/implementation/internal/common/auth.go similarity index 99% rename from pkg/plugin/implementation/internal/upstream/auth.go rename to pkg/plugin/implementation/internal/common/auth.go index 9a3edc3a..a016acbc 100644 --- a/pkg/plugin/implementation/internal/upstream/auth.go +++ b/pkg/plugin/implementation/internal/common/auth.go @@ -1,6 +1,6 @@ // Credentials: one profile per provider, how a profile is read from config, // and how it is presented on a request. -package upstream +package common import ( "context" diff --git a/pkg/plugin/implementation/internal/upstream/upstream.go b/pkg/plugin/implementation/internal/common/common.go similarity index 88% rename from pkg/plugin/implementation/internal/upstream/upstream.go rename to pkg/plugin/implementation/internal/common/common.go index 17a678e1..c8f4efac 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream.go +++ b/pkg/plugin/implementation/internal/common/common.go @@ -1,17 +1,29 @@ -// Package upstream serves a Beckn capability by calling an ordinary API that has +// Package common serves a Beckn capability by calling an ordinary API that has // never heard of Beckn. // -// "upstream" is the registry's own word for such an API -- a Participant of type -// upstream, as against a node that speaks Beckn. This package is the machinery -// for calling one: recognise the capability, resolve the call plan, translate -// out, call, translate back. +// Common to every capability plugin, and to nothing else: WeatherObservation, +// MandiPrice and KnowledgeAdvisory are each a name, a set of binding keys and a +// credential profile over this machinery. Recognise the capability, resolve the +// call plan, translate out, call, translate back. +// +// "UPSTREAM" IS STILL THE WORD FOR THE API BEING CALLED. It is the registry's +// own -- a Participant of type upstream, as against a node that speaks Beckn -- +// so it stays in the errors, the logs and the comments here even though the +// package no longer carries it. An operator reading "upstream: provider did not +// answer" is being told which leg of the call failed, and "common:" would tell +// them nothing. // // It holds nothing about any provider or any domain. What varies per capability // comes from the registry (endpoint, method, budget, which mapping) and from the // mapping itself (what the payload must satisfy, what to send, what to return). // A domain package wraps this, supplying only its name and whatever prerequisite // work a mapping cannot express. -package upstream +// +// NOT A DUMPING GROUND. Everything here is one subject -- calling an upstream on +// a capability's behalf. Something shared by fewer than all three plugins, or +// unrelated to that call, belongs in its own package rather than here on the +// strength of the name. +package common import ( "context" diff --git a/pkg/plugin/implementation/internal/upstream/upstream_test.go b/pkg/plugin/implementation/internal/common/common_test.go similarity index 99% rename from pkg/plugin/implementation/internal/upstream/upstream_test.go rename to pkg/plugin/implementation/internal/common/common_test.go index 0f06f8e8..56688999 100644 --- a/pkg/plugin/implementation/internal/upstream/upstream_test.go +++ b/pkg/plugin/implementation/internal/common/common_test.go @@ -1,4 +1,4 @@ -package upstream +package common import ( "context" diff --git a/pkg/plugin/implementation/internal/upstream/constant.go b/pkg/plugin/implementation/internal/common/constant.go similarity index 99% rename from pkg/plugin/implementation/internal/upstream/constant.go rename to pkg/plugin/implementation/internal/common/constant.go index 1262d607..398d57fc 100644 --- a/pkg/plugin/implementation/internal/upstream/constant.go +++ b/pkg/plugin/implementation/internal/common/constant.go @@ -3,7 +3,7 @@ // // authFields is deliberately NOT here. It is a var -- Go has no constant maps -- // and it belongs beside the parser in auth.go that reads it. -package upstream +package common import ( "time" diff --git a/pkg/plugin/implementation/internal/upstream/dispatch_test.go b/pkg/plugin/implementation/internal/common/dispatch_test.go similarity index 94% rename from pkg/plugin/implementation/internal/upstream/dispatch_test.go rename to pkg/plugin/implementation/internal/common/dispatch_test.go index 27901961..d2eea389 100644 --- a/pkg/plugin/implementation/internal/upstream/dispatch_test.go +++ b/pkg/plugin/implementation/internal/common/dispatch_test.go @@ -1,4 +1,4 @@ -package upstream_test +package common_test import ( "context" @@ -10,7 +10,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/upstream" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" ) // dispatchMappingRef stands in for the one reference an action carries. This @@ -75,16 +75,16 @@ func TestTwoProviderStepsDispatchByBindingKey(t *testing.T) { "select": {Method: http.MethodGet, Path: "/x", Mappings: dispatchMappingRef, RetryMax: 1}, }, } - step, closer, err := upstream.New(context.Background(), + step, closer, err := common.New(context.Background(), &stubRegistry{plan: plan}, fixedMapper{answer: answer}, nil, - &upstream.Config{ + &common.Config{ BindingKeys: []string{bindingKey}, // Auth is per provider, so each step declares its own even // when neither upstream needs a credential. - AuthByProvider: map[string]*upstream.AuthProfile{ - strings.Split(bindingKey, "|")[0]: {Scheme: upstream.AuthSchemeNone}, + AuthByProvider: map[string]*common.AuthProfile{ + strings.Split(bindingKey, "|")[0]: {Scheme: common.AuthSchemeNone}, }, }) if err != nil { diff --git a/pkg/plugin/implementation/internal/upstream/http.go b/pkg/plugin/implementation/internal/common/http.go similarity index 99% rename from pkg/plugin/implementation/internal/upstream/http.go rename to pkg/plugin/implementation/internal/common/http.go index 927b20b3..800078e5 100644 --- a/pkg/plugin/implementation/internal/upstream/http.go +++ b/pkg/plugin/implementation/internal/common/http.go @@ -1,6 +1,6 @@ // Talking HTTP to a provider: building the endpoint and body, the attempt and // its retry budget, and reading what came back. -package upstream +package common import ( "bytes" diff --git a/pkg/plugin/implementation/internal/upstream/perprovider_test.go b/pkg/plugin/implementation/internal/common/perprovider_test.go similarity index 99% rename from pkg/plugin/implementation/internal/upstream/perprovider_test.go rename to pkg/plugin/implementation/internal/common/perprovider_test.go index 6aa58ea3..8f32b6da 100644 --- a/pkg/plugin/implementation/internal/upstream/perprovider_test.go +++ b/pkg/plugin/implementation/internal/common/perprovider_test.go @@ -1,4 +1,4 @@ -package upstream +package common import ( "context" diff --git a/pkg/plugin/implementation/internal/upstream/redact.go b/pkg/plugin/implementation/internal/common/redact.go similarity index 99% rename from pkg/plugin/implementation/internal/upstream/redact.go rename to pkg/plugin/implementation/internal/common/redact.go index 85cda7b4..aeabab49 100644 --- a/pkg/plugin/implementation/internal/upstream/redact.go +++ b/pkg/plugin/implementation/internal/common/redact.go @@ -1,5 +1,5 @@ // Keeping credentials out of anything this package logs or returns. -package upstream +package common import ( "encoding/base64" diff --git a/pkg/plugin/implementation/internal/upstream/serve.go b/pkg/plugin/implementation/internal/common/serve.go similarity index 99% rename from pkg/plugin/implementation/internal/upstream/serve.go rename to pkg/plugin/implementation/internal/common/serve.go index 305e5f35..bbc98237 100644 --- a/pkg/plugin/implementation/internal/upstream/serve.go +++ b/pkg/plugin/implementation/internal/common/serve.go @@ -1,6 +1,6 @@ // The request pipeline: recognise the capability, resolve its call plan, // translate out, call, translate back. -package upstream +package common import ( "context" diff --git a/pkg/plugin/implementation/internal/upstream/token.go b/pkg/plugin/implementation/internal/common/token.go similarity index 99% rename from pkg/plugin/implementation/internal/upstream/token.go rename to pkg/plugin/implementation/internal/common/token.go index 624dd813..fd2b08af 100644 --- a/pkg/plugin/implementation/internal/upstream/token.go +++ b/pkg/plugin/implementation/internal/common/token.go @@ -1,5 +1,5 @@ // The oauth2 client_credentials exchange and the token each provider holds. -package upstream +package common import ( "context" From 559c4c69187b4cc556a830719d9b78b9f24935a1 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 18:24:43 +0530 Subject: [PATCH 110/130] refactor: fold capabilitybinding into common [#7] capabilitybinding had exactly ONE consumer -- internal/common, in three files -- so it was a package boundary with nothing on the other side of it. Its contents are now two files in common: capabilitybinding/capabilitybinding.go -> common/binding.go capabilitybinding/paths.go -> common/paths.go capabilitybinding_test.go -> common/binding_test.go The move forces two renames, because a name that reads well qualified does not read well bare: capabilitybinding.From -> bindingFrom From(paths, body) alone says nothing capabilitybinding.ErrNoBinding -> errNoBinding nothing outside needs it Error prefixes move from "capabilitybinding: " to "upstream: ", which is what the rest of the package already uses and what tells an operator which leg of the call failed. Two of the path messages gained the word "binding path" so they still say what was wrong: "upstream: binding path providerIdAt is empty". No test asserted on the old prefix. The package doc is no longer a package doc -- it now heads binding.go as a section comment, keeping the reason the binding lives here rather than in one capability: it is a property of the network's payloads, not of any provider. 65 packages green rather than 66, and the missing one is capabilitybinding itself -- verified by diffing the tested package lists, nothing else dropped out. Vet clean, no races. --- .../binding.go} | 30 +++++----- .../binding_test.go} | 56 +++++++++---------- .../implementation/internal/common/common.go | 15 +++-- .../internal/common/common_test.go | 3 +- .../{capabilitybinding => common}/paths.go | 6 +- .../implementation/internal/common/serve.go | 5 +- 6 files changed, 56 insertions(+), 59 deletions(-) rename pkg/plugin/implementation/internal/{capabilitybinding/capabilitybinding.go => common/binding.go} (74%) rename pkg/plugin/implementation/internal/{capabilitybinding/capabilitybinding_test.go => common/binding_test.go} (90%) rename pkg/plugin/implementation/internal/{capabilitybinding => common}/paths.go (95%) diff --git a/pkg/plugin/implementation/internal/capabilitybinding/capabilitybinding.go b/pkg/plugin/implementation/internal/common/binding.go similarity index 74% rename from pkg/plugin/implementation/internal/capabilitybinding/capabilitybinding.go rename to pkg/plugin/implementation/internal/common/binding.go index e72b1dd7..b3b745bd 100644 --- a/pkg/plugin/implementation/internal/capabilitybinding/capabilitybinding.go +++ b/pkg/plugin/implementation/internal/common/binding.go @@ -1,10 +1,10 @@ -// Package capabilitybinding derives the capability binding a Beckn request is asking -// for, so a provider step can tell whether the request is its work and, if it -// is, which registry row describes the call. +// Deriving the capability binding a Beckn request is asking for, so a step can +// tell whether the request is its work and, if it is, which registry row +// describes the call. // -// It is shared by every provider step rather than living in one, because the -// binding is a property of the network's payloads and not of any provider. -package capabilitybinding +// The binding is a property of the network's payloads, not of any provider, +// which is why it sits here rather than in one capability's package. +package common import ( "encoding/json" @@ -16,10 +16,10 @@ import ( // separator joins a binding key's two halves. const separator = "|" -// ErrNoBinding reports a payload that names no capability binding. It is not a +// errNoBinding reports a payload that names no capability binding. It is not a // fault: a request for something else entirely reaches a provider step too, and // the step's answer is to do nothing. -var ErrNoBinding = errors.New("capabilitybinding: payload names no capability binding") +var errNoBinding = errors.New("upstream: payload names no capability binding") // Binding identifies one provider capability. type Binding struct { @@ -34,7 +34,7 @@ func (b Binding) Key() string { // From derives the capability binding a payload is asking for. // -// Returns ErrNoBinding when the payload names no provider or no type, which is +// Returns errNoBinding when the payload names no provider or no type, which is // the ordinary case for a request a provider step is not meant to serve. // // A payload carrying more than one distinct provider or type is refused rather @@ -44,10 +44,10 @@ func (b Binding) Key() string { // // Where the halves live is BecknV2 unless a deployment says otherwise -- see // Paths for why that is a default and not a setting. -func From(paths Paths, body []byte) (Binding, error) { +func bindingFrom(paths Paths, body []byte) (Binding, error) { var payload any if err := json.Unmarshal(body, &payload); err != nil { - return Binding{}, fmt.Errorf("capabilitybinding: payload could not be read: %w", err) + return Binding{}, fmt.Errorf("upstream: payload could not be read: %w", err) } // Before distinctness: N commitments naming the SAME provider and type @@ -64,7 +64,7 @@ func From(paths Paths, body []byte) (Binding, error) { // the paragraph above says is refused. if commitments := countAt(payload, paths.ProviderID); commitments > 1 { return Binding{}, fmt.Errorf( - "capabilitybinding: payload carries %d commitments; one request maps to one call, "+ + "upstream: payload carries %d commitments; one request maps to one call, "+ "so send them separately rather than have all but the first dropped", commitments) } @@ -73,14 +73,14 @@ func From(paths Paths, body []byte) (Binding, error) { types := distinct(valuesAt(payload, paths.CapabilityCode)) if len(providers) == 0 || len(types) == 0 { - return Binding{}, ErrNoBinding + return Binding{}, errNoBinding } if len(providers) > 1 { - return Binding{}, fmt.Errorf("capabilitybinding: payload names %d providers (%s); one request maps to one call", + return Binding{}, fmt.Errorf("upstream: payload names %d providers (%s); one request maps to one call", len(providers), strings.Join(providers, ", ")) } if len(types) > 1 { - return Binding{}, fmt.Errorf("capabilitybinding: payload names %d resource types (%s); one request maps to one call", + return Binding{}, fmt.Errorf("upstream: payload names %d resource types (%s); one request maps to one call", len(types), strings.Join(types, ", ")) } diff --git a/pkg/plugin/implementation/internal/capabilitybinding/capabilitybinding_test.go b/pkg/plugin/implementation/internal/common/binding_test.go similarity index 90% rename from pkg/plugin/implementation/internal/capabilitybinding/capabilitybinding_test.go rename to pkg/plugin/implementation/internal/common/binding_test.go index 43213053..a7507f2a 100644 --- a/pkg/plugin/implementation/internal/capabilitybinding/capabilitybinding_test.go +++ b/pkg/plugin/implementation/internal/common/binding_test.go @@ -1,4 +1,4 @@ -package capabilitybinding +package common import ( "encoding/json" @@ -43,9 +43,9 @@ const realSelectPayload = `{ func TestFromReadsARealSelectPayload(t *testing.T) { t.Parallel() - got, err := From(BecknV2, []byte(realSelectPayload)) + got, err := bindingFrom(BecknV2, []byte(realSelectPayload)) if err != nil { - t.Fatalf("From() returned an unexpected error: %v", err) + t.Fatalf("bindingFrom() returned an unexpected error: %v", err) } if got.ParticipantID != "mausamgram" { t.Errorf("participant = %q, want mausamgram", got.ParticipantID) @@ -80,8 +80,8 @@ func TestFromReportsAPayloadWithNoBinding(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - if _, err := From(BecknV2, []byte(tc.body)); !errors.Is(err, ErrNoBinding) { - t.Errorf("expected ErrNoBinding, got %v", err) + if _, err := bindingFrom(BecknV2, []byte(tc.body)); !errors.Is(err, errNoBinding) { + t.Errorf("expected errNoBinding, got %v", err) } }) } @@ -119,12 +119,12 @@ func TestFromRefusesAnAmbiguousPayload(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - _, err := From(BecknV2, []byte(tc.body)) + _, err := bindingFrom(BecknV2, []byte(tc.body)) if err == nil { t.Fatal("expected an ambiguous payload to be refused") } - if errors.Is(err, ErrNoBinding) { - t.Error("ambiguity is not absence: it must not report ErrNoBinding") + if errors.Is(err, errNoBinding) { + t.Error("ambiguity is not absence: it must not report errNoBinding") } if !strings.Contains(err.Error(), tc.wants) { t.Errorf("error %q should say %q", err, tc.wants) @@ -136,11 +136,11 @@ func TestFromRefusesAnAmbiguousPayload(t *testing.T) { func TestFromReportsAnUnreadablePayload(t *testing.T) { t.Parallel() - _, err := From(BecknV2, []byte(`{"message":`)) + _, err := bindingFrom(BecknV2, []byte(`{"message":`)) if err == nil { t.Fatal("expected unreadable JSON to be reported") } - if errors.Is(err, ErrNoBinding) { + if errors.Is(err, errNoBinding) { t.Error("a broken payload is not an absent binding") } } @@ -161,9 +161,9 @@ func TestBecknV2IsTheDefault(t *testing.T) { if BecknV2.ProviderID == "" || BecknV2.CapabilityCode == "" { t.Fatal("the default paths must be set") } - got, err := From(BecknV2, []byte(realSelectPayload)) + got, err := bindingFrom(BecknV2, []byte(realSelectPayload)) if err != nil { - t.Fatalf("From() returned an unexpected error: %v", err) + t.Fatalf("bindingFrom() returned an unexpected error: %v", err) } if got.ParticipantID != "mausamgram" || got.CapabilityCode != "openagrinet:WeatherObservation" { t.Errorf("binding = %+v, want the Beckn v2 convention's answer", got) @@ -176,12 +176,12 @@ func TestFromReadsAnOverriddenPath(t *testing.T) { t.Parallel() body := `{"who":{"provider":"agmarknet"},"what":[{"type":"openagrinet:MandiPrice"}]}` - got, err := From(Paths{ + got, err := bindingFrom(Paths{ ProviderID: "who.provider", CapabilityCode: "what[].type", }, []byte(body)) if err != nil { - t.Fatalf("From() returned an unexpected error: %v", err) + t.Fatalf("bindingFrom() returned an unexpected error: %v", err) } if got.Key() != "agmarknet|openagrinet:MandiPrice" { t.Errorf("binding key = %q, want it read from the overridden paths", got.Key()) @@ -193,10 +193,10 @@ func TestFromReadsAnOverriddenPath(t *testing.T) { func TestFromReportsNoBindingWhenAPathMatchesNothing(t *testing.T) { t.Parallel() - _, err := From(Paths{ProviderID: "nowhere.at.all", CapabilityCode: "what[].type"}, + _, err := bindingFrom(Paths{ProviderID: "nowhere.at.all", CapabilityCode: "what[].type"}, []byte(`{"what":[{"type":"x"}]}`)) - if !errors.Is(err, ErrNoBinding) { - t.Errorf("expected ErrNoBinding, got %v", err) + if !errors.Is(err, errNoBinding) { + t.Errorf("expected errNoBinding, got %v", err) } } @@ -205,9 +205,9 @@ func TestFromReportsNoBindingWhenAPathMatchesNothing(t *testing.T) { func TestFromStillRefusesSeveralValuesUnderAnOverride(t *testing.T) { t.Parallel() - _, err := From(Paths{ProviderID: "who[].provider", CapabilityCode: "what[].type"}, + _, err := bindingFrom(Paths{ProviderID: "who[].provider", CapabilityCode: "what[].type"}, []byte(`{"who":[{"provider":"a"},{"provider":"b"}],"what":[{"type":"x"}]}`)) - if err == nil || errors.Is(err, ErrNoBinding) { + if err == nil || errors.Is(err, errNoBinding) { t.Errorf("expected a refusal naming both providers, got %v", err) } } @@ -303,11 +303,11 @@ func TestFromRefusesSeveralCommitments(t *testing.T) { "resources":[{"resourceAttributes":{"@type":"openagrinet:WeatherObservation"}}]} ]}}}` - _, err := From(BecknV2, []byte(body)) + _, err := bindingFrom(BecknV2, []byte(body)) if err == nil { t.Fatal("expected two commitments to be refused rather than halved") } - if errors.Is(err, ErrNoBinding) { + if errors.Is(err, errNoBinding) { t.Error("this is not an absent binding; it is one request asking for two calls") } if !strings.Contains(err.Error(), "2 commitments") { @@ -319,7 +319,7 @@ func TestFromRefusesSeveralCommitments(t *testing.T) { {"offer":{"provider":{"id":"mausamgram"}}, "resources":[{"resourceAttributes":{"@type":"openagrinet:WeatherObservation"}}]} ]}}}` - binding, err := From(BecknV2, []byte(single)) + binding, err := bindingFrom(BecknV2, []byte(single)) if err != nil { t.Fatalf("one commitment must still resolve: %v", err) } @@ -370,11 +370,11 @@ func TestFromRefusesSeveralCommitmentsEvenWhenOneDoesNotResolve(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - _, err := From(BecknV2, []byte(tt.payload)) + _, err := bindingFrom(BecknV2, []byte(tt.payload)) if err == nil { t.Fatal("two commitments must be refused, not half answered") } - if errors.Is(err, ErrNoBinding) { + if errors.Is(err, errNoBinding) { t.Errorf("err = %v; this is a refusal, not a payload for another step", err) } if !strings.Contains(err.Error(), "2 commitments") { @@ -385,7 +385,7 @@ func TestFromRefusesSeveralCommitmentsEvenWhenOneDoesNotResolve(t *testing.T) { } // One commitment whose provider id does not resolve is not this step's work -- -// it must stay ErrNoBinding rather than becoming a refusal, so the next step +// it must stay errNoBinding rather than becoming a refusal, so the next step // in the pipeline still sees it. func TestFromStillPassesThroughASingleUnresolvableCommitment(t *testing.T) { t.Parallel() @@ -394,9 +394,9 @@ func TestFromStillPassesThroughASingleUnresolvableCommitment(t *testing.T) { {"offer":{"provider":{}}, "resources":[{"resourceAttributes":{"@type":"openagrinet:WeatherObservation"}}]} ]}}}` - _, err := From(BecknV2, []byte(payload)) - if !errors.Is(err, ErrNoBinding) { - t.Errorf("err = %v, want ErrNoBinding so the payload passes through", err) + _, err := bindingFrom(BecknV2, []byte(payload)) + if !errors.Is(err, errNoBinding) { + t.Errorf("err = %v, want errNoBinding so the payload passes through", err) } } diff --git a/pkg/plugin/implementation/internal/common/common.go b/pkg/plugin/implementation/internal/common/common.go index c8f4efac..8c17b5b0 100644 --- a/pkg/plugin/implementation/internal/common/common.go +++ b/pkg/plugin/implementation/internal/common/common.go @@ -34,7 +34,6 @@ import ( "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/capabilitybinding" ) // Prerequisites are the values a capability needs that its payload does not @@ -105,7 +104,7 @@ type Config struct { // safe for concurrent use. type Step struct { config *Config - paths capabilitybinding.Paths + paths Paths prerequisites Prerequisites registry definition.ProviderRecordLookup mapper definition.Mapper @@ -168,19 +167,19 @@ func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper d // Both halves or neither: overriding one and leaving the other on the default // is a half-configured deployment that would match nothing, and it would do so // silently on every request rather than once at startup. -func bindingPaths(cfg *Config) (capabilitybinding.Paths, error) { +func bindingPaths(cfg *Config) (Paths, error) { if cfg.ProviderIDAt == "" && cfg.CapabilityCodeAt == "" { - return capabilitybinding.BecknV2, nil + return BecknV2, nil } if cfg.ProviderIDAt == "" { - return capabilitybinding.Paths{}, errors.New("upstream: capabilityCodeAt is set without providerIdAt") + return Paths{}, errors.New("upstream: capabilityCodeAt is set without providerIdAt") } if cfg.CapabilityCodeAt == "" { - return capabilitybinding.Paths{}, errors.New("upstream: providerIdAt is set without capabilityCodeAt") + return Paths{}, errors.New("upstream: providerIdAt is set without capabilityCodeAt") } - paths := capabilitybinding.Paths{ProviderID: cfg.ProviderIDAt, CapabilityCode: cfg.CapabilityCodeAt} + paths := Paths{ProviderID: cfg.ProviderIDAt, CapabilityCode: cfg.CapabilityCodeAt} if err := paths.Validate(); err != nil { - return capabilitybinding.Paths{}, err + return Paths{}, err } return paths, nil } diff --git a/pkg/plugin/implementation/internal/common/common_test.go b/pkg/plugin/implementation/internal/common/common_test.go index 56688999..2ac2443f 100644 --- a/pkg/plugin/implementation/internal/common/common_test.go +++ b/pkg/plugin/implementation/internal/common/common_test.go @@ -17,7 +17,6 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/capabilitybinding" ) const selectBody = `{ @@ -1119,7 +1118,7 @@ func TestNewUsesTheBecknConventionByDefault(t *testing.T) { t.Parallel() step := newStep(t, &stubRegistry{}, &stubMapper{}) - if step.paths != capabilitybinding.BecknV2 { + if step.paths != BecknV2 { t.Errorf("paths = %+v, want the Beckn v2 convention", step.paths) } } diff --git a/pkg/plugin/implementation/internal/capabilitybinding/paths.go b/pkg/plugin/implementation/internal/common/paths.go similarity index 95% rename from pkg/plugin/implementation/internal/capabilitybinding/paths.go rename to pkg/plugin/implementation/internal/common/paths.go index f1f89a2a..1fc37b4a 100644 --- a/pkg/plugin/implementation/internal/capabilitybinding/paths.go +++ b/pkg/plugin/implementation/internal/common/paths.go @@ -1,4 +1,4 @@ -package capabilitybinding +package common import ( "fmt" @@ -38,11 +38,11 @@ func (p Paths) Validate() error { "capabilityCodeAt": p.CapabilityCode, } { if strings.TrimSpace(path) == "" { - return fmt.Errorf("capabilitybinding: %s is empty", name) + return fmt.Errorf("upstream: binding path %s is empty", name) } for _, segment := range strings.Split(path, ".") { if strings.TrimSpace(strings.TrimSuffix(segment, arrayMarker)) == "" { - return fmt.Errorf("capabilitybinding: %s (%q) has a blank segment", name, path) + return fmt.Errorf("upstream: binding path %s (%q) has a blank segment", name, path) } } } diff --git a/pkg/plugin/implementation/internal/common/serve.go b/pkg/plugin/implementation/internal/common/serve.go index bbc98237..fa8d40cb 100644 --- a/pkg/plugin/implementation/internal/common/serve.go +++ b/pkg/plugin/implementation/internal/common/serve.go @@ -13,7 +13,6 @@ import ( "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/capabilitybinding" ) // Run serves the request when it is for this step's capability, and does @@ -23,8 +22,8 @@ import ( // pipeline and each recognises its own work, so adding a provider is one more // entry rather than a change to a routing table. func (s *Step) Run(ctx *model.StepContext) error { - binding, err := capabilitybinding.From(s.paths, ctx.Body) - if errors.Is(err, capabilitybinding.ErrNoBinding) { + binding, err := bindingFrom(s.paths, ctx.Body) + if errors.Is(err, errNoBinding) { return nil } if err != nil { From ffc121a7d6c02237b13c671e140e3594dbec77f7 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 18:34:43 +0530 Subject: [PATCH 111/130] docs(common): a comment pointed at a line that moved files [#7] serves said the config order is preserved in "the log line above". That line is New's "Upstream step created for ...", which the split moved to common.go, so the reference pointed at nothing in serve.go. It now names New instead of a position. Found by auditing the rearrangement for stale references rather than by reading past it. Three other positional comments were checked and are correct: two are about a paragraph or a mapping file rather than code position, and common_test.go's "budget() above" still holds because the test file was not split. --- pkg/plugin/implementation/internal/common/serve.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugin/implementation/internal/common/serve.go b/pkg/plugin/implementation/internal/common/serve.go index fa8d40cb..88e32e5f 100644 --- a/pkg/plugin/implementation/internal/common/serve.go +++ b/pkg/plugin/implementation/internal/common/serve.go @@ -81,7 +81,7 @@ func (s *Step) resolve(ctx context.Context, bindingKey string, beckn any) (map[s // // A slice rather than a set: a step serves a handful of capabilities at most, so // the scan costs less than the map would, and the config order is preserved in -// the log line above. +// the line New logs on startup. func (s *Step) serves(key string) bool { return slices.Contains(s.config.BindingKeys, key) } From 20e86cbb0cb6b7b96ed6f08670ca8aa6ea292a93 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 18:44:22 +0530 Subject: [PATCH 112/130] refactor(common): name asQueryValue for what it produces [#7] renderScalar said how it worked, not what it was for, and its doc comment only restated the name. It renders one mapped field as a query parameter value, so: renderScalar -> asQueryValue Matching asQuery directly above it, and the asX convention the package already uses -- asQuery, asString, asNestedBlock. The comment now says what the bool means and why only three types are carried: an object or an array has no single obvious encoding, so choosing one here would put a convention in Go that belongs in the mapping, where the upstream's shape is known. It also documents null, which was undocumented and untested. null is refused rather than sent as empty, because a parameter present but empty and a parameter absent mean different things to some upstreams -- a mapping says which it wants by omitting the field or setting "". Two tests now pin that: objects, arrays and null are refused with the field named, and an empty string IS carried as token=. Negative control: making null render as empty fails the null case, so the test is not vacuous. --- .../internal/common/common_test.go | 44 +++++++++++++++++++ .../implementation/internal/common/http.go | 19 ++++++-- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/pkg/plugin/implementation/internal/common/common_test.go b/pkg/plugin/implementation/internal/common/common_test.go index 2ac2443f..1e794541 100644 --- a/pkg/plugin/implementation/internal/common/common_test.go +++ b/pkg/plugin/implementation/internal/common/common_test.go @@ -1644,6 +1644,50 @@ func TestRunRefusesAMappedQueryItCannotRender(t *testing.T) { // --- query rendering -------------------------------------------------------- +// asQueryValue carries the three JSON scalars and refuses everything else, +// naming the field so a mapping that produced one is told which. null is in the +// refused set on purpose: a parameter present but empty and a parameter absent +// mean different things to some upstreams, and a mapping says which it wants by +// omitting the field or setting "". +func TestAsQueryRefusesWhatCannotBeAQueryParameter(t *testing.T) { + t.Parallel() + + for _, tc := range []struct{ name, mapped string }{ + {"an object", `{"location":{"lat":19.9975}}`}, + {"an array", `{"days":[1,2,3]}`}, + {"null", `{"station":null}`}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + _, err := asQuery([]byte(tc.mapped)) + if err == nil { + t.Fatalf("asQuery(%s) was accepted; it cannot become a query parameter", tc.mapped) + } + // The field, so a mapping author knows which one to fix. + if !strings.Contains(err.Error(), "station") && + !strings.Contains(err.Error(), "location") && + !strings.Contains(err.Error(), "days") { + t.Errorf("error %q should name the offending field", err) + } + }) + } +} + +// An empty string IS carried: it is a scalar, and a mapping setting one is +// asking for the parameter to be present and empty. +func TestAsQueryCarriesAnEmptyString(t *testing.T) { + t.Parallel() + + got, err := asQuery([]byte(`{"token":""}`)) + if err != nil { + t.Fatalf("asQuery() returned an unexpected error: %v", err) + } + if got != "token=" { + t.Errorf("query = %q, want token= -- an empty string is a value, not an absence", got) + } +} + func TestAsQueryRendersScalarsWithoutInventingPrecision(t *testing.T) { t.Parallel() diff --git a/pkg/plugin/implementation/internal/common/http.go b/pkg/plugin/implementation/internal/common/http.go index 800078e5..26eef704 100644 --- a/pkg/plugin/implementation/internal/common/http.go +++ b/pkg/plugin/implementation/internal/common/http.go @@ -363,7 +363,7 @@ func asQuery(mapped []byte) (string, error) { values := url.Values{} for name, value := range fields { - rendered, ok := renderScalar(value) + rendered, ok := asQueryValue(value) if !ok { return "", fmt.Errorf("upstream: mapped field %q is not a scalar and cannot become a query parameter", name) } @@ -372,8 +372,21 @@ func asQuery(mapped []byte) (string, error) { return values.Encode(), nil } -// renderScalar renders a JSON scalar as a query parameter value. -func renderScalar(value any) (string, bool) { +// asQueryValue renders one mapped field as a query parameter value, reporting +// false when the field cannot be one. +// +// The three JSON scalars and nothing else. An object or an array has no single +// obvious encoding -- repeated keys, comma-joined, indexed, JSON-in-a-parameter +// are all in use somewhere -- so choosing here would put a convention in Go +// that belongs in the mapping, where the upstream's actual shape is known. The +// caller turns false into an error naming the field, so a mapping that produces +// one is told which. +// +// null is refused for the same reason rather than sent as empty: a parameter +// present but empty and a parameter absent mean different things to some +// upstreams, and a mapping says which it wants by omitting the field or setting +// "". An empty string IS carried, being a scalar. +func asQueryValue(value any) (string, bool) { switch typed := value.(type) { case string: return typed, true From ec64ae37cac210da07b7f30d252cfa62879e004b Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 18:51:20 +0530 Subject: [PATCH 113/130] refactor(common): drop the "upstream: " prefix from messages [#7] 58 error and log messages carried it. The structured log already records module_id, message_id and the step, so the prefix repeated what the fields say and cost a reader eight characters on every line: upstream: provider did not answer after 3 attempts provider did not answer after 3 attempts Consistent with sunbirdRegistry and router, which carry no prefix. jsonmapper still does; not touched here. One test mentioned the prefix and it was building its own wrapper string, not asserting on ours -- checked before changing, and it still passes. The package doc went from 25 lines to 14. It had a paragraph justifying the prefix, which no longer exists, and the blanket replace had mangled that paragraph's own quoted example into a sentence contradicting itself. "Upstream" remains the word for the API being called -- it is the registry's, for a Participant of type upstream as against a node that speaks Beckn. 65 packages green, vet clean. --- .../implementation/internal/common/auth.go | 22 ++++----- .../implementation/internal/common/binding.go | 10 ++-- .../implementation/internal/common/common.go | 49 +++++++------------ .../internal/common/common_test.go | 2 +- .../implementation/internal/common/http.go | 34 ++++++------- .../implementation/internal/common/paths.go | 4 +- .../implementation/internal/common/serve.go | 22 ++++----- .../implementation/internal/common/token.go | 4 +- 8 files changed, 68 insertions(+), 79 deletions(-) diff --git a/pkg/plugin/implementation/internal/common/auth.go b/pkg/plugin/implementation/internal/common/auth.go index a016acbc..1fe83445 100644 --- a/pkg/plugin/implementation/internal/common/auth.go +++ b/pkg/plugin/implementation/internal/common/auth.go @@ -86,30 +86,30 @@ func (a *AuthProfile) validate() error { case AuthSchemeBasic: if a.UsernameEnv == "" || a.PasswordEnv == "" { return fmt.Errorf( - "upstream: %s: authScheme basic requires usernameEnv and passwordEnv", a.Provider) + "%s: authScheme basic requires usernameEnv and passwordEnv", a.Provider) } case AuthSchemeHeader: if a.HeaderName == "" || a.HeaderValueEnv == "" { return fmt.Errorf( - "upstream: %s: authScheme header requires headerName and headerValueEnv", a.Provider) + "%s: authScheme header requires headerName and headerValueEnv", a.Provider) } case AuthSchemeQuery: if a.QueryName == "" || a.QueryValueEnv == "" { return fmt.Errorf( - "upstream: %s: authScheme query requires queryName and queryValueEnv", a.Provider) + "%s: authScheme query requires queryName and queryValueEnv", a.Provider) } case AuthSchemeOAuth2: if a.TokenURL == "" || a.ClientIDEnv == "" || a.ClientSecretEnv == "" { return fmt.Errorf( - "upstream: %s: authScheme oauth2 requires tokenUrl, clientIdEnv and clientSecretEnv", + "%s: authScheme oauth2 requires tokenUrl, clientIdEnv and clientSecretEnv", a.Provider) } case "": - return fmt.Errorf("upstream: %s: authScheme is required, "+ + return fmt.Errorf("%s: authScheme is required, "+ "and is none where the upstream needs no credential", a.Provider) default: return fmt.Errorf( - "upstream: %s: unknown authScheme %q: must be none, basic, header, query or oauth2", + "%s: unknown authScheme %q: must be none, basic, header, query or oauth2", a.Provider, a.Scheme) } return nil @@ -153,7 +153,7 @@ func ParseProviderAuth(config map[string]string) (map[string]*AuthProfile, error // credential at all, which reads as the provider rejecting us. if authFields[key] { return nil, fmt.Errorf( - "upstream: %q is set for the whole step; auth is per provider now, "+ + "%q is set for the whole step; auth is per provider now, "+ "so it belongs in a block named for the participant id", key) } continue @@ -162,10 +162,10 @@ func ParseProviderAuth(config map[string]string) (map[string]*AuthProfile, error // Not an auth setting, and nothing else on a provider step carries // a dash -- so this is a misspelled field rather than something to // pass through. - return nil, fmt.Errorf("upstream: %q is not a credential setting", key) + return nil, fmt.Errorf("%q is not a credential setting", key) } if strings.TrimSpace(provider) == "" { - return nil, fmt.Errorf("upstream: %q names no provider after the dash", key) + return nil, fmt.Errorf("%q names no provider after the dash", key) } profile, seen := profiles[provider] @@ -237,8 +237,8 @@ func providerIDFrom(bindingKey string) string { // the fix is entirely the operator's. So the name goes to the log, where the // operator is, and the wire gets the scheme that failed. func (s *Step) missingCredential(ctx context.Context, provider, scheme, envNames string) error { - err := fmt.Errorf("upstream: the %s credential for %s is not configured", scheme, provider) - log.Errorf(ctx, err, "upstream: %s auth is configured for %s but %s is not set", + err := fmt.Errorf("the %s credential for %s is not configured", scheme, provider) + log.Errorf(ctx, err, "%s auth is configured for %s but %s is not set", scheme, provider, envNames) return err } diff --git a/pkg/plugin/implementation/internal/common/binding.go b/pkg/plugin/implementation/internal/common/binding.go index b3b745bd..3731299b 100644 --- a/pkg/plugin/implementation/internal/common/binding.go +++ b/pkg/plugin/implementation/internal/common/binding.go @@ -19,7 +19,7 @@ const separator = "|" // errNoBinding reports a payload that names no capability binding. It is not a // fault: a request for something else entirely reaches a provider step too, and // the step's answer is to do nothing. -var errNoBinding = errors.New("upstream: payload names no capability binding") +var errNoBinding = errors.New("payload names no capability binding") // Binding identifies one provider capability. type Binding struct { @@ -47,7 +47,7 @@ func (b Binding) Key() string { func bindingFrom(paths Paths, body []byte) (Binding, error) { var payload any if err := json.Unmarshal(body, &payload); err != nil { - return Binding{}, fmt.Errorf("upstream: payload could not be read: %w", err) + return Binding{}, fmt.Errorf("payload could not be read: %w", err) } // Before distinctness: N commitments naming the SAME provider and type @@ -64,7 +64,7 @@ func bindingFrom(paths Paths, body []byte) (Binding, error) { // the paragraph above says is refused. if commitments := countAt(payload, paths.ProviderID); commitments > 1 { return Binding{}, fmt.Errorf( - "upstream: payload carries %d commitments; one request maps to one call, "+ + "payload carries %d commitments; one request maps to one call, "+ "so send them separately rather than have all but the first dropped", commitments) } @@ -76,11 +76,11 @@ func bindingFrom(paths Paths, body []byte) (Binding, error) { return Binding{}, errNoBinding } if len(providers) > 1 { - return Binding{}, fmt.Errorf("upstream: payload names %d providers (%s); one request maps to one call", + return Binding{}, fmt.Errorf("payload names %d providers (%s); one request maps to one call", len(providers), strings.Join(providers, ", ")) } if len(types) > 1 { - return Binding{}, fmt.Errorf("upstream: payload names %d resource types (%s); one request maps to one call", + return Binding{}, fmt.Errorf("payload names %d resource types (%s); one request maps to one call", len(types), strings.Join(types, ", ")) } diff --git a/pkg/plugin/implementation/internal/common/common.go b/pkg/plugin/implementation/internal/common/common.go index 8c17b5b0..06e1cbd5 100644 --- a/pkg/plugin/implementation/internal/common/common.go +++ b/pkg/plugin/implementation/internal/common/common.go @@ -1,28 +1,17 @@ // Package common serves a Beckn capability by calling an ordinary API that has -// never heard of Beckn. +// never heard of Beckn: recognise the capability, resolve the call plan, +// translate out, call, translate back. // -// Common to every capability plugin, and to nothing else: WeatherObservation, -// MandiPrice and KnowledgeAdvisory are each a name, a set of binding keys and a -// credential profile over this machinery. Recognise the capability, resolve the -// call plan, translate out, call, translate back. +// Common to the three capability plugins and to nothing else. Each of them -- +// WeatherObservation, MandiPrice, KnowledgeAdvisory -- is a name, a set of +// binding keys and a credential profile over this machinery. Something shared +// by fewer than all three belongs in its own package, not here on the strength +// of the name. // -// "UPSTREAM" IS STILL THE WORD FOR THE API BEING CALLED. It is the registry's -// own -- a Participant of type upstream, as against a node that speaks Beckn -- -// so it stays in the errors, the logs and the comments here even though the -// package no longer carries it. An operator reading "upstream: provider did not -// answer" is being told which leg of the call failed, and "common:" would tell -// them nothing. -// -// It holds nothing about any provider or any domain. What varies per capability -// comes from the registry (endpoint, method, budget, which mapping) and from the -// mapping itself (what the payload must satisfy, what to send, what to return). -// A domain package wraps this, supplying only its name and whatever prerequisite -// work a mapping cannot express. -// -// NOT A DUMPING GROUND. Everything here is one subject -- calling an upstream on -// a capability's behalf. Something shared by fewer than all three plugins, or -// unrelated to that call, belongs in its own package rather than here on the -// strength of the name. +// Nothing about any provider or domain is held here. What varies per capability +// comes from the registry (endpoint, method, budget, which mapping) and from +// the mapping itself. "Upstream" is the registry's word for the API being +// called, and stays the word for it throughout. package common import ( @@ -120,10 +109,10 @@ type Step struct { func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper definition.Mapper, prerequisites Prerequisites, cfg *Config) (*Step, func() error, error) { if registry == nil { - return nil, nil, errors.New("upstream: a provider record lookup is required") + return nil, nil, errors.New("a provider record lookup is required") } if mapper == nil { - return nil, nil, errors.New("upstream: a mapper is required") + return nil, nil, errors.New("a mapper is required") } if cfg == nil { cfg = &Config{} @@ -172,10 +161,10 @@ func bindingPaths(cfg *Config) (Paths, error) { return BecknV2, nil } if cfg.ProviderIDAt == "" { - return Paths{}, errors.New("upstream: capabilityCodeAt is set without providerIdAt") + return Paths{}, errors.New("capabilityCodeAt is set without providerIdAt") } if cfg.CapabilityCodeAt == "" { - return Paths{}, errors.New("upstream: providerIdAt is set without capabilityCodeAt") + return Paths{}, errors.New("providerIdAt is set without capabilityCodeAt") } paths := Paths{ProviderID: cfg.ProviderIDAt, CapabilityCode: cfg.CapabilityCodeAt} if err := paths.Validate(); err != nil { @@ -190,11 +179,11 @@ func applyDefaults(cfg *Config) error { // for, so a default would have to name one provider's capability -- wrong // for every other domain built on it, and silently wrong rather than loudly. if len(cfg.BindingKeys) == 0 { - return errors.New("upstream: bindingKeys is required: it is what this step answers to") + return errors.New("bindingKeys is required: it is what this step answers to") } for _, key := range cfg.BindingKeys { if strings.TrimSpace(key) == "" { - return errors.New("upstream: bindingKeys carries an empty entry") + return errors.New("bindingKeys carries an empty entry") } } if cfg.MaxResponseBytes <= 0 { @@ -216,7 +205,7 @@ func applyDefaults(cfg *Config) error { for provider := range cfg.AuthByProvider { if !served[provider] { return fmt.Errorf( - "upstream: auth is configured for %q, which is not a provider in bindingKeys (%s)", + "auth is configured for %q, which is not a provider in bindingKeys (%s)", provider, strings.Join(cfg.BindingKeys, ", ")) } } @@ -224,7 +213,7 @@ func applyDefaults(cfg *Config) error { profile, ok := cfg.AuthByProvider[provider] if !ok { return fmt.Errorf( - "upstream: %q is served but has no auth block; every provider declares its own, "+ + "%q is served but has no auth block; every provider declares its own, "+ "using authScheme none where the upstream needs no credential", provider) } profile.Provider = provider diff --git a/pkg/plugin/implementation/internal/common/common_test.go b/pkg/plugin/implementation/internal/common/common_test.go index 1e794541..5ea5b987 100644 --- a/pkg/plugin/implementation/internal/common/common_test.go +++ b/pkg/plugin/implementation/internal/common/common_test.go @@ -1820,7 +1820,7 @@ func TestRedactKeepsTheErrorChainMatchable(t *testing.T) { } // And wrapping it again, which is what the 502 does, must not undo either // property. - wrapped := fmt.Errorf("upstream: provider did not answer: %w", got) + wrapped := fmt.Errorf("provider did not answer: %w", got) if strings.Contains(wrapped.Error(), "s3cr3t") { t.Errorf("the credential reappeared once wrapped: %v", wrapped) } diff --git a/pkg/plugin/implementation/internal/common/http.go b/pkg/plugin/implementation/internal/common/http.go index 26eef704..ad18044b 100644 --- a/pkg/plugin/implementation/internal/common/http.go +++ b/pkg/plugin/implementation/internal/common/http.go @@ -56,10 +56,10 @@ func (s *Step) call(ctx context.Context, auth *authenticator, baseURL string, ca timeout, retries := budget(call) if d := time.Duration(call.TimeoutMs) * time.Millisecond; d > timeout { - log.Warnf(ctx, "upstream: registry asks for a %v timeout; using the %v ceiling", d, timeout) + log.Warnf(ctx, "registry asks for a %v timeout; using the %v ceiling", d, timeout) } if call.RetryMax > retries { - log.Warnf(ctx, "upstream: registry asks for %d retries; using the %d ceiling", + log.Warnf(ctx, "registry asks for %d retries; using the %d ceiling", call.RetryMax, retries) } attempts := retries + 1 @@ -81,7 +81,7 @@ func (s *Step) call(ctx context.Context, auth *authenticator, baseURL string, ca return body, nil } lastErr = s.redact(err) - log.Warnf(ctx, "upstream: attempt %d/%d failed: %v", attempt, attempts, lastErr) + log.Warnf(ctx, "attempt %d/%d failed: %v", attempt, attempts, lastErr) // Only some failures are worth repeating. A 4xx, a request this step // could not build and a credential it could not read will fail @@ -98,7 +98,7 @@ func (s *Step) call(ctx context.Context, auth *authenticator, baseURL string, ca } } return nil, model.NewCodedErr(http.StatusBadGateway, codeUpstreamUnavailable, - fmt.Errorf("upstream: provider did not answer after %d attempts: %w", attempts, lastErr)) + fmt.Errorf("provider did not answer after %d attempts: %w", attempts, lastErr)) } // permanentErr marks a failure no retry can fix. Kept unexported and detected @@ -188,7 +188,7 @@ func (s *Step) attempt(ctx context.Context, auth *authenticator, call model.Acti if err != nil { return nil, fmt.Errorf("could not read the response: %w", err) } - log.Infof(ctx, "upstream: %s %s -> %s, %d bytes", method, requested, resp.Status, len(body)) + log.Infof(ctx, "%s %s -> %s, %d bytes", method, requested, resp.Status, len(body)) if int64(len(body)) > s.config.MaxResponseBytes { // Asking again will not make the answer smaller. return nil, doNotRetry(fmt.Errorf("response exceeds the %d byte limit", s.config.MaxResponseBytes)) @@ -207,7 +207,7 @@ func (s *Step) attempt(ctx context.Context, auth *authenticator, call model.Acti // request often quotes it back, credential and all -- so the body is // exactly where a query-string token turns up, and moving it from the // error to the log would only move the leak. - log.Warnf(ctx, "upstream: provider returned %s for %s %s: %s", + log.Warnf(ctx, "provider returned %s for %s %s: %s", resp.Status, method, requested, s.redactString(explain(body))) err := fmt.Errorf("provider returned %s", resp.Status) // 5xx and 429 are the provider asking to be tried again. Every other @@ -286,15 +286,15 @@ func buildEndpoint(baseURL string, call model.ActionPlan, mapped []byte) (string // the URL the operator published. func verifyPath(path string) error { if path == "" { - return model.NewBadReqErr("", errors.New("upstream: the registry publishes no path for this action")) + return model.NewBadReqErr("", errors.New("the registry publishes no path for this action")) } if !strings.HasPrefix(path, "/") { return model.NewBadReqErr("", fmt.Errorf( - "upstream: path %q does not begin with a slash, so it cannot be joined to a base url", path)) + "path %q does not begin with a slash, so it cannot be joined to a base url", path)) } if strings.Contains(path, "//") { return model.NewBadReqErr("", fmt.Errorf( - "upstream: path %q has an empty segment; write it with single slashes", path)) + "path %q has an empty segment; write it with single slashes", path)) } // A dot segment is refused rather than resolved. The registry says which // path answers an action, and a row that climbs out of it is either a @@ -304,7 +304,7 @@ func verifyPath(path string) error { for _, segment := range strings.Split(path, "/") { if segment == ".." || segment == "." { return model.NewBadReqErr("", fmt.Errorf( - "upstream: path %q contains the %q segment; publish the path it resolves to instead", + "path %q contains the %q segment; publish the path it resolves to instead", path, segment)) } } @@ -313,7 +313,7 @@ func verifyPath(path string) error { // transport, which would make the row look honoured. if strings.Contains(path, "#") { return model.NewBadReqErr("", fmt.Errorf( - "upstream: path %q contains a fragment, which is never sent to a server", path)) + "path %q contains a fragment, which is never sent to a server", path)) } return nil } @@ -330,18 +330,18 @@ func verifyPath(path string) error { // same check on the other url the registry publishes. func verifyBaseURL(baseURL string) error { if baseURL == "" { - return model.NewBadReqErr("", errors.New("upstream: the registry publishes no base url for this provider")) + return model.NewBadReqErr("", errors.New("the registry publishes no base url for this provider")) } parsed, err := url.Parse(baseURL) if err != nil { - return model.NewBadReqErr("", fmt.Errorf("upstream: invalid base url %q: %w", baseURL, err)) + return model.NewBadReqErr("", fmt.Errorf("invalid base url %q: %w", baseURL, err)) } if parsed.Scheme != "http" && parsed.Scheme != "https" { return model.NewBadReqErr("", fmt.Errorf( - "upstream: base url %q must be http or https", baseURL)) + "base url %q must be http or https", baseURL)) } if parsed.Host == "" { - return model.NewBadReqErr("", fmt.Errorf("upstream: base url %q names no host", baseURL)) + return model.NewBadReqErr("", fmt.Errorf("base url %q names no host", baseURL)) } return nil } @@ -358,14 +358,14 @@ func asQuery(mapped []byte) (string, error) { } var fields map[string]any if err := json.Unmarshal(mapped, &fields); err != nil { - return "", fmt.Errorf("upstream: mapped request is not an object, so it cannot become a query: %w", err) + return "", fmt.Errorf("mapped request is not an object, so it cannot become a query: %w", err) } values := url.Values{} for name, value := range fields { rendered, ok := asQueryValue(value) if !ok { - return "", fmt.Errorf("upstream: mapped field %q is not a scalar and cannot become a query parameter", name) + return "", fmt.Errorf("mapped field %q is not a scalar and cannot become a query parameter", name) } values.Set(name, rendered) } diff --git a/pkg/plugin/implementation/internal/common/paths.go b/pkg/plugin/implementation/internal/common/paths.go index 1fc37b4a..cbab19e7 100644 --- a/pkg/plugin/implementation/internal/common/paths.go +++ b/pkg/plugin/implementation/internal/common/paths.go @@ -38,11 +38,11 @@ func (p Paths) Validate() error { "capabilityCodeAt": p.CapabilityCode, } { if strings.TrimSpace(path) == "" { - return fmt.Errorf("upstream: binding path %s is empty", name) + return fmt.Errorf("binding path %s is empty", name) } for _, segment := range strings.Split(path, ".") { if strings.TrimSpace(strings.TrimSuffix(segment, arrayMarker)) == "" { - return fmt.Errorf("upstream: binding path %s (%q) has a blank segment", name, path) + return fmt.Errorf("binding path %s (%q) has a blank segment", name, path) } } } diff --git a/pkg/plugin/implementation/internal/common/serve.go b/pkg/plugin/implementation/internal/common/serve.go index 88e32e5f..06be4124 100644 --- a/pkg/plugin/implementation/internal/common/serve.go +++ b/pkg/plugin/implementation/internal/common/serve.go @@ -34,7 +34,7 @@ func (s *Step) Run(ctx *model.StepContext) error { return model.NewBadReqErr("", err) } if !s.serves(binding.Key()) { - log.Debugf(ctx, "upstream: %s is not one of this step's capabilities, passing through", binding.Key()) + log.Debugf(ctx, "%s is not one of this step's capabilities, passing through", binding.Key()) return nil } @@ -49,9 +49,9 @@ func (s *Step) Run(ctx *model.StepContext) error { // %w, not %v: the sentinel has to stay unwrappable, or anything // upstream testing errors.Is against it silently stops matching. return model.NewNotFoundErr("", fmt.Errorf( - "upstream: the registry publishes no active binding for %s: %w", binding.Key(), err)) + "the registry publishes no active binding for %s: %w", binding.Key(), err)) } - return fmt.Errorf("upstream: no call plan for %s: %w", binding.Key(), err) + return fmt.Errorf("no call plan for %s: %w", binding.Key(), err) } return s.serve(ctx, plan) @@ -69,7 +69,7 @@ func (s *Step) resolve(ctx context.Context, bindingKey string, beckn any) (map[s } local, err := prerequisite(ctx, beckn) if err != nil { - return nil, fmt.Errorf("upstream: %s could not resolve what it needs before the call: %w", bindingKey, err) + return nil, fmt.Errorf("%s could not resolve what it needs before the call: %w", bindingKey, err) } if local == nil { return map[string]any{}, nil @@ -96,7 +96,7 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { // happened to be on the record -- naming what it does serve turns a // registry mistake into a one-line fix. return model.NewBadReqErr("", fmt.Errorf( - "upstream: %s does not serve action %q; it serves %s", + "%s does not serve action %q; it serves %s", plan.BindingKey, action, strings.Join(plan.ServedActions(), ", "))) } @@ -131,7 +131,7 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { // where a record arrives for a key the config never declared. auth, configured := s.auth[providerIDFrom(plan.BindingKey)] if !configured { - return fmt.Errorf("upstream: no credential is configured for %s", plan.BindingKey) + return fmt.Errorf("no credential is configured for %s", plan.BindingKey) } upstreamResponse, err := s.call(ctx, auth, plan.BaseURL, call, upstreamRequest) @@ -141,7 +141,7 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { answer, err := decodeBody(upstreamResponse) if err != nil { - return fmt.Errorf("upstream: provider answered with something that is not JSON: %w", err) + return fmt.Errorf("provider answered with something that is not JSON: %w", err) } // The same mapping reference as the request, other half: one file carries @@ -162,12 +162,12 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { // in this answer. Both leave no Beckn response to return, and returning // the provider's own shape instead would be worse than failing. The // message says what was observed rather than guessing which it was. - return fmt.Errorf("upstream: the response half of %s produced nothing, so %s cannot be answered", + return fmt.Errorf("the response half of %s produced nothing, so %s cannot be answered", call.Mappings, plan.BindingKey) } ctx.ResponseBody = becknResponse - log.Infof(ctx, "upstream: served %s in %d bytes", plan.BindingKey, len(becknResponse)) + log.Infof(ctx, "served %s in %d bytes", plan.BindingKey, len(becknResponse)) return nil } @@ -190,7 +190,7 @@ func (s *Step) buildRequest(ctx context.Context, call model.ActionPlan, beckn an return nil, err } if len(mapped) == 0 { - log.Debugf(ctx, "upstream: the request half of %s produced nothing; sending an empty request", call.Mappings) + log.Debugf(ctx, "the request half of %s produced nothing; sending an empty request", call.Mappings) } return mapped, nil } @@ -212,7 +212,7 @@ func extractAction(body []byte) string { func decodeBody(body []byte) (any, error) { var decoded any if err := json.Unmarshal(body, &decoded); err != nil { - return nil, fmt.Errorf("upstream: could not read JSON: %w", err) + return nil, fmt.Errorf("could not read JSON: %w", err) } return decoded, nil } diff --git a/pkg/plugin/implementation/internal/common/token.go b/pkg/plugin/implementation/internal/common/token.go index fd2b08af..0c03db70 100644 --- a/pkg/plugin/implementation/internal/common/token.go +++ b/pkg/plugin/implementation/internal/common/token.go @@ -103,7 +103,7 @@ func (s *Step) exchangeToken(ctx context.Context, auth *authenticator) (string, if resp.StatusCode < 200 || resp.StatusCode >= 300 { // The status, not the body: what an issuer puts in a failure body is its // own business, and it routinely quotes the request back. - log.Warnf(ctx, "upstream: token endpoint %s returned %s: %s", + log.Warnf(ctx, "token endpoint %s returned %s: %s", cfg.TokenURL, resp.Status, s.redactString(explain(body))) return "", 0, s.tokenErr(fmt.Errorf("token endpoint returned %s", resp.Status)) } @@ -130,7 +130,7 @@ func (s *Step) exchangeToken(ctx context.Context, auth *authenticator) (string, // provider's issuer, so its failure is upstream's, never the caller's. func (s *Step) tokenErr(err error) error { return model.NewCodedErr(http.StatusBadGateway, codeUpstreamUnavailable, - fmt.Errorf("upstream: oauth2 token exchange failed: %w", err)) + fmt.Errorf("oauth2 token exchange failed: %w", err)) } // tokenLifetime turns a token response's expires_in into how long we may hold From 25ef3b5c6da64ee88f50409cecf2c1ff4512a5ee Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 18:55:36 +0530 Subject: [PATCH 114/130] docs(common): shorten redact.go's comments [#7] 105 comment lines to 48, 196 total to 139. Comments only -- code verified identical by stripping every comment and blank line from both versions and diffing: 82 lines each, byte for byte. A DUPLICATE FOUND WHILE READING. The nineteen-line per-scheme block appeared on BOTH secretForms methods, copied there when the file was split. It belongs only on the one that switches on the scheme; Step.secretForms now says what it does in three lines. What went, and why it was safe: the incident detail on redactedErr -- that isPermanent tests before redaction, so nothing failed visibly. Pinned by TestRedactKeepsTheErrorChainMatchable. the paragraph justifying logging bodies and URLs at all. That argues for logging, not for redacting, and the two redaction tests cover the behaviour. "basic is the scheme the reference config ships" -- a fact about config/provider-adapter.yaml, checkable there. What stayed, because a reader could otherwise undo it: errors.New breaks the chain and %w undoes the redaction; the merged set must sort by length because one provider's token can be a substring of another's; the username and client id are deliberately not redacted; escaping uses the same function Encode does. 65 packages green, vet clean. --- .../implementation/internal/common/redact.go | 123 +++++------------- 1 file changed, 33 insertions(+), 90 deletions(-) diff --git a/pkg/plugin/implementation/internal/common/redact.go b/pkg/plugin/implementation/internal/common/redact.go index aeabab49..fded2101 100644 --- a/pkg/plugin/implementation/internal/common/redact.go +++ b/pkg/plugin/implementation/internal/common/redact.go @@ -9,16 +9,11 @@ import ( "strings" ) -// redact removes a query-string credential from an error's text. +// redact removes a credential from an error's text. // // Go's transport errors quote the whole URL -- `Get "http://host/p?token=..." -// dial tcp: ...` -- so without this, one unreachable host writes the credential -// into the log at warn level. Nothing else in this package puts a URL in a -// message, which is why this is the only place it is needed. -// -// A plain string replacement, because the value is what leaks and the value is -// what we hold. Parsing the error to find it would assume a shape net/http does -// not promise. +// dial tcp: ...` -- so an unreachable host would write the credential into the +// log. Nothing else here puts a URL in a message. func (s *Step) redact(err error) error { if err == nil { return nil @@ -30,23 +25,12 @@ func (s *Step) redact(err error) error { return redactedErr{text: text, err: err} } -// redactedErr reports a redacted message while keeping the original reachable -// for errors.Is and errors.As. -// -// errors.New(text) was the obvious thing and it broke the chain: the redacted -// value is what gets %w-wrapped into the final 502, so under a query-string -// scheme -- and only then, since nothing else redacts -- errors.Is(err, -// context.DeadlineExceeded) silently stopped matching. Retry classification -// was never affected, because isPermanent tests the error before redaction, -// which is why nothing failed visibly. -// -// fmt.Errorf("%s: %w", text, err) would have restored the chain and undone the -// redaction with it: %w formats the original, credential included. Reporting -// the redacted text from Error() and the original from Unwrap() keeps both. +// redactedErr reports the redacted text from Error() and the original from +// Unwrap(), so errors.Is and errors.As still match. // -// The original's text is reachable through errors.Unwrap, which is a -// deliberate act by a caller who wants the cause -- and %v, %s and %w on the -// value itself all go through Error() and stay redacted. +// Two simpler things do not work. errors.New(text) breaks the chain. +// fmt.Errorf("%s: %w", text, err) restores it and undoes the redaction, because +// %w formats the original. type redactedErr struct { text string err error @@ -56,21 +40,12 @@ func (e redactedErr) Error() string { return e.text } func (e redactedErr) Unwrap() error { return e.err } -// redactString removes the configured credential from any text about to be -// logged or returned -- an error, a provider's response body, or the URL that -// was requested. +// redactString removes every configured credential from text about to be logged +// or returned -- an error, a response body, or a requested URL. // -// Logging those is deliberate: they say what was asked of whom and what came -// back, which is the first thing anyone wants when a provider misbehaves. This -// is what makes that safe to do at info and warn level. -// -// EVERY scheme, not just query. This used to return early unless the scheme was -// query, on the reasoning that only a query credential reaches a URL -- true of -// the URL, and wrong about the body. A provider quoting the request it rejected -// is the ordinary shape of a 401 or 403 body, an API gateway echoing the -// Authorization header is routine, and a wrong-credential 4xx is not retried, -// so it lands in the log once per request for as long as the credential is -// wrong. basic is the scheme the reference config ships. +// Every scheme, not only query. A provider quoting the request it rejected is +// the ordinary shape of a 401 body, and gateways echo the Authorization header, +// so a credential reaches a message under any scheme. func (s *Step) redactString(text string) string { for _, secret := range s.secretForms() { text = strings.ReplaceAll(text, secret, redactedMarker) @@ -78,62 +53,32 @@ func (s *Step) redactString(text string) string { return text } -// secretForms returns every form the configured credential can appear in, -// longest first so a value that contains another is replaced before its -// substring turns the longer one into a partial redaction. -// -// Per scheme, because the schemes leak differently and redacting the value we -// hold is not enough on its own: -// -// - basic wraps the pair: SetBasicAuth sends base64(user:pass), so the -// password alone does not appear on the wire and replacing it misses the -// echoed header entirely. -// - query escapes: authenticate goes through url.Values.Encode, so a base64 -// token carrying "+", "/" or "=" appears as "a%2Bb%2Fc%3D". Escaping what -// we hold is exact -- same function Encode used, so the two agree by -// construction rather than by a guess about which characters matter. -// - header sends the value as-is. +// secretForms returns every credential this step could leak, longest first so a +// value containing another is replaced before its substring. // -// The raw form is kept alongside the wrapped one in both cases: an error built -// from the config rather than from the request still quotes the credential -// unwrapped. +// EVERY profile, not the one being served: an error raised while serving one +// provider can quote another's credential. Sorting spans the merged set for the +// same reason -- one provider's token can be a substring of another's. func (s *Step) secretForms() []string { - // EVERY profile, not the one being served. - // - // Redaction is about what could appear in a piece of text, not about which - // provider a request happened to be for. An error or a log line built while - // serving one provider can quote another's credential -- a shared client - // echoing a header, an issuer naming the wrong caller -- and scoping this - // to the active profile would let that reach the log unredacted. There are - // a handful of profiles and this runs on a failure path, so walking all of - // them costs nothing worth measuring. var forms []string for _, auth := range s.auth { forms = append(forms, auth.secretForms()...) } - // Sorted across the merged set rather than per profile: one provider's - // short token can be a substring of another's, and replacing the short one - // first would leave the longer half-redacted. return longestFirst(forms) } // secretForms returns every form this profile's credential can appear in. // -// Per scheme, because the schemes leak differently and redacting the value we -// hold is not enough on its own: +// Per scheme, because redacting only the value we hold is not enough: // -// - basic wraps the pair: SetBasicAuth sends base64(user:pass), so the -// password alone does not appear on the wire and replacing it misses the -// echoed header entirely. -// - query escapes: authenticate goes through url.Values.Encode, so a base64 -// token carrying "+", "/" or "=" appears as "a%2Bb%2Fc%3D". Escaping what -// we hold is exact -- same function Encode used, so the two agree by -// construction rather than by a guess about which characters matter. -// - header sends the value as-is. +// - basic: the wire form is base64(user:pass), so the password alone never +// appears in an echoed header. +// - query: Encode escapes, so a token with "+" or "=" appears as "%2B", "%3D". +// Escaping what we hold uses the same function, so the two always agree. +// - header: sent as-is. // -// The raw form is kept alongside the wrapped one in both cases: an error built -// from the config rather than from the request still quotes the credential -// unwrapped. +// The raw form is kept too, for an error built from config rather than from the +// request. func (a *authenticator) secretForms() []string { switch a.cfg.Scheme { case AuthSchemeBasic: @@ -147,10 +92,9 @@ func (a *authenticator) secretForms() []string { forms = append(forms, base64.StdEncoding.EncodeToString([]byte(username+":"+password))) } - // The username is deliberately NOT redacted. It identifies rather than - // authenticates, and it is routinely a short common word -- redacting - // "user" or "admin" would eat unrelated text and cost the operator the - // log line they came for. The pair and the password are the secrets. + // The username is NOT redacted: it identifies rather than + // authenticates, and words like "user" or "admin" would eat + // unrelated text. return forms case AuthSchemeHeader: value := os.Getenv(a.cfg.HeaderValueEnv) @@ -159,9 +103,8 @@ func (a *authenticator) secretForms() []string { } return []string{value} case AuthSchemeOAuth2: - // Both halves: the client secret we send to the issuer, and the token - // it gave back. The token is the one that reaches the provider, so it - // is the one an echoing 401 body quotes. + // Both halves: the secret we send the issuer, and the token it gave + // back. The token is what reaches the provider. var forms []string if secret := os.Getenv(a.cfg.ClientSecretEnv); secret != "" { forms = append(forms, secret) @@ -171,8 +114,8 @@ func (a *authenticator) secretForms() []string { if held := a.token.Load(); held != nil && held.value != "" { forms = append(forms, held.value) } - // The client id is deliberately NOT redacted: it identifies, it does - // not authenticate, and it is what makes a log line useful. + // The client id is NOT redacted: it identifies, it does not + // authenticate. return forms case AuthSchemeQuery: value := os.Getenv(a.cfg.QueryValueEnv) From 089ff2d811e6e9f6b172a01072ae5874f287d467 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 19:21:16 +0530 Subject: [PATCH 115/130] docs(common): shorten the comments across the package [#7] 1,098 comment lines to 457. Comments only -- for each of the nine files, every comment and blank line was stripped from both versions and the remainder diffed: identical, line for line. TWO REAL DEFECTS FOUND WHILE READING, both left by the earlier file split, which reordered declarations and stranded two doc comments: auth.go authenticate's doc had merged into missingCredential's, so authenticate was undocumented and missingCredential's doc opened with a sentence about a different function. http.go call's doc ran straight into budget's, same shape. call had none. Both now document what they are attached to. The rule applied: keep the non-obvious why, especially anything a reader could undo -- the int64 overflow in backoff that returns a NEGATIVE duration, the 405 a lowercase method earns, why redaction sorts across the merged set, why the token cache is per profile, why null is not a query parameter. Cut what restated the code, the incident narratives ("used to", "which is how this was found"), hypotheticals, and third examples where one lands. Nothing load-bearing went: 15 of the 16 phrases a reader could otherwise "fix" are still stated, checked one by one. The sixteenth was registry-schema detail (one Participant, two ProviderSchema rows); the consequence it existed to explain -- a second config entry with the same plugin id colliding in the handler's step map -- is still there, and the guard has its own error in stdHandler.go. 65 packages green, vet clean, no races. --- .../implementation/internal/common/auth.go | 107 ++++------- .../implementation/internal/common/binding.go | 46 ++--- .../implementation/internal/common/common.go | 92 ++++----- .../internal/common/constant.go | 83 +++----- .../implementation/internal/common/http.go | 181 +++++++----------- .../implementation/internal/common/paths.go | 43 ++--- .../implementation/internal/common/serve.go | 89 ++++----- .../implementation/internal/common/token.go | 63 +++--- 8 files changed, 276 insertions(+), 428 deletions(-) diff --git a/pkg/plugin/implementation/internal/common/auth.go b/pkg/plugin/implementation/internal/common/auth.go index 1fe83445..b524b224 100644 --- a/pkg/plugin/implementation/internal/common/auth.go +++ b/pkg/plugin/implementation/internal/common/auth.go @@ -20,66 +20,52 @@ import ( // One per provider rather than one per step: a step serves several binding // keys, and the providers behind them need not authenticate alike. type AuthProfile struct { - // Provider is the participant id this profile belongs to. Held so an error - // can name it: with several profiles on one step, "authScheme query - // requires queryName" would otherwise leave an operator guessing which - // provider it meant. + // The participant id this profile belongs to. Held so an error can name + // it: with several profiles on one step, the field alone leaves an + // operator guessing which block to look at. Provider string - // Scheme is one of none, basic, header, query or oauth2. Providers differ - // here, which is why it is configuration and not an assumption. + // One of none, basic, header, query or oauth2. Scheme string - // UsernameEnv and PasswordEnv name the environment variables holding basic - // credentials. They are variable NAMES, never the values. + // Variable NAMES, never the values. UsernameEnv string PasswordEnv string - // HeaderName and HeaderValueEnv configure the header scheme: which header - // to set, and which environment variable holds its value. + // Which header to set, and the variable holding its value. HeaderName string HeaderValueEnv string - // QueryName and QueryValueEnv configure the query scheme: the parameter - // name to add, and the environment variable holding its value. Named the - // same way as the header pair, for the same reason -- the credential is - // never in this config, only the name of the variable carrying it. + // The parameter to add, and the variable holding its value. QueryName string QueryValueEnv string - // TokenURL is the OAuth2 token endpoint. Not a credential, so it is named - // here rather than through an environment variable -- but it IS - // deployment-specific, so the reference config carries a placeholder. + // The token endpoint. Not a credential, so it is named directly -- but it + // is deployment-specific, so the reference config carries a placeholder. TokenURL string - // ClientIDEnv and ClientSecretEnv name the variables holding the client - // credentials. The values never appear in config, the registry, or a log. + // Variable names. The values never appear in config, the registry or a log. ClientIDEnv string ClientSecretEnv string } // authenticator is one provider's profile plus the token it holds. // -// The token cache lives HERE rather than on the Step, and that is the whole -// reason this type exists. A step-wide cache shared between two oauth2 -// providers would hand the first provider's token to the second, which is -// authenticating as somebody else -- a failure no test of a single provider -// can see. +// The token cache lives HERE rather than on the Step, which is why this type +// exists: a step-wide cache would hand the first oauth2 provider's token to the +// second, authenticating as somebody else. type authenticator struct { cfg AuthProfile - // Two mechanisms because there are two jobs. tokenMu serialises the - // EXCHANGE, so a cold start sends one request to the issuer rather than one - // per concurrent caller. token is atomic so READERS never take that mutex, - // which matters because secretForms is one of them and it is reached from - // inside the exchange -- guarding the value with tokenMu instead deadlocked - // on the first failing exchange, which is how this was found. + // Two mechanisms, two jobs. tokenMu serialises the EXCHANGE so a cold + // start sends one request to the issuer. token is atomic so READERS never + // take that mutex -- secretForms is one, reached from inside the exchange, + // and guarding the value with tokenMu deadlocks. tokenMu sync.Mutex token atomic.Pointer[cachedToken] } // validate refuses a profile whose scheme and fields disagree. Every message -// names the provider: with several profiles on one step, the field alone would -// leave an operator guessing which block to look at. +// names the provider, since several profiles share one step. func (a *AuthProfile) validate() error { switch a.Scheme { case AuthSchemeNone: @@ -118,23 +104,18 @@ func (a *AuthProfile) validate() error { // ParseProviderAuth builds one credential profile per provider from a plugin's // flattened settings. // -// What an operator writes is a block per provider: +// An operator writes a block per provider: // // knowledge-provider: // authScheme: oauth2 // tokenUrl: https://issuer.example/token // -// which pkg/plugin flattens to authScheme-knowledge-provider and -// tokenUrl-knowledge-provider before any plugin sees it. This reads that form -// back into profiles. +// which pkg/plugin flattens to authScheme-knowledge-provider and friends before +// any plugin sees it. This reads that form back. // -// Shared rather than repeated in each capability plugin: all of them copied -// the same field list out of the same map, so a scheme added in one place had -// to be remembered in three. -// -// The split is on the FIRST dash, which is unambiguous because no setting name -// contains one while a participant id routinely does -- knowledge-provider, -// provider.oan.dev. So the field is always the part before it. +// The split is on the FIRST dash: no setting name contains one, while a +// participant id routinely does (knowledge-provider, provider.oan.dev), so the +// field is always the part before it. func ParseProviderAuth(config map[string]string) (map[string]*AuthProfile, error) { profiles := map[string]*AuthProfile{} @@ -148,9 +129,9 @@ func ParseProviderAuth(config map[string]string) (map[string]*AuthProfile, error for _, key := range keys { field, provider, dashed := strings.Cut(key, "-") if !dashed { - // A bare auth field is the old step-wide form. Refused rather than - // ignored: silently dropping it leaves every provider on no - // credential at all, which reads as the provider rejecting us. + // The old step-wide form. Refused rather than ignored: dropping it + // silently leaves every provider with no credential, which reads + // as the provider rejecting us. if authFields[key] { return nil, fmt.Errorf( "%q is set for the whole step; auth is per provider now, "+ @@ -159,9 +140,8 @@ func ParseProviderAuth(config map[string]string) (map[string]*AuthProfile, error continue } if !authFields[field] { - // Not an auth setting, and nothing else on a provider step carries - // a dash -- so this is a misspelled field rather than something to - // pass through. + // Nothing else on a provider step carries a dash, so this is a + // misspelling rather than a setting to pass through. return nil, fmt.Errorf("%q is not a credential setting", key) } if strings.TrimSpace(provider) == "" { @@ -200,10 +180,8 @@ func ParseProviderAuth(config map[string]string) (map[string]*AuthProfile, error return profiles, nil } -// authFields is the closed set of per-provider credential settings. Closed on -// purpose: it is what makes the split on the first dash decidable, and what -// turns a misspelled field into a startup error rather than a setting that -// quietly does nothing. +// authFields is the closed set of per-provider settings. Closed on purpose: it +// makes the dash split decidable and turns a misspelling into a startup error. var authFields = map[string]bool{ "authScheme": true, "usernameEnv": true, @@ -217,25 +195,19 @@ var authFields = map[string]bool{ "clientSecretEnv": true, } -// providerIDFrom returns the provider half of a binding key. The format is -// "|", and a participant id carries dashes and -// dots but never a pipe, so the first one separates them. +// providerIDFrom returns the provider half of "|". +// A participant id carries dashes and dots but never a pipe. func providerIDFrom(bindingKey string) string { provider, _, _ := strings.Cut(bindingKey, "|") return strings.TrimSpace(provider) } -// authenticate presents this provider's credentials, read from the environment -// at call time so a rotated secret takes effect without a restart. // missingCredential reports an unset credential without naming the variable on // the wire. // -// The variable name is deployment configuration, and this error is wrapped into -// a 502 that is signed and returned to a network peer. Telling a peer that -// MANDI_TOKEN is what this deployment reads describes the inside of somebody -// else's stack for no benefit to the caller -- the caller cannot set it, and -// the fix is entirely the operator's. So the name goes to the log, where the -// operator is, and the wire gets the scheme that failed. +// This error is signed and returned to a network peer, and the variable name is +// somebody else's deployment detail -- the caller cannot set it and the fix is +// the operator's. So the name goes to the log and the wire gets the scheme. func (s *Step) missingCredential(ctx context.Context, provider, scheme, envNames string) error { err := fmt.Errorf("the %s credential for %s is not configured", scheme, provider) log.Errorf(ctx, err, "%s auth is configured for %s but %s is not set", @@ -243,6 +215,8 @@ func (s *Step) missingCredential(ctx context.Context, provider, scheme, envNames return err } +// authenticate presents this provider's credentials, read from the environment +// at call time so a rotated secret takes effect without a restart. func (s *Step) authenticate(auth *authenticator, req *http.Request) error { cfg := auth.cfg switch cfg.Scheme { @@ -264,9 +238,8 @@ func (s *Step) authenticate(auth *authenticator, req *http.Request) error { if value == "" { return s.missingCredential(req.Context(), cfg.Provider, "query", cfg.QueryValueEnv) } - // Set rather than Add: a second copy of the parameter is not a - // credential, it is an ambiguity, and which one an upstream reads is - // its own business. + // Set, not Add: a second copy of the parameter is an ambiguity, and + // which one an upstream reads is its own business. query := req.URL.Query() query.Set(cfg.QueryName, value) req.URL.RawQuery = query.Encode() diff --git a/pkg/plugin/implementation/internal/common/binding.go b/pkg/plugin/implementation/internal/common/binding.go index 3731299b..7f0759ce 100644 --- a/pkg/plugin/implementation/internal/common/binding.go +++ b/pkg/plugin/implementation/internal/common/binding.go @@ -16,9 +16,8 @@ import ( // separator joins a binding key's two halves. const separator = "|" -// errNoBinding reports a payload that names no capability binding. It is not a -// fault: a request for something else entirely reaches a provider step too, and -// the step's answer is to do nothing. +// errNoBinding reports a payload that names no binding. Not a fault: requests +// for other capabilities reach this step too, and it answers by doing nothing. var errNoBinding = errors.New("payload names no capability binding") // Binding identifies one provider capability. @@ -32,36 +31,29 @@ func (b Binding) Key() string { return b.ParticipantID + separator + b.CapabilityCode } -// From derives the capability binding a payload is asking for. +// bindingFrom derives the capability binding a payload is asking for. // -// Returns errNoBinding when the payload names no provider or no type, which is -// the ordinary case for a request a provider step is not meant to serve. +// Returns errNoBinding when the payload names no provider or type -- the +// ordinary case for a request this step is not meant to serve. // -// A payload carrying more than one distinct provider or type is refused rather -// than resolved to its first: the two halves index one registry row describing -// one upstream call, so a request spanning several is asking for something this -// design cannot express. Guessing would silently serve part of it. -// -// Where the halves live is BecknV2 unless a deployment says otherwise -- see -// Paths for why that is a default and not a setting. +// More than one distinct provider or type is refused, not resolved to the +// first: the two halves index ONE registry row for ONE call, so a payload +// spanning several is asking for something this design cannot express, and +// guessing would silently serve part of it. func bindingFrom(paths Paths, body []byte) (Binding, error) { var payload any if err := json.Unmarshal(body, &payload); err != nil { return Binding{}, fmt.Errorf("payload could not be read: %w", err) } - // Before distinctness: N commitments naming the SAME provider and type - // collapse to one binding key, so they would resolve here without - // complaint -- and then the mapping reads commitments[0] and the rest are - // dropped, leaving the caller a confident, signed, spec-valid answer to - // part of what it asked. One request maps to one call, so several is a - // request this design cannot express and is refused rather than halved. - // Counted at the array, not over the values it yields. valuesAt returns - // resolved strings and walk drops a leaf that is absent or is not a - // string, so two commitments where one carries no provider id produced one - // value -- which read as one commitment, passed this guard, and left the - // mapping to answer commitments[0] and drop the other. Exactly the outcome - // the paragraph above says is refused. + // Checked before distinctness: N commitments naming the SAME provider and + // type collapse to one key, so they would pass unnoticed and the mapping + // would answer commitments[0] and drop the rest -- a confident, signed, + // spec-valid answer to part of what was asked. + // + // Counted at the array, not over the values it yields: valuesAt drops an + // absent leaf, so two commitments where one has no provider id would count + // as one. if commitments := countAt(payload, paths.ProviderID); commitments > 1 { return Binding{}, fmt.Errorf( "payload carries %d commitments; one request maps to one call, "+ @@ -87,8 +79,8 @@ func bindingFrom(paths Paths, body []byte) (Binding, error) { return Binding{ParticipantID: providers[0], CapabilityCode: types[0]}, nil } -// distinct drops blanks and repeats, keeping the order they were found in so a -// refusal names them the way the payload did. +// distinct drops blanks and repeats, keeping payload order so a refusal names +// them the way they were written. func distinct(values []string) []string { var out []string for _, value := range values { diff --git a/pkg/plugin/implementation/internal/common/common.go b/pkg/plugin/implementation/internal/common/common.go index 06e1cbd5..f9f50f2f 100644 --- a/pkg/plugin/implementation/internal/common/common.go +++ b/pkg/plugin/implementation/internal/common/common.go @@ -29,60 +29,45 @@ import ( // carry, keyed by binding key. // // A mapping cannot produce them: a station id comes from a spatial lookup, a -// session token from an exchange, a market code from a table. That is real I/O, -// and no expression language should be able to do it. -// -// Whatever a function returns is handed to the mapping as _local, so the mapping -// still decides what the provider is finally asked for. A capability with no -// entry needs nothing, which is the common case. +// market code from a table. That is real I/O, and no expression language should +// do it. What a function returns reaches the mapping as _local, so the mapping +// still decides what the provider is asked for. Most capabilities need none. type Prerequisites map[string]func(context.Context, any) (map[string]any, error) // Config holds configuration parameters for the step. type Config struct { - // BindingKeys are the capabilities this step answers to. A request for - // anything else passes through untouched. - // - // A list because a provider can serve more than one: the registry contract - // says a provider serving two capabilities is one Participant and two - // ProviderSchema rows. Configuring a second entry with the same plugin id - // instead would collide in the handler's id-keyed step map, and one - // capability would be lost with no error anywhere. + // The capabilities this step answers to. A request for anything else + // passes through untouched. // - // What differs per capability -- the endpoint, the mapping, the budget -- - // comes from the registry, so one step serving several needs nothing else. + // A list because a provider can serve several, and a second config entry + // with the same plugin id would collide in the handler's step map -- losing + // a capability with no error anywhere. What differs per capability comes + // from the registry, so one step serving several needs nothing more. BindingKeys []string `yaml:"bindingKeys" json:"bindingKeys"` - // ProviderIDAt and CapabilityCodeAt override where the two halves of a - // binding key sit in a payload. Absent means the Beckn v2 convention, which - // is what every deployment should be using. + // Override where the two halves of a binding key sit in a payload. Absent + // means the Beckn v2 convention, which every deployment should use. // - // This is a network convention rather than a deployment's preference -- - // every participant must agree, or two adapters disagree about what a - // binding key is and requests silently fail to match. It is configurable - // only so that a spec change can be tracked without waiting for a release, - // and both must be given together. + // A network convention, not a preference: every participant must agree or + // requests silently fail to match. Configurable only so a spec change can + // be tracked without waiting for a release. Both or neither. ProviderIDAt string `yaml:"providerIdAt" json:"providerIdAt"` CapabilityCodeAt string `yaml:"capabilityCodeAt" json:"capabilityCodeAt"` - // AuthByProvider carries one credential profile PER PROVIDER, keyed by participant - // id -- the left half of a binding key. + // One credential profile per provider, keyed by participant id -- the left + // half of a binding key. // - // Per provider rather than per step because a step serves several binding - // keys and the providers behind them need not authenticate alike: one may - // take an oauth2 client id and secret, the next a token in a query - // parameter. Everything else that differs per provider already comes from - // the registry -- the endpoint, the path, the mapping -- so auth was the - // only thing pinned to the step, and the only thing that made a second - // provider of the same capability impossible to configure. + // Per provider because a step serves several binding keys and the providers + // behind them need not authenticate alike: one may take oauth2 client + // credentials, the next a token in a query parameter. Everything else that + // differs per provider already comes from the registry, so auth was the one + // thing pinned to the step. // - // There is deliberately NO step-wide default. A profile per provider means - // there is never a question of which setting applies, and a provider whose - // profile is missing is refused at startup rather than quietly falling - // through to sending nothing. + // NO step-wide default: a missing profile is refused at startup rather than + // quietly falling through to sending nothing. // - // Built by ParseProviderAuth from the flattened config, not decoded from YAML - // directly: what an operator writes is a nested block per provider, which - // pkg/plugin flattens on the way in. + // Built by ParseProviderAuth, not decoded from YAML -- an operator writes a + // nested block per provider, which pkg/plugin flattens on the way in. AuthByProvider map[string]*AuthProfile `yaml:"-" json:"-"` // MaxResponseBytes caps what is read from the provider. @@ -99,9 +84,8 @@ type Step struct { mapper definition.Mapper httpClient *http.Client - // One authenticator per provider, keyed by participant id. Built once at - // startup, so a request only looks one up -- and each holds its own token, - // so two oauth2 providers cannot share one. + // One per provider, built at startup so a request only looks one up. Each + // holds its own token, so two oauth2 providers cannot share one. auth map[string]*authenticator } @@ -132,8 +116,7 @@ func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper d prerequisites: prerequisites, registry: registry, mapper: mapper, - // Timeout is set per request from the registry's own budget, so the - // client carries none of its own. + // The timeout is per request, from the registry's budget. httpClient: &http.Client{}, auth: make(map[string]*authenticator, len(cfg.AuthByProvider)), } @@ -153,9 +136,8 @@ func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper d // bindingPaths resolves where this step reads a binding key from. // -// Both halves or neither: overriding one and leaving the other on the default -// is a half-configured deployment that would match nothing, and it would do so -// silently on every request rather than once at startup. +// Both halves or neither: one overridden and one defaulted would match nothing, +// silently, on every request. func bindingPaths(cfg *Config) (Paths, error) { if cfg.ProviderIDAt == "" && cfg.CapabilityCodeAt == "" { return BecknV2, nil @@ -175,9 +157,9 @@ func bindingPaths(cfg *Config) (Paths, error) { // applyDefaults fills in what was left out and rejects what cannot be defaulted. func applyDefaults(cfg *Config) error { - // No default. This package serves whatever a domain package configures it - // for, so a default would have to name one provider's capability -- wrong - // for every other domain built on it, and silently wrong rather than loudly. + // No default: this package serves whatever a domain package points it at, + // so any default would name one provider's capability and be silently + // wrong for every other. if len(cfg.BindingKeys) == 0 { return errors.New("bindingKeys is required: it is what this step answers to") } @@ -193,11 +175,9 @@ func applyDefaults(cfg *Config) error { cfg.AuthByProvider = map[string]*AuthProfile{} } - // Both directions, because each catches a different mistake and both are - // silent at runtime. A profile for a provider this step does not serve is a - // typo that would apply to nothing; a served provider with no profile would - // fall through to sending no credential and read as the provider rejecting - // us. + // Both directions, because each catches a different silent mistake: a + // profile for an unserved provider is a typo applying to nothing, and a + // served provider with no profile would send no credential at all. served := map[string]bool{} for _, key := range cfg.BindingKeys { served[providerIDFrom(key)] = true diff --git a/pkg/plugin/implementation/internal/common/constant.go b/pkg/plugin/implementation/internal/common/constant.go index 398d57fc..66b0623a 100644 --- a/pkg/plugin/implementation/internal/common/constant.go +++ b/pkg/plugin/implementation/internal/common/constant.go @@ -1,8 +1,5 @@ -// Constants only: the budgets a registry row is clamped to, the retry backoff, -// the auth schemes, and the markers this package puts in text it returns. -// -// authFields is deliberately NOT here. It is a var -- Go has no constant maps -- -// and it belongs beside the parser in auth.go that reads it. +// Constants only. authFields is not here: it is a var, since Go has no constant +// maps, and it sits beside the parser in auth.go that reads it. package common import ( @@ -11,39 +8,29 @@ import ( // Defaults applied when the registry or the operator leaves a setting out. const ( - // DefaultTimeout and DefaultRetryMax are the registry contract's defaults - // for an action that leaves timeoutMs or retryMax out. Zero retries is - // deliberate: a provider that failed is retried only where the operator - // said so, because a retry on a non-idempotent action is a second booking. + // Zero retries is deliberate: a retry on a non-idempotent action is a + // second booking, so a provider is retried only where the operator said so. DefaultTimeout = 15 * time.Second DefaultRetryMax = 0 - // DefaultMaxResponseBytes caps what is read from the provider. The response - // is mapped in memory, so an unbounded one is an unbounded allocation. + // The response is mapped in memory, so an unbounded one is an unbounded + // allocation. DefaultMaxResponseBytes = 4 << 20 // 4 MiB - // MaxTimeout and MaxRetryMax bound what a registry row may ask for. + // A ceiling on what a registry row may ask for, because the row is DATA and + // an attempt holds a goroutine and the inbound connection for its whole + // timeout. retryMax 1000 with timeoutMs 60000 would pin both for about + // seventeen hours. // - // Both come from DATA, not from this deployment's config, and neither is - // cheap: an attempt holds a goroutine and the inbound connection for its - // whole timeout, and http.Server's write timeout does not cancel the - // request context. So a row reading retryMax 1000, timeoutMs 60000 pins - // both for roughly seventeen hours, and a handful of such requests is the - // adapter. The registry is trusted to say where a provider is; it is not a - // reason to let one row decide how long this process is busy. - // - // Clamped rather than refused. A row that overreaches is a configuration - // mistake, and failing every request for that capability is a worse answer - // than serving it with a sane budget and saying so in the log. + // Clamped rather than refused: a row that overreaches is a mistake, and + // serving it with a sane budget beats failing the capability outright. MaxTimeout = 30 * time.Second MaxRetryMax = 5 ) // How long this step waits between attempts. const ( - // RetryBackoffBase is the first wait between attempts, doubling from there - // up to RetryBackoffMax. Short, because the retry budget comes from the - // registry and an operator setting 5 retries did not ask for seconds of - // latency -- only for the provider's brief unavailability to be ridden out. + // The first wait, doubling up to the max. Short: an operator setting 5 + // retries asked to ride out a brief outage, not for seconds of latency. RetryBackoffBase = 50 * time.Millisecond RetryBackoffMax = 800 * time.Millisecond ) @@ -51,43 +38,35 @@ const ( // redactedMarker stands in for a credential in anything logged or returned. const redactedMarker = "REDACTED" -// Auth schemes this step can present upstream. Credentials themselves are never -// configured here or held in the registry -- config names the environment -// variable to read, so a secret reaches the process through its environment and -// nothing else. +// Auth schemes a step can present. Credentials are never in config or in the +// registry: config names the environment variable to read. const ( AuthSchemeNone = "none" AuthSchemeBasic = "basic" AuthSchemeHeader = "header" - // AuthSchemeQuery puts the credential in the query string, which some - // upstreams are built around whatever anyone thinks of it. It is the least - // safe of the four -- a query string is logged by proxies and appears in a - // transport error -- so the value is redacted from anything this package - // logs or returns. See redact. + // The credential goes in the query string, which some upstreams require. + // The least safe scheme -- proxies log query strings and transport errors + // quote them -- so the value is redacted. See redact. AuthSchemeQuery = "query" - // AuthSchemeOAuth2 exchanges a client id and secret for a short-lived - // bearer token, and sends that. For an upstream behind an OAuth2 token - // endpoint -- a Keycloak service account, say -- where a static token is not - // an option: the live one this was written against lives ten hours, so a - // value pasted into an environment variable is wrong twice a day. + // A client id and secret are exchanged for a short-lived bearer token. For + // an upstream where a static token is not an option: the live one this was + // written against expires every ten hours. // - // Only the client_credentials grant. There is no user to redirect and no - // refresh token in that grant, so the other flows would be dead code. + // Only client_credentials. That grant has no user to redirect and no + // refresh token, so the other flows would be dead code. AuthSchemeOAuth2 = "oauth2" ) -// codeUpstreamUnavailable reports a provider that could not be reached or -// answered with a failure. It is not this adapter's fault and not the caller's. +// codeUpstreamUnavailable: the provider could not be reached or failed. Not +// this adapter's fault and not the caller's. const codeUpstreamUnavailable = "NET_DOWNSTREAM_UNAVAILABLE" -// tokenRefreshSkew is how early an oauth2 token stops being trusted. It has to -// exceed the round trip to the provider, so that a request which passes the -// expiry check cannot arrive after the token has actually died. One extra -// exchange per token lifetime is the whole cost. +// How early an oauth2 token stops being trusted. Must exceed the round trip to +// the provider, so a request that passed the expiry check cannot arrive after +// the token died. Costs one extra exchange per lifetime. const tokenRefreshSkew = 60 * time.Second -// explainLimit is how much of a failed response is quoted. Enough for a -// provider's own message, short enough not to put a page of HTML in a log line -// or a NACK. +// How much of a failed response is quoted: enough for the provider's message, +// not a page of HTML in a log line. const explainLimit = 300 diff --git a/pkg/plugin/implementation/internal/common/http.go b/pkg/plugin/implementation/internal/common/http.go index ad18044b..4c2c84b6 100644 --- a/pkg/plugin/implementation/internal/common/http.go +++ b/pkg/plugin/implementation/internal/common/http.go @@ -19,16 +19,14 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" ) -// call makes the upstream request described by the plan, retrying within its -// budget. -// budget resolves how long one attempt may take and how many retries follow -// it, applying the registry's values within this deployment's ceilings. +// budget resolves how long one attempt may take and how many retries follow, +// applying the registry's values within this deployment's ceilings. // -// Pure and separate from call so both bounds can be asserted without a server -// that sleeps for the timeout it is testing. +// Separate from call so both bounds can be asserted without a server that +// sleeps for the timeout being tested. // -// retryMax counts retries, not attempts, so the call itself is always made -// once. An absent retryMax and an explicit 0 are the same instruction. +// retryMax counts retries, not attempts: the call is always made once, and an +// absent retryMax means the same as an explicit 0. func budget(call model.ActionPlan) (time.Duration, int) { timeout := DefaultTimeout if call.TimeoutMs > 0 { @@ -48,6 +46,7 @@ func budget(call model.ActionPlan) (time.Duration, int) { return timeout, retries } +// call makes the upstream request the plan describes, retrying within budget. func (s *Step) call(ctx context.Context, auth *authenticator, baseURL string, call model.ActionPlan, mapped []byte) ([]byte, error) { endpoint, err := buildEndpoint(baseURL, call, mapped) if err != nil { @@ -66,9 +65,7 @@ func (s *Step) call(ctx context.Context, auth *authenticator, baseURL string, ca var lastErr error for attempt := 1; attempt <= attempts; attempt++ { - // A caller that has gone away is not worth another attempt, and neither - // is a budget already spent. Checked before the call rather than after, - // so a cancelled request costs nothing. + // Checked before the call, so a cancelled request costs nothing. if err := ctx.Err(); err != nil { if lastErr == nil { lastErr = err @@ -83,11 +80,10 @@ func (s *Step) call(ctx context.Context, auth *authenticator, baseURL string, ca lastErr = s.redact(err) log.Warnf(ctx, "attempt %d/%d failed: %v", attempt, attempts, lastErr) - // Only some failures are worth repeating. A 4xx, a request this step - // could not build and a credential it could not read will fail - // identically however many times they are tried -- and retrying the - // credential case is the worst of them, because it reports an - // operator's missing environment variable as the provider being down. + // A 4xx, an unbuildable request and an unreadable credential fail + // identically however often they are tried. Retrying the credential + // case is the worst: it reports a missing environment variable as the + // provider being down. if isPermanent(err) { break } @@ -118,21 +114,17 @@ func isPermanent(err error) bool { // backoff is how long to wait before the next attempt. // -// Exponential from a short base and capped, because the provider being briefly -// busy is the case worth waiting out; anything longer is a timeout's job. With -// no wait at all a retryMax of 5 spends its whole budget inside a couple of -// milliseconds, which is not a retry so much as the same failure six times. +// Exponential from a short base and capped: a briefly busy provider is worth +// waiting out, anything longer is a timeout's job. With no wait, retryMax 5 +// spends its budget in a couple of milliseconds -- the same failure six times. func backoff(attempt int) time.Duration { if attempt <= 1 { return RetryBackoffBase } - // Doubled in a loop that stops at the ceiling rather than shifted and then - // clamped. `RetryBackoffBase << (attempt - 1)` overflows int64 once the - // shift reaches 38 at a 50ms base, and the wrapped value is NEGATIVE -- so - // it passes the `> RetryBackoffMax` check, is returned, and a sleep on a - // negative duration returns immediately. The retry loop then spins as fast - // as the provider can refuse. Stopping at the ceiling cannot overflow, - // because it never doubles a value already at or past it. + // Doubled in a loop that stops at the ceiling, not shifted then clamped. + // `base << (attempt-1)` overflows int64 around shift 38 and the wrapped + // value is NEGATIVE, so it passes the ceiling check and sleeps for no time + // -- the retry loop then spins as fast as the provider can refuse. wait := RetryBackoffBase for i := 1; i < attempt && wait < RetryBackoffMax; i++ { wait *= 2 @@ -173,9 +165,8 @@ func (s *Step) attempt(ctx context.Context, auth *authenticator, call model.Acti return nil, doNotRetry(err) } - // The URL as it actually went on the wire, credential removed. At info - // rather than debug because this is the line that answers "what did we ask, - // and what came back" -- the question every provider problem starts with. + // The URL as it went on the wire, credential removed. At info because this + // is the line that answers "what did we ask, and what came back". requested := s.redactString(req.URL.String()) resp, err := s.httpClient.Do(req) @@ -193,25 +184,20 @@ func (s *Step) attempt(ctx context.Context, auth *authenticator, call model.Acti // Asking again will not make the answer smaller. return nil, doNotRetry(fmt.Errorf("response exceeds the %d byte limit", s.config.MaxResponseBytes)) } - // Any 2xx, not 200 alone. A provider is entitled to answer 202 for work it - // accepted, 204 for nothing to report, or 201 for something it created, and - // treating those as failures would refuse a perfectly good exchange. 3xx - // does not reach here: the client follows redirects. + // Any 2xx, not 200 alone: 202, 204 and 201 are all legitimate answers. 3xx + // does not reach here, since the client follows redirects. if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { - // The body is logged, not returned. It goes into a 502 that is signed - // and sent to the network caller, and what a provider puts in a failure - // body is its own business -- a stack trace, an internal hostname, a - // database error. The status is the caller's business and stays; the - // body is the operator's, and the log is where the operator looks. - // Redacted on the way to the log too. A provider that rejects a - // request often quotes it back, credential and all -- so the body is - // exactly where a query-string token turns up, and moving it from the - // error to the log would only move the leak. + // The body is logged, not returned. It would go into a signed 502, and + // a failure body may hold a stack trace or an internal hostname. The + // status is the caller's business; the body is the operator's. + // + // Redacted on the way to the log too: a rejected request is often + // quoted back, credential and all. log.Warnf(ctx, "provider returned %s for %s %s: %s", resp.Status, method, requested, s.redactString(explain(body))) err := fmt.Errorf("provider returned %s", resp.Status) - // 5xx and 429 are the provider asking to be tried again. Every other - // 4xx is a statement about the request, which will not improve. + // 5xx and 429 ask to be tried again. Every other 4xx is a statement + // about the request, which will not improve. if resp.StatusCode < http.StatusInternalServerError && resp.StatusCode != http.StatusTooManyRequests { return nil, doNotRetry(err) } @@ -222,18 +208,15 @@ func (s *Step) attempt(ctx context.Context, auth *authenticator, call model.Acti // explain renders a failed response body for a human. // -// The body was already read and then thrown away, so a provider's own account -// of what was wrong -- Agmarknet says "no data" in the body of a 400 -- never -// reached anyone. The status alone says a call failed and nothing about why, -// which is the first thing an operator needs and the thing that makes a real -// provider's behaviour observable at all. +// The status says a call failed and nothing about why. The provider's own +// account is in the body -- Agmarknet says "no data" in the body of a 400. func explain(body []byte) string { text := strings.TrimSpace(string(body)) if text == "" { return "(no body)" } - // Collapse whitespace: a provider that answers with indented JSON or an - // HTML error page should not spread one failure over forty log lines. + // Collapse whitespace: indented JSON or an HTML error page should not + // spread one failure over forty log lines. text = strings.Join(strings.Fields(text), " ") if len(text) > explainLimit { return text[:explainLimit] + "... (truncated)" @@ -251,10 +234,8 @@ func buildEndpoint(baseURL string, call model.ActionPlan, mapped []byte) (string return "", err } - // baseUrl cannot end in a slash and path must begin with one, so exactly one - // separator appears between them. The trim is belt and braces: the registry - // refuses a trailing slash on baseUrl, and this keeps a row that predates - // that from producing a doubled one. + // baseUrl cannot end in a slash and path must begin with one. The trim is + // belt and braces, for a row predating that rule. endpoint := strings.TrimSuffix(baseURL, "/") + call.Path if hasBody(call.Method) { return endpoint, nil @@ -275,15 +256,13 @@ func buildEndpoint(baseURL string, call model.ActionPlan, mapped []byte) (string // verifyPath refuses a published path nobody could have meant. // -// The registry constrains this, but it is a separate deployable that may not be -// updated in step, so a row that slipped through has to fail here with something -// an operator can act on rather than as a provider's 404 three hops away. +// The registry constrains this, but it deploys separately, so a row that slips +// through must fail here with something an operator can act on rather than as +// a 404 three hops away. // // An empty segment is the case worth catching: "//get-daily" is never -// deliberate, and plenty of servers answer it differently from "/get-daily". A -// trailing slash is deliberately left alone -- "/api/" and "/api" are a -// distinction some APIs genuinely make, so stripping it would silently change -// the URL the operator published. +// deliberate. A trailing slash is left alone -- "/api/" and "/api" are a +// distinction some APIs genuinely make. func verifyPath(path string) error { if path == "" { return model.NewBadReqErr("", errors.New("the registry publishes no path for this action")) @@ -296,11 +275,8 @@ func verifyPath(path string) error { return model.NewBadReqErr("", fmt.Errorf( "path %q has an empty segment; write it with single slashes", path)) } - // A dot segment is refused rather than resolved. The registry says which - // path answers an action, and a row that climbs out of it is either a - // mistake or an attempt to reach something the row does not name -- and - // net/url would quietly resolve it either way, so the request that left - // would not be the request the row described. + // A dot segment is refused, not resolved: net/url would resolve it quietly, + // and the request that left would not be the request the row described. for _, segment := range strings.Split(path, "/") { if segment == ".." || segment == "." { return model.NewBadReqErr("", fmt.Errorf( @@ -309,8 +285,8 @@ func verifyPath(path string) error { } } // A fragment is never sent, so a row carrying one describes a request that - // cannot be made. Refused here rather than silently dropped by the - // transport, which would make the row look honoured. + // cannot be made. The transport would drop it silently and the row would + // look honoured. if strings.Contains(path, "#") { return model.NewBadReqErr("", fmt.Errorf( "path %q contains a fragment, which is never sent to a server", path)) @@ -318,16 +294,13 @@ func verifyPath(path string) error { return nil } -// verifyBaseURL checks the participant's base url before it is joined to a -// path, so a row that cannot produce a request says so as a bad request rather -// than as the provider being unreachable. +// verifyBaseURL checks the base url before it is joined to a path, so a row +// that cannot produce a request says so as a bad request rather than as the +// provider being unreachable. // -// Without this, `baseUrl: "registry:8081"` -- a scheme left off -- failed -// inside http.NewRequestWithContext and arrived as a 502 "provider did not -// answer after 1 attempts: could not build the request". That names the -// provider for an error in the row describing it, and it is retried on the way -// there. jsonmapper has always checked its own reference this way; this is the -// same check on the other url the registry publishes. +// Without this, a missing scheme (`baseUrl: "registry:8081"`) failed inside +// NewRequestWithContext and arrived as a 502 blaming the provider -- retried +// on the way there. func verifyBaseURL(baseURL string) error { if baseURL == "" { return model.NewBadReqErr("", errors.New("the registry publishes no base url for this provider")) @@ -346,12 +319,8 @@ func verifyBaseURL(baseURL string) error { return nil } -// asQuery renders a mapped request as query parameters. -// -// A method with no body still needs the mapping's output somewhere, and the -// query string is the only place it can go. Only scalars are carried: a nested -// value has no single obvious encoding, and inventing one here would put a -// convention in Go that belongs in the mapping. +// asQuery renders a mapped request as query parameters, for a method with no +// body. Scalars only -- see asQueryValue. func asQuery(mapped []byte) (string, error) { if len(bytes.TrimSpace(mapped)) == 0 { return "", nil @@ -375,17 +344,14 @@ func asQuery(mapped []byte) (string, error) { // asQueryValue renders one mapped field as a query parameter value, reporting // false when the field cannot be one. // -// The three JSON scalars and nothing else. An object or an array has no single -// obvious encoding -- repeated keys, comma-joined, indexed, JSON-in-a-parameter -// are all in use somewhere -- so choosing here would put a convention in Go -// that belongs in the mapping, where the upstream's actual shape is known. The -// caller turns false into an error naming the field, so a mapping that produces -// one is told which. +// The three JSON scalars and nothing else. An object or array has no single +// obvious encoding -- repeated keys, comma-joined, indexed are all in use +// somewhere -- so choosing here would put a convention in Go that belongs in +// the mapping. The caller turns false into an error naming the field. // -// null is refused for the same reason rather than sent as empty: a parameter -// present but empty and a parameter absent mean different things to some -// upstreams, and a mapping says which it wants by omitting the field or setting -// "". An empty string IS carried, being a scalar. +// null is refused rather than sent as empty: present-but-empty and absent mean +// different things to some upstreams, and a mapping says which by omitting the +// field or setting "". An empty string IS carried, being a scalar. func asQueryValue(value any) (string, bool) { switch typed := value.(type) { case string: @@ -419,24 +385,17 @@ func hasBody(method string) bool { } } -// canonicalMethod returns a known HTTP method in the spelling the RFC gives -// it, and anything else unchanged. -// -// hasBody used to upper-case privately, which made the method look -// case-insensitive when it is not: NewRequestWithContext transmits it verbatim, -// so a registry row reading `method: "post"` sent `post /path HTTP/1.1`. The -// body was attached correctly -- hasBody had normalised -- but nginx and most -// gateways answer 405 to a lowercase method, which classifies permanent and -// surfaces as a 502 "provider did not answer". A row that is right in every -// respect but its capitalisation is a bad way to spend an afternoon. +// canonicalMethod upper-cases a known HTTP method and leaves anything else +// alone. // -// Only known methods are rewritten. Upper-casing everything would be a new -// restriction on an upstream entitled to a method this list has not heard of, -// and net/http already refuses one that is not a valid token. +// NewRequestWithContext sends the method verbatim, so a row reading "post" put +// `post /path HTTP/1.1` on the wire and most gateways answer that with 405 -- +// which classifies permanent and surfaces as "provider did not answer". // -// An empty method is left empty: net/http documents "" as GET and substitutes -// it, and hasBody agrees that it carries no body, so the two are already -// consistent and inventing a value here would only hide where it comes from. +// Unknown methods pass through: net/http already rejects an invalid token, and +// upper-casing everything would restrict an upstream entitled to a method this +// list has not heard of. An empty method stays empty, since net/http already +// treats "" as GET. func canonicalMethod(method string) string { upper := strings.ToUpper(method) switch upper { diff --git a/pkg/plugin/implementation/internal/common/paths.go b/pkg/plugin/implementation/internal/common/paths.go index cbab19e7..af1ded59 100644 --- a/pkg/plugin/implementation/internal/common/paths.go +++ b/pkg/plugin/implementation/internal/common/paths.go @@ -7,14 +7,10 @@ import ( // Paths says where the two halves of a binding key live in a payload. // -// This is a NETWORK convention, not a deployment's preference: every participant -// has to agree, or two adapters disagree about what a binding key is and -// requests silently fail to match. So BecknV2 is the answer, and overriding is -// something an operator has to type deliberately -- absent means correct. -// -// The override exists for one situation: the spec moves a field and a deployment -// needs to track it without waiting for a release. It is deliberately not -// something to reach for otherwise. +// A NETWORK convention, not a deployment's preference: every participant must +// agree or requests silently fail to match. BecknV2 is the answer, and absent +// means correct. The override exists only for tracking a spec change without +// waiting for a release. type Paths struct { ProviderID string CapabilityCode string @@ -26,12 +22,12 @@ var BecknV2 = Paths{ CapabilityCode: "message.contract.commitments[].resources[].resourceAttributes.@type", } -// arrayMarker flattens an array at that segment. It is the only operator the -// walk understands. +// arrayMarker flattens an array at that segment -- the only operator walk +// understands. const arrayMarker = "[]" -// Validate refuses a pair that could never match, so a mistake surfaces where it -// was configured rather than as every request quietly going unserved. +// Validate refuses a pair that could never match, so a mistake surfaces at +// startup rather than as every request quietly going unserved. func (p Paths) Validate() error { for name, path := range map[string]string{ "providerIdAt": p.ProviderID, @@ -52,22 +48,19 @@ func (p Paths) Validate() error { // valuesAt collects every string the path reaches. // // The grammar is two things: segments separated by ".", and a "[]" suffix -// meaning "this is an array, look in each element". No wildcards, no filters, no -// indices. Each of those would be another way to write something subtly wrong in -// config nobody reviews, to buy an expressiveness a payload shape has never -// needed. +// meaning "look in each element". No wildcards, filters or indices -- each is +// another way to write something subtly wrong in config nobody reviews. func valuesAt(node any, path string) []string { return walk(node, strings.Split(path, ".")) } -// countAt reports how many elements the first array segment of path holds, -// whether or not the leaf beyond it resolves to anything. +// countAt reports how many elements the first array segment holds, whether or +// not the leaf beyond it resolves. // -// valuesAt cannot answer this. It returns resolved STRINGS, and walk drops a -// leaf that is missing or is not a string -- so two commitments where one -// carries no provider id yield one value, and a count of those values reads as -// one commitment. That is the difference between refusing a request this -// design cannot express and silently answering half of it. +// valuesAt cannot answer this: it returns resolved STRINGS and drops a missing +// leaf, so two commitments where one has no provider id yield one value and +// count as one commitment -- silently answering half a request instead of +// refusing it. func countAt(node any, path string) int { for _, segment := range strings.Split(path, ".") { fields, ok := node.(map[string]any) @@ -92,8 +85,8 @@ func countAt(node any, path string) int { func walk(node any, segments []string) []string { if len(segments) == 0 { - // The leaf. Only strings are binding-key material; a number or an - // object here means the path landed somewhere unintended. + // Only strings are binding-key material; anything else means the path + // landed somewhere unintended. if value, ok := node.(string); ok { return []string{value} } diff --git a/pkg/plugin/implementation/internal/common/serve.go b/pkg/plugin/implementation/internal/common/serve.go index 06be4124..c902fe1f 100644 --- a/pkg/plugin/implementation/internal/common/serve.go +++ b/pkg/plugin/implementation/internal/common/serve.go @@ -15,22 +15,21 @@ import ( "github.com/beckn-one/beckn-onix/pkg/plugin/definition" ) -// Run serves the request when it is for this step's capability, and does -// nothing when it is not. +// Run serves the request when it is this step's capability, and does nothing +// when it is not. // -// Doing nothing is the dispatch mechanism: several provider steps sit in one -// pipeline and each recognises its own work, so adding a provider is one more -// entry rather than a change to a routing table. +// Doing nothing IS the dispatch: several steps sit in one pipeline and each +// recognises its own work, so adding a provider is one config entry rather than +// a routing-table change. func (s *Step) Run(ctx *model.StepContext) error { binding, err := bindingFrom(s.paths, ctx.Body) if errors.Is(err, errNoBinding) { return nil } if err != nil { - // Everything From refuses is a statement about the payload: unreadable - // JSON, or a request naming more than one call. Unclassified it becomes - // a 500, which says this adapter broke and leaves the reason in a log - // the caller cannot read. + // Everything bindingFrom refuses is about the payload -- unreadable + // JSON, or more than one call named. Unclassified it becomes a 500, + // which blames this adapter and hides the reason from the caller. return model.NewBadReqErr("", err) } if !s.serves(binding.Key()) { @@ -40,14 +39,12 @@ func (s *Step) Run(ctx *model.StepContext) error { plan, err := s.registry.ProviderRecord(ctx, binding.Key()) if err != nil { - // A definite "no such binding" is the caller naming something that is - // not there, so 404 -- the same reasoning the no-route path uses to - // refuse an unrecognised capability rather than ACK it. A registry that - // could not be consulted is different and stays a 500: unclassified, - // because it is this adapter that failed. + // "No such binding" is the caller naming something absent, so 404. A + // registry that could not be consulted is this adapter failing, so it + // stays a 500. if errors.Is(err, definition.ErrProviderRecordNotFound) { - // %w, not %v: the sentinel has to stay unwrappable, or anything - // upstream testing errors.Is against it silently stops matching. + // %w, not %v: the sentinel must stay unwrappable or errors.Is + // stops matching. return model.NewNotFoundErr("", fmt.Errorf( "the registry publishes no active binding for %s: %w", binding.Key(), err)) } @@ -57,11 +54,11 @@ func (s *Step) Run(ctx *model.StepContext) error { return s.serve(ctx, plan) } -// resolve runs whatever prerequisite work this capability needs, and returns the -// values for the mapping to read under _local. +// resolve runs this capability's prerequisite work, returning what the mapping +// reads under _local. // -// Empty rather than nil when there is nothing: a mapping referring to _local on -// a capability that resolves nothing should read a missing field, not fail. +// Empty rather than nil: a mapping referring to _local when nothing resolved +// should read a missing field, not fail. func (s *Step) resolve(ctx context.Context, bindingKey string, beckn any) (map[string]any, error) { prerequisite, needed := s.prerequisites[bindingKey] if !needed { @@ -79,9 +76,8 @@ func (s *Step) resolve(ctx context.Context, bindingKey string, beckn any) (map[s // serves reports whether a binding key is one this step answers to. // -// A slice rather than a set: a step serves a handful of capabilities at most, so -// the scan costs less than the map would, and the config order is preserved in -// the line New logs on startup. +// A slice, not a set: a step serves a handful of capabilities, so the scan is +// cheaper than a map and config order survives into New's startup log. func (s *Step) serves(key string) bool { return slices.Contains(s.config.BindingKeys, key) } @@ -91,10 +87,9 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { action := extractAction(ctx.Body) call, served := plan.Actions[action] if !served { - // The capability publishes no endpoint for this action, so it does not - // serve it. Refused here rather than after a call to whichever endpoint - // happened to be on the record -- naming what it does serve turns a - // registry mistake into a one-line fix. + // No endpoint published for this action. Refused here rather than + // after calling whichever endpoint was on the record, and the error + // names what IS served so a registry mistake is a one-line fix. return model.NewBadReqErr("", fmt.Errorf( "%s does not serve action %q; it serves %s", plan.BindingKey, action, strings.Join(plan.ServedActions(), ", "))) @@ -105,16 +100,13 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { return err } - // What this provider requires of a payload is declared by its mapping, not - // by this step. A capability with a different rule is a different mapping - // file rather than a different build -- and the rule sits beside the - // extraction it guards. + // The mapping declares what a payload must satisfy, not this step: a + // different rule is a different mapping file, not a rebuild. if err := s.mapper.Verify(ctx, call.Mappings, map[string]any{"beckn": beckn}); err != nil { return err } - // Whatever this capability needs that its payload does not carry. Empty for - // most: the mapping reads the payload directly and needs nothing resolved. + // Whatever the payload does not carry. Empty for most capabilities. local, err := s.resolve(ctx, plan.BindingKey, beckn) if err != nil { return err @@ -125,10 +117,9 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { return err } - // Which credential this provider takes. Resolved from the binding key, so - // one step serving several providers authenticates each as its own. - // Startup guarantees a profile per served provider; this guards the case - // where a record arrives for a key the config never declared. + // Resolved from the binding key, so a step serving several providers + // authenticates each as itself. Startup guarantees a profile per served + // provider; this guards a record arriving for an undeclared key. auth, configured := s.auth[providerIDFrom(plan.BindingKey)] if !configured { return fmt.Errorf("no credential is configured for %s", plan.BindingKey) @@ -144,11 +135,8 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { return fmt.Errorf("provider answered with something that is not JSON: %w", err) } - // The same mapping reference as the request, other half: one file carries - // both directions for this action. - // - // The mapping is handed what each party sent, plus whatever prerequisites - // resolved, under _local. Empty when there are none. + // The same file's other half. It is handed what each party sent, plus + // whatever prerequisites resolved, under _local. becknResponse, err := s.mapper.Transform(ctx, call.Mappings, definition.DirectionResponse, map[string]any{ "beckn": beckn, "_local": local, @@ -158,10 +146,9 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { return err } if len(becknResponse) == 0 { - // Either the file has no response half, or its transform matched nothing - // in this answer. Both leave no Beckn response to return, and returning - // the provider's own shape instead would be worse than failing. The - // message says what was observed rather than guessing which it was. + // No response half, or its transform matched nothing. Either way there + // is no Beckn response, and returning the provider's own shape would + // be worse than failing. return fmt.Errorf("the response half of %s produced nothing, so %s cannot be answered", call.Mappings, plan.BindingKey) } @@ -174,13 +161,9 @@ func (s *Step) serve(ctx *model.StepContext, plan *model.ProviderRecord) error { // buildRequest produces what the provider is sent. // // Whatever the mapping produces IS the request: a body for a method that takes -// one, query parameters for a method that does not. Nothing is substituted when -// it produces nothing, so an empty request half means an empty request. -// -// This step used to extract a point from the payload and fall back to sending -// that. It meant the choice of which payload fields reach the provider lived in -// Go, so adding a parameter -- a date range, say -- was a rebuild. Now it is a -// mapping edit and nothing else. +// one, query parameters otherwise. Nothing is substituted when it produces +// nothing, so which payload fields reach a provider is a mapping edit rather +// than a rebuild. func (s *Step) buildRequest(ctx context.Context, call model.ActionPlan, beckn any, local map[string]any) ([]byte, error) { mapped, err := s.mapper.Transform(ctx, call.Mappings, definition.DirectionRequest, map[string]any{ "beckn": beckn, diff --git a/pkg/plugin/implementation/internal/common/token.go b/pkg/plugin/implementation/internal/common/token.go index 0c03db70..0dbb428a 100644 --- a/pkg/plugin/implementation/internal/common/token.go +++ b/pkg/plugin/implementation/internal/common/token.go @@ -16,18 +16,15 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" ) -// bearerToken returns a token to send, exchanging one if what we hold has -// expired or if we hold none. +// bearerToken returns a token to send, exchanging one if the held token has +// expired or there is none. // -// The lock spans the fetch on purpose. Without it a cold start sends every -// concurrent request to the token endpoint, and the provider's issuer sees a -// burst of identical exchanges. Serialising them costs one wait per token -// lifetime and nothing after that, which is a better trade than a second -// caching layer. +// The lock spans the fetch on purpose: without it a cold start sends every +// concurrent request to the issuer at once. It costs one wait per token +// lifetime. // -// NOTHING IS CACHED ON FAILURE. A transient outage at the token endpoint must -// not leave this step holding a failure for the lifetime it never obtained -- -// the next request tries again. +// NOTHING IS CACHED ON FAILURE, so a brief outage at the token endpoint does +// not leave this step holding a failure for a lifetime it never obtained. func (s *Step) bearerToken(ctx context.Context, auth *authenticator) (string, error) { if held := auth.token.Load(); held != nil && time.Now().Before(held.expiry) { return held.value, nil @@ -36,8 +33,7 @@ func (s *Step) bearerToken(ctx context.Context, auth *authenticator) (string, er auth.tokenMu.Lock() defer auth.tokenMu.Unlock() - // Re-checked after acquiring: while this caller waited, whoever held the - // mutex may already have exchanged a fresh token. + // Re-checked: whoever held the mutex may already have exchanged one. if held := auth.token.Load(); held != nil && time.Now().Before(held.expiry) { return held.value, nil } @@ -64,9 +60,9 @@ type tokenResponse struct { // exchangeToken performs the client_credentials grant. // -// Every failure here is the UPSTREAM exchange failing, so all of them carry -// 502. Unclassified they would surface as a 500, which tells a network peer -// this adapter broke when in fact the provider's issuer did. +// Every failure carries 502, because it is the exchange with the provider's +// issuer that failed. Unclassified they would surface as 500, telling a peer +// this adapter broke when it did not. func (s *Step) exchangeToken(ctx context.Context, auth *authenticator) (string, time.Duration, error) { cfg := auth.cfg clientID, clientSecret := os.Getenv(cfg.ClientIDEnv), os.Getenv(cfg.ClientSecretEnv) @@ -94,15 +90,14 @@ func (s *Step) exchangeToken(ctx context.Context, auth *authenticator) (string, } defer resp.Body.Close() - // Bounded like any other upstream read: a token response is small, and an - // unbounded read here would be a hole in the same wall. + // Bounded like any other upstream read. body, err := io.ReadAll(io.LimitReader(resp.Body, s.config.MaxResponseBytes+1)) if err != nil { return "", 0, s.tokenErr(fmt.Errorf("token response could not be read: %w", err)) } if resp.StatusCode < 200 || resp.StatusCode >= 300 { - // The status, not the body: what an issuer puts in a failure body is its - // own business, and it routinely quotes the request back. + // The status, not the body: a failure body routinely quotes the + // request back. log.Warnf(ctx, "token endpoint %s returned %s: %s", cfg.TokenURL, resp.Status, s.redactString(explain(body))) return "", 0, s.tokenErr(fmt.Errorf("token endpoint returned %s", resp.Status)) @@ -126,36 +121,30 @@ func (s *Step) exchangeToken(ctx context.Context, auth *authenticator) (string, return parsed.AccessToken, lifetime, nil } -// tokenErr classifies a failed exchange. Always 502: the exchange is with the -// provider's issuer, so its failure is upstream's, never the caller's. +// tokenErr classifies a failed exchange. Always 502: the failure is the +// issuer's, never the caller's. func (s *Step) tokenErr(err error) error { return model.NewCodedErr(http.StatusBadGateway, codeUpstreamUnavailable, fmt.Errorf("oauth2 token exchange failed: %w", err)) } -// tokenLifetime turns a token response's expires_in into how long we may hold -// that token, or reports that we may not hold it at all. +// tokenLifetime turns expires_in into how long the token may be held, or +// reports that it may not be held at all. // -// The issuer owns this number, so it is read from the response and never from -// our config: a configured copy is a second version of the same fact, and it -// is wrong the moment a realm's token lifetime is retuned -- silently, with -// every request 401ing until someone edits a file. +// Read from the response, never from config: the issuer owns this number, and a +// configured copy is wrong the moment a realm is retuned -- silently, with +// every request 401ing. // -// skew is subtracted so a request that passes the check cannot arrive at the -// provider after expiry. It has to exceed the round trip, and costs one extra -// refresh per token lifetime. -// -// Two answers rather than one duration, because "we were not told" is not a -// lifetime. A response with no expires_in is refused rather than cached for a -// guessed period or re-fetched on every single request. +// skew is subtracted so a request that passed the check cannot arrive after +// expiry. The false return says "we were not told", which is not a lifetime: +// no expires_in is refused rather than cached for a guessed period. func tokenLifetime(expiresIn int, skew time.Duration) (time.Duration, bool) { if expiresIn <= 0 { return 0, false } lifetime := time.Duration(expiresIn) * time.Second - // A lifetime at or under the skew would put the expiry in the past, and - // then every request fetches a fresh token. Half is still early enough to - // refresh before the real expiry. + // A lifetime at or under the skew would put the expiry in the past, making + // every request fetch a fresh token. Half is still early enough. if lifetime <= skew { return lifetime / 2, true } From fabeacfcc354622b0e5025a07f5a428fb79e0e63 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 22:12:06 +0530 Subject: [PATCH 116/130] chore: remove AgricultureFacility testdata I committed by mistake [#7] 16 files, 204K of cached schema documents under pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/. They belong to PR #22, which is somebody else's work and not merged. They were sitting untracked in the checkout when I ran `git add -A` for the internal/ rename in 53b4bd7, and that swept them in. Not present on feat/7-knowledge-advisory or on development, so this branch introduced them. Nothing references them: the only matches for the name are inside the files themselves. 65 packages green after removing them. The mistake was `git add -A` on a path I did not fully own. Every commit since has staged named paths instead. --- .../AgricultureFacility/v0.1/attributes.yaml | 149 ------------- .../v0.1/examples/custom-hiring-centre.json | 30 --- .../v0.1/examples/krishi-vigyan-kendra.json | 34 --- .../on-demand-facility-discovery.json | 23 -- .../v0.1/examples/soil-testing-facility.json | 28 --- .../v0.1/examples/warehouse.json | 40 ---- .../AgricultureResource/v0.1/attributes.yaml | 197 ------------------ .../Address/v2.0/attributes.yaml | 45 ---- .../Contact/v2.0/attributes.yaml | 36 ---- .../Descriptor/v2.1/attributes.yaml | 52 ----- .../GeoJSONGeometry/v2.0/attributes.yaml | 57 ----- .../Location/v2.0/attributes.yaml | 26 --- .../Address/v2.0/attributes.yaml | 45 ---- .../Document/v2.0/attributes.yaml | 56 ----- .../GeoJSONGeometry/v2.0/attributes.yaml | 57 ----- .../MediaFile/v2.0/attributes.yaml | 32 --- 16 files changed, 907 deletions(-) delete mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/attributes.yaml delete mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/custom-hiring-centre.json delete mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/krishi-vigyan-kendra.json delete mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/on-demand-facility-discovery.json delete mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/soil-testing-facility.json delete mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/warehouse.json delete mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureResource/v0.1/attributes.yaml delete mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Address/v2.0/attributes.yaml delete mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Contact/v2.0/attributes.yaml delete mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Descriptor/v2.1/attributes.yaml delete mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/GeoJSONGeometry/v2.0/attributes.yaml delete mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Location/v2.0/attributes.yaml delete mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/Address/v2.0/attributes.yaml delete mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/Document/v2.0/attributes.yaml delete mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/GeoJSONGeometry/v2.0/attributes.yaml delete mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/MediaFile/v2.0/attributes.yaml diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/attributes.yaml deleted file mode 100644 index d4b063a8..00000000 --- a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/attributes.yaml +++ /dev/null @@ -1,149 +0,0 @@ -openapi: 3.1.1 -info: - title: OpenAgriNet - Agriculture Facility Attributes - version: 0.1.0 - description: > - Agricultural facilities that may be discovered through a Provider or - supplied directly. Combines with AgricultureResource through allOf and - attaches through resourceAttributes. - -components: - schemas: - AgricultureFacility: - type: object - x-beckn-container: resourceAttributes - x-jsonld: - "@context": "https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld" - "@type": openagrinet:AgricultureFacility - allOf: - - $ref: "../../AgricultureResource/v0.1/attributes.yaml#/components/schemas/AgricultureResource" - - type: object - required: ["@type"] - properties: - "@type": - oneOf: - - type: string - const: openagrinet:AgricultureFacility - - type: array - minItems: 2 - uniqueItems: true - contains: - const: openagrinet:AgricultureFacility - items: - oneOf: - - const: openagrinet:AgricultureFacility - - type: string - minLength: 1 - not: - pattern: "^openagrinet:" - description: Canonical OAN type, optionally accompanied by Provider-defined JSON-LD types. - supportedFacilityTypes: - type: array - minItems: 1 - uniqueItems: true - description: Facility types the Provider can return on demand. - items: - $ref: "#/components/schemas/FacilityType" - x-jsonld-id: "openagrinet:supportedFacilityTypes" - facilityType: - allOf: - - $ref: "#/components/schemas/FacilityType" - description: Governed type of the facility represented by a Direct Resource. - x-jsonld-id: "openagrinet:facilityType" - location: - allOf: - - $ref: "../../AgricultureResource/v0.1/attributes.yaml#/components/schemas/CompleteLocation" - description: Verified facility geometry and optional address. Do not populate it with the search origin or another inferred point. - x-jsonld-id: "openagrinet:location" - address: - allOf: - - $ref: "../../AgricultureResource/v0.1/attributes.yaml#/components/schemas/NonEmptyAddress" - description: Human-readable facility address when verified geometry is unavailable. - x-jsonld-id: "openagrinet:address" - services: - type: array - minItems: 1 - uniqueItems: true - description: Services publicly offered at the facility. - items: - $ref: "../../AgricultureResource/v0.1/attributes.yaml#/components/schemas/IdentifiedDescriptor" - x-jsonld-id: "openagrinet:services" - capacity: - allOf: - - $ref: "#/components/schemas/FacilityCapacity" - description: Published operating or storage capacity when applicable. - x-jsonld-id: "openagrinet:capacity" - publicContact: - allOf: - - $ref: "https://schema.beckn.io/Contact/v2.0/attributes.yaml#/components/schemas/Contact" - description: Public organizational contact explicitly approved by the Provider for catalog publication. Do not use for private personal contact data. - x-jsonld-id: "openagrinet:publicContact" - website: - type: string - format: uri - description: Public facility or service website. - x-jsonld-id: "openagrinet:website" - source: - allOf: - - $ref: "../../AgricultureResource/v0.1/attributes.yaml#/components/schemas/SourceReference" - description: Provider or authoritative source of the facility record. - x-jsonld-id: "openagrinet:source" - lastUpdatedAt: - type: string - format: date-time - description: Time at which the Provider last updated or verified the facility record. - x-jsonld-id: "openagrinet:lastUpdatedAt" - - if: - properties: - informationMode: - const: OnDemand - required: [informationMode] - then: - required: [supportedFacilityTypes] - not: - anyOf: - - required: [facilityType] - - required: [location] - - required: [address] - - required: [services] - - required: [capacity] - - required: [publicContact] - - required: [website] - - required: [source] - - required: [lastUpdatedAt] - - if: - properties: - informationMode: - const: Direct - required: [informationMode] - then: - required: [facilityType, source] - anyOf: - - required: [location] - - required: [address] - - FacilityType: - type: string - description: Governed agriculture facility category. SoilTestingFacility covers a Soil Health Centre or Soil Testing Laboratory that provides soil-testing services. - enum: [CustomHiringCentre, KrishiVigyanKendra, Warehouse, SoilTestingFacility] - - FacilityCapacity: - type: object - required: [value, unit] - properties: - value: - type: number - minimum: 0 - description: Numeric capacity value. - x-jsonld-id: "openagrinet:value" - unit: - type: string - minLength: 1 - description: Governed or source unit, such as metric tonne. - x-jsonld-id: "openagrinet:unit" - basis: - type: string - minLength: 1 - description: Capacity basis when required, such as total storage or daily throughput. - x-jsonld-id: "openagrinet:capacityBasis" - additionalProperties: false diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/custom-hiring-centre.json b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/custom-hiring-centre.json deleted file mode 100644 index e97b9fcf..00000000 --- a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/custom-hiring-centre.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "@context": "https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld", - "@type": "openagrinet:AgricultureFacility", - "informationMode": "Direct", - "languages": [ - "en", - "mr" - ], - "facilityType": "CustomHiringCentre", - "address": { - "streetAddress": "Krushisafalya Farmer Producer Company Limited", - "addressLocality": "Gogalgaon", - "addressRegion": "Maharashtra", - "postalCode": "423107", - "addressCountry": "IN", - "extendedAddress": "Rahta, Ahmednagar district" - }, - "services": [ - { - "code": "FARM_MACHINERY_HIRE", - "name": "Farm machinery hire", - "shortDesc": "Access to agricultural machinery and implements." - } - ], - "source": { - "sourceId": "provider:agriculture-common-services", - "sourceName": "Agriculture Common Services Provider" - }, - "lastUpdatedAt": "2026-08-29T00:00:00Z" -} diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/krishi-vigyan-kendra.json b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/krishi-vigyan-kendra.json deleted file mode 100644 index 234646e9..00000000 --- a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/krishi-vigyan-kendra.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "@context": "https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld", - "@type": "openagrinet:AgricultureFacility", - "informationMode": "Direct", - "languages": [ - "en", - "hi", - "mr" - ], - "facilityType": "KrishiVigyanKendra", - "address": { - "streetAddress": "Krishi Vigyan Kendra, PO Babhleshwar", - "addressLocality": "Babhleshwar", - "addressRegion": "Maharashtra", - "postalCode": "413737", - "addressCountry": "IN", - "extendedAddress": "Rahata, Ahmednagar district" - }, - "services": [ - { - "code": "FARM_ADVISORY", - "name": "Farm advisory" - }, - { - "code": "FARMER_TRAINING", - "name": "Farmer training" - } - ], - "source": { - "sourceId": "provider:agriculture-common-services", - "sourceName": "Agriculture Common Services Provider" - }, - "lastUpdatedAt": "2026-08-29T00:00:00Z" -} diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/on-demand-facility-discovery.json b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/on-demand-facility-discovery.json deleted file mode 100644 index fb57f3ee..00000000 --- a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/on-demand-facility-discovery.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "@context": "https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld", - "@type": "openagrinet:AgricultureFacility", - "informationMode": "OnDemand", - "languages": [ - "en", - "mr" - ], - "supportedFacilityTypes": [ - "CustomHiringCentre", - "KrishiVigyanKendra", - "Warehouse", - "SoilTestingFacility" - ], - "coverageAreas": [ - { - "codeScheme": "ISO-3166-2", - "areaCode": "IN-MH", - "areaLevel": "State", - "areaName": "Maharashtra" - } - ] -} diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/soil-testing-facility.json b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/soil-testing-facility.json deleted file mode 100644 index 3ff39abb..00000000 --- a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/soil-testing-facility.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "@context": "https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld", - "@type": "openagrinet:AgricultureFacility", - "informationMode": "Direct", - "languages": [ - "en", - "mr" - ], - "facilityType": "SoilTestingFacility", - "address": { - "streetAddress": "District Soil Survey and Soil Testing Laboratory, Pumping Station Road, Bhutkarwadi", - "addressLocality": "Ahilyanagar", - "addressRegion": "Maharashtra", - "addressCountry": "IN" - }, - "services": [ - { - "code": "SOIL_TESTING", - "name": "Soil sample testing", - "shortDesc": "Laboratory testing of soil samples for agricultural use." - } - ], - "source": { - "sourceId": "provider:agriculture-common-services", - "sourceName": "Agriculture Common Services Provider" - }, - "lastUpdatedAt": "2026-08-29T00:00:00Z" -} diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/warehouse.json b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/warehouse.json deleted file mode 100644 index 47d1f76f..00000000 --- a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureFacility/v0.1/examples/warehouse.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "@context": "https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld", - "@type": "openagrinet:AgricultureFacility", - "informationMode": "Direct", - "languages": [ - "en", - "mr" - ], - "facilityType": "Warehouse", - "address": { - "streetAddress": "Survey No. 142/2, Musalgaon MIDC Area", - "addressLocality": "Musalgaon", - "addressRegion": "Maharashtra", - "postalCode": "422112", - "addressCountry": "IN", - "extendedAddress": "Sinnar, Nashik district" - }, - "services": [ - { - "code": "GENERAL_STORAGE", - "name": "General agricultural storage" - } - ], - "capacity": { - "value": 500, - "unit": "metric tonne", - "basis": "total storage" - }, - "publicContact": { - "company": "Example Warehouse Services", - "phone": "+910000000000", - "email": "warehouse@example.org" - }, - "website": "https://warehouse.example.org/facilities/musalgaon", - "source": { - "sourceId": "provider:warehouse-services", - "sourceName": "Warehouse Services Provider" - }, - "lastUpdatedAt": "2026-08-29T00:00:00Z" -} diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureResource/v0.1/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureResource/v0.1/attributes.yaml deleted file mode 100644 index 3fd3984d..00000000 --- a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/raw.githubusercontent.com/OpenAgriNet/network-specs/a39d2f372349401df5e4f5efd9e8b6992ae60d46/schema/AgricultureResource/v0.1/attributes.yaml +++ /dev/null @@ -1,197 +0,0 @@ -openapi: 3.1.1 -info: - title: OpenAgriNet - Agriculture Resource Attributes - version: 0.1.0 - description: > - Shared agriculture attributes for information published on OpenAgriNet. - Attaches to beckn:Resource through resourceAttributes. - -components: - schemas: - AgricultureResource: - type: object - x-beckn-container: resourceAttributes - x-jsonld: - "@context": "https://openagrinet.github.io/network-specs/schema/AgricultureResource/v0.1/context.jsonld" - "@type": openagrinet:AgricultureResource - required: [informationMode] - properties: - informationMode: - type: string - enum: [OnDemand, Direct] - description: Whether obtaining specific information requires a Provider invocation or the Resource already contains or directly references that information. - x-jsonld-id: "openagrinet:informationMode" - subjectCategories: - type: array - minItems: 1 - uniqueItems: true - description: Optional broad agricultural subjects used for discovery. The Resource type is expressed by @type, not repeated here. - items: - type: string - enum: [Crop, Livestock, Weather, Market, Scheme, Practice] - x-jsonld-id: "openagrinet:subjectCategories" - agricultureSubjects: - type: array - minItems: 1 - uniqueItems: true - description: Optional governed subjects such as a named crop, livestock species, pest, or scheme. Absence means the Resource applies broadly within subjectCategories. - items: - $ref: "#/components/schemas/AgricultureSubjectReference" - x-jsonld-id: "openagrinet:agricultureSubjects" - languages: - type: array - minItems: 1 - uniqueItems: true - description: BCP 47 language tags supported by or present in the Resource. - items: - type: string - minLength: 2 - x-jsonld-id: "openagrinet:languages" - coverageAreas: - type: array - minItems: 1 - uniqueItems: true - description: Geographic areas within which the Resource applies. Each item is either an administrative area reference or a GeoJSON geometry. - items: - oneOf: - - $ref: "#/components/schemas/AdministrativeAreaReference" - - $ref: "#/components/schemas/CompleteGeoJSONGeometry" - x-jsonld-id: "openagrinet:coverageAreas" - - AdministrativeAreaReference: - type: object - description: A governed administrative or postal area identified within a stated code system. Use a GeoJSON geometry instead when coverage is defined by coordinates. - required: [codeScheme, areaCode] - properties: - codeScheme: - type: string - minLength: 1 - description: Geographic code system, such as ISO-3166-1, ISO-3166-2, LGD, or IN-PIN. - x-jsonld-id: "openagrinet:areaCodeScheme" - areaCode: - type: string - minLength: 1 - description: Area identifier in the stated scheme. - x-jsonld-id: "openagrinet:areaCode" - areaLevel: - type: string - enum: [Country, State, District, Block, Village, PostalCode, Other] - description: Administrative or postal level represented by the code when known. - x-jsonld-id: "openagrinet:areaLevel" - areaName: - type: string - minLength: 1 - description: Optional human-readable area name. - x-jsonld-id: "openagrinet:areaName" - additionalProperties: false - - AgricultureSubjectReference: - type: object - description: Stable reference to a governed agriculture subject and its human-readable Beckn descriptor. - required: [subjectId, subjectType, descriptor] - properties: - subjectId: - type: string - format: uri - description: Stable identifier from a governed agriculture taxonomy. - x-jsonld-id: "openagrinet:subjectId" - subjectType: - type: string - enum: [Crop, Livestock, Commodity, Pest, Disease, Practice, Scheme, Market] - description: Kind of agricultural subject referenced by subjectId. - x-jsonld-id: "openagrinet:subjectType" - descriptor: - $ref: "#/components/schemas/IdentifiedDescriptor" - description: Beckn descriptor carrying the governed code and display name. - x-jsonld-id: "https://schema.beckn.io/descriptor" - additionalProperties: false - - SourceReference: - type: object - description: Reference to the authoritative upstream source of information. - required: [sourceId] - properties: - sourceId: - type: string - minLength: 1 - description: Stable identifier of the upstream source. - x-jsonld-id: "openagrinet:sourceId" - sourceName: - type: string - minLength: 1 - description: Human-readable source name. - x-jsonld-id: "openagrinet:sourceName" - sourceUri: - type: string - format: uri - description: Authoritative public or Provider-supplied source URI. - x-jsonld-id: "openagrinet:sourceUri" - additionalProperties: false - - TimePeriod: - type: object - description: Inclusive validity or applicability period. - minProperties: 1 - properties: - startsAt: - type: string - format: date-time - description: Start of the period. - x-jsonld-id: "openagrinet:startsAt" - endsAt: - type: string - format: date-time - description: End of the period. - x-jsonld-id: "openagrinet:endsAt" - additionalProperties: false - - ClosedTimePeriod: - allOf: - - $ref: "#/components/schemas/TimePeriod" - - type: object - required: [startsAt, endsAt] - description: A bounded applicability period with both start and end timestamps. Conformance also verifies that startsAt is not after endsAt. - - IdentifiedDescriptor: - allOf: - - $ref: "https://schema.beckn.io/Descriptor/v2.1/attributes.yaml#/components/schemas/Descriptor" - - type: object - anyOf: - - required: [code] - - required: [name] - description: A Beckn Descriptor with at least a machine-readable code or human-readable name. - - CompleteGeoJSONGeometry: - allOf: - - $ref: "https://schema.beckn.io/GeoJSONGeometry/v2.0/attributes.yaml#/components/schemas/GeoJSONGeometry" - - oneOf: - - type: object - properties: - type: - const: GeometryCollection - required: [type, geometries] - - type: object - properties: - type: - enum: [Point, LineString, Polygon, MultiPoint, MultiLineString, MultiPolygon] - required: [type, coordinates] - description: A GeoJSON geometry that includes coordinates, or geometries for a GeometryCollection. - - NonEmptyAddress: - allOf: - - $ref: "https://schema.beckn.io/Address/v2.0/attributes.yaml#/components/schemas/Address" - - type: object - minProperties: 1 - description: A Beckn postal address containing at least one address field. - - CompleteLocation: - allOf: - - $ref: "https://schema.beckn.io/Location/v2.0/attributes.yaml#/components/schemas/Location" - - type: object - required: [geo] - properties: - geo: - $ref: "#/components/schemas/CompleteGeoJSONGeometry" - address: - $ref: "#/components/schemas/NonEmptyAddress" - description: A Beckn Location with a complete GeoJSON geometry and an optional non-empty address. diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Address/v2.0/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Address/v2.0/attributes.yaml deleted file mode 100644 index e6517b90..00000000 --- a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Address/v2.0/attributes.yaml +++ /dev/null @@ -1,45 +0,0 @@ -openapi: 3.1.1 -info: - title: Address - version: 2.0.0 - description: "Container schemas fetched from beckn.yaml. This cannot be extended as it is a reserved schema in beckn protocol. Any additional properties added to this schema can only be made using its *Attributes property" - license: - name: CC-BY-NC-SA 4.0 International - contact: - name: Beckn Labs - url: https://beckn.io -components: - schemas: - Address: - x-iri: https://schema.nfh.global/Address/v2.0 - description: '**Postal address** aligned with schema.org `PostalAddress`. Use for human-readable addresses. Geometry lives in `Location.geo` as GeoJSON.' - title: Address - type: object - properties: - addressCountry: - description: Country name or ISO-3166-1 alpha-2 code. - type: string - example: IN - addressLocality: - description: City/locality. - type: string - example: Bengaluru - addressRegion: - description: State/region/province. - type: string - example: Karnataka - extendedAddress: - description: Address extension (apt/suite/floor, C/O). - type: string - example: Apt 4B - postalCode: - description: Postal/ZIP code. - type: string - example: '560001' - streetAddress: - description: Street address (building name/number and street). - type: string - example: 123 Tech Street - additionalProperties: false - x-tags: - - common diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Contact/v2.0/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Contact/v2.0/attributes.yaml deleted file mode 100644 index f2564773..00000000 --- a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Contact/v2.0/attributes.yaml +++ /dev/null @@ -1,36 +0,0 @@ -openapi: 3.1.1 -info: - title: Contact - version: 2.0.0 - description: Schema for Contact in Beckn Logistics domain. - license: - name: CC-BY-NC-SA 4.0 International - contact: - name: Beckn Labs - url: https://beckn.io -components: - schemas: - Contact: - x-iri: https://schema.nfh.global/Contact/v2.0 - type: object - title: Contact - description: Contact information for sender, receiver, driver, or operator. - properties: - name: - type: string - example: Priya Mehta - phone: - type: string - example: '+919876543210' - email: - type: string - format: email - example: priya.mehta@email.com - alternatePhone: - type: string - company: - type: string - description: Company name if B2B contact - gstNumber: - type: string - description: GST number for B2B invoicing diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Descriptor/v2.1/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Descriptor/v2.1/attributes.yaml deleted file mode 100644 index 0d53150d..00000000 --- a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Descriptor/v2.1/attributes.yaml +++ /dev/null @@ -1,52 +0,0 @@ -openapi: 3.1.1 -info: - title: Descriptor - version: 2.1.0 - description: "Container schemas fetched from beckn.yaml. This cannot be extended as it is a reserved schema in beckn protocol. Any additional properties added to this schema can only be made using its *Attributes property" - license: - name: CC-BY-NC-SA 4.0 International - contact: - name: Beckn Labs - url: https://beckn.io -components: - schemas: - Descriptor: - x-iri: https://schema.nfh.global/Descriptor/v2.1 - description: Schema definition for Descriptor in the Beckn Protocol v2.0.1 - title: Descriptor - type: object - properties: - code: - description: A machine-readable code identifying the state or type of the - entity being described. The valid values for this field are defined by - the context in which the Descriptor is used (e.g. DRAFT, ACTIVE, CANCELLED, - COMPLETE for a Contract status). - type: string - longDesc: - description: Detailed description of the item - type: string - example: Powerful gaming laptop with NVIDIA RTX graphics, fast SSD storage, - and high-refresh display - shortDesc: - description: Short description of the item - type: string - example: High-performance gaming laptop with RTX graphics - name: - description: Name of the entity being described - type: string - example: Premium Gaming Laptop Pro - thumbnailImage: - description: Name of the entity being described - type: string - format: uri - docs: - description: Links to downloadable business/supporting documents - type: array - items: - $ref: https://schema.nfh.global/Document/v2.0/attributes.yaml#/components/schemas/Document - mediaFile: - description: Links to multimedia files and images - type: array - items: - $ref: https://schema.nfh.global/MediaFile/v2.0/attributes.yaml#/components/schemas/MediaFile - additionalProperties: false diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/GeoJSONGeometry/v2.0/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/GeoJSONGeometry/v2.0/attributes.yaml deleted file mode 100644 index d5045971..00000000 --- a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/GeoJSONGeometry/v2.0/attributes.yaml +++ /dev/null @@ -1,57 +0,0 @@ -openapi: 3.1.1 -info: - title: GeoJSONGeometry - version: 2.0.0 - description: "Container schemas fetched from beckn.yaml. This cannot be extended as it is a reserved schema in beckn protocol. Any additional properties added to this schema can only be made using its *Attributes property" - license: - name: CC-BY-NC-SA 4.0 International - contact: - name: Beckn Labs - url: https://beckn.io -components: - schemas: - GeoJSONGeometry: - x-iri: https://schema.nfh.global/GeoJSONGeometry/v2.0 - description: "**GeoJSON geometry** per RFC 7946. Coordinates are in **EPSG:4326\ - \ (WGS-84)** and MUST follow **[longitude, latitude, (altitude?)]** order. \ - Supported types: - Point, LineString, Polygon - MultiPoint, MultiLineString,\ - \ MultiPolygon - GeometryCollection (uses `geometries` instead of `coordinates`) \ - Notes: - For rectangles, use a Polygon with a single linear ring where the\ - \ first and last positions are identical. - Circles are **not native**\ - \ to GeoJSON. For circular searches, use `SpatialConstraint` with `op: s_dwithin`\ - \ and a Point + `distanceMeters`, or approximate the circle as a Polygon. \ - - Optional `bbox` is `[west, south, east, north]` in degrees." - title: GeoJSONGeometry - type: object - properties: - bbox: - description: Optional bounding box `[west, south, east, north]` in degrees. - type: array - minItems: 4 - maxItems: 4 - items: - type: number - coordinates: - description: Coordinates per RFC 7946 for all types **except** GeometryCollection. - Order is **[lon, lat, (alt)]**. For Polygons, this is an array of linear - rings; each ring is an array of positions. - type: array - items: {} - geometries: - description: Member geometries when `type` is **GeometryCollection**. - type: array - items: - $ref: https://schema.nfh.global/GeoJSONGeometry/v2.0/attributes.yaml#/components/schemas/GeoJSONGeometry - type: - type: string - enum: - - Point - - LineString - - Polygon - - MultiPoint - - MultiLineString - - MultiPolygon - - GeometryCollection - required: - - type - additionalProperties: true diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Location/v2.0/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Location/v2.0/attributes.yaml deleted file mode 100644 index c7636123..00000000 --- a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.beckn.io/Location/v2.0/attributes.yaml +++ /dev/null @@ -1,26 +0,0 @@ -openapi: 3.1.1 -info: - title: Location - version: 2.0.0 - description: "Container schemas fetched from beckn.yaml. This cannot be extended as it is a reserved schema in beckn protocol. Any additional properties added to this schema can only be made using its *Attributes property" - license: - name: CC-BY-NC-SA 4.0 International - contact: - name: Beckn Labs - url: https://beckn.io -components: - schemas: - Location: - x-iri: https://schema.nfh.global/Location/v2.0 - description: 'A place represented by GeoJSON geometry and optional address. - - Source: main/schema/core/v2/attributes.yaml#Location' - type: object - required: - - geo - additionalProperties: false - properties: - geo: - $ref: https://schema.nfh.global/GeoJSONGeometry/v2.0/attributes.yaml#/components/schemas/GeoJSONGeometry - address: - $ref: https://schema.nfh.global/Address/v2.0/attributes.yaml#/components/schemas/Address diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/Address/v2.0/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/Address/v2.0/attributes.yaml deleted file mode 100644 index e6517b90..00000000 --- a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/Address/v2.0/attributes.yaml +++ /dev/null @@ -1,45 +0,0 @@ -openapi: 3.1.1 -info: - title: Address - version: 2.0.0 - description: "Container schemas fetched from beckn.yaml. This cannot be extended as it is a reserved schema in beckn protocol. Any additional properties added to this schema can only be made using its *Attributes property" - license: - name: CC-BY-NC-SA 4.0 International - contact: - name: Beckn Labs - url: https://beckn.io -components: - schemas: - Address: - x-iri: https://schema.nfh.global/Address/v2.0 - description: '**Postal address** aligned with schema.org `PostalAddress`. Use for human-readable addresses. Geometry lives in `Location.geo` as GeoJSON.' - title: Address - type: object - properties: - addressCountry: - description: Country name or ISO-3166-1 alpha-2 code. - type: string - example: IN - addressLocality: - description: City/locality. - type: string - example: Bengaluru - addressRegion: - description: State/region/province. - type: string - example: Karnataka - extendedAddress: - description: Address extension (apt/suite/floor, C/O). - type: string - example: Apt 4B - postalCode: - description: Postal/ZIP code. - type: string - example: '560001' - streetAddress: - description: Street address (building name/number and street). - type: string - example: 123 Tech Street - additionalProperties: false - x-tags: - - common diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/Document/v2.0/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/Document/v2.0/attributes.yaml deleted file mode 100644 index ef0f1e88..00000000 --- a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/Document/v2.0/attributes.yaml +++ /dev/null @@ -1,56 +0,0 @@ -openapi: 3.1.1 -info: - title: Document - version: 2.0.0 - description: "Container schemas fetched from beckn.yaml. This cannot be extended as it is a reserved schema in beckn protocol. Any additional properties added to this schema can only be made using its *Attributes property" - license: - name: CC-BY-NC-SA 4.0 International - contact: - name: Beckn Labs - url: https://beckn.io -components: - schemas: - Document: - x-iri: https://schema.nfh.global/Document/v2.0 - description: A document, that can be parsed, printed, download or displayed. This has intentionally been kept separate from MediaFile as they may contain additional attributes like signature, schema etc. - title: Document - x-tags: - - common - type: object - properties: - label: - description: The display name of the media file - type: string - mimeType: - description: MIME type if 'data' is provided (application/pdf, image/png, application/json). - type: string - standard: - description: Describes the schema type in case the document follows a standard schema like a verifiableCredential - type: string - url: - description: URL to the document - type: string - format: uri - security: - type: object - properties: - checksum: - type: string - checksumAlgorithm: - type: string - enum: - - SHA256 - - SHA512 - - OTHER - signature: - type: string - signatureAlgorithm: - type: string - description: e.g., Ed25519, ES256 - keyId: - type: string - required: - - label - - url - - mimeType - additionalProperties: false diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/GeoJSONGeometry/v2.0/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/GeoJSONGeometry/v2.0/attributes.yaml deleted file mode 100644 index d5045971..00000000 --- a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/GeoJSONGeometry/v2.0/attributes.yaml +++ /dev/null @@ -1,57 +0,0 @@ -openapi: 3.1.1 -info: - title: GeoJSONGeometry - version: 2.0.0 - description: "Container schemas fetched from beckn.yaml. This cannot be extended as it is a reserved schema in beckn protocol. Any additional properties added to this schema can only be made using its *Attributes property" - license: - name: CC-BY-NC-SA 4.0 International - contact: - name: Beckn Labs - url: https://beckn.io -components: - schemas: - GeoJSONGeometry: - x-iri: https://schema.nfh.global/GeoJSONGeometry/v2.0 - description: "**GeoJSON geometry** per RFC 7946. Coordinates are in **EPSG:4326\ - \ (WGS-84)** and MUST follow **[longitude, latitude, (altitude?)]** order. \ - Supported types: - Point, LineString, Polygon - MultiPoint, MultiLineString,\ - \ MultiPolygon - GeometryCollection (uses `geometries` instead of `coordinates`) \ - Notes: - For rectangles, use a Polygon with a single linear ring where the\ - \ first and last positions are identical. - Circles are **not native**\ - \ to GeoJSON. For circular searches, use `SpatialConstraint` with `op: s_dwithin`\ - \ and a Point + `distanceMeters`, or approximate the circle as a Polygon. \ - - Optional `bbox` is `[west, south, east, north]` in degrees." - title: GeoJSONGeometry - type: object - properties: - bbox: - description: Optional bounding box `[west, south, east, north]` in degrees. - type: array - minItems: 4 - maxItems: 4 - items: - type: number - coordinates: - description: Coordinates per RFC 7946 for all types **except** GeometryCollection. - Order is **[lon, lat, (alt)]**. For Polygons, this is an array of linear - rings; each ring is an array of positions. - type: array - items: {} - geometries: - description: Member geometries when `type` is **GeometryCollection**. - type: array - items: - $ref: https://schema.nfh.global/GeoJSONGeometry/v2.0/attributes.yaml#/components/schemas/GeoJSONGeometry - type: - type: string - enum: - - Point - - LineString - - Polygon - - MultiPoint - - MultiLineString - - MultiPolygon - - GeometryCollection - required: - - type - additionalProperties: true diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/MediaFile/v2.0/attributes.yaml b/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/MediaFile/v2.0/attributes.yaml deleted file mode 100644 index 7906ca96..00000000 --- a/pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache/schema.nfh.global/MediaFile/v2.0/attributes.yaml +++ /dev/null @@ -1,32 +0,0 @@ -openapi: 3.1.1 -info: - title: MediaFile - version: 2.0.0 - description: "Container schemas fetched from beckn.yaml. This cannot be extended as it is a reserved schema in beckn protocol. Any additional properties added to this schema can only be made using its *Attributes property" - license: - name: CC-BY-NC-SA 4.0 International - contact: - name: Beckn Labs - url: https://beckn.io -components: - schemas: - MediaFile: - x-iri: https://schema.nfh.global/MediaFile/v2.0 - description: A image, audio, or video typically intended for display purposes - title: MediaFile - x-tags: - - common - type: object - properties: - label: - description: The display name of the media file - type: string - mimeType: - description: MIME type if 'data' is provided (application/pdf, image/png). - type: string - example: image/png - uri: - description: URL to the document - type: string - format: uri - additionalProperties: false From a07ecd701c7c27110545362662d4364306a7f309 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 22:21:58 +0530 Subject: [PATCH 117/130] refactor(common): move the HTTP helpers into a httputil subpackage [#7] http.go was 408 lines: two Step methods and fourteen helpers that need nothing from Step. The helpers move to common/httputil, grouped by subject: httputil/retry.go 88 Budget, Backoff, Sleep, DoNotRetry, IsPermanent httputil/endpoint.go 107 BuildEndpoint, verifyPath, verifyBaseURL httputil/query.go 58 asQuery, asQueryValue httputil/request.go 49 RequestBody, HasBody, CanonicalMethod httputil/explain.go 22 Explain httputil/constant.go 40 the retry, timeout and explain constants http.go is now 125 lines holding call and attempt, which stay because they are Step methods -- a method must be in the same package as its receiver. THE COST, measured rather than estimated. Ten helpers become exported, because call and attempt still use them from common. Only four stay private -- verifyPath, verifyBaseURL, asQuery, asQueryValue -- reached solely through BuildEndpoint, which moved with them. CONSTANT.GO IS NOW SPLIT, which contradicts the one-constants-file decision. Seven constants had to move: httputil reads them and common imports httputil, so a shared file would be an import cycle. common/constant.go keeps the nine about auth schemes, redaction and the response cap; httputil/constant.go takes the seven about the call. Both files say why. Flagging it rather than burying it -- say the word and this commit is one revert. Thirteen tests moved to httputil/httputil_test.go as package httputil, not _test, so the four private helpers are still tested directly. common_test.go drops from 2,506 to 2,211 lines. The moved logic is unchanged: comparing code lines only, before and after normalising the rename, 251 against 252 -- the one difference being a single-line `import "strings"` the filter did not recognise. 66 packages green, vet clean, no races. --- .../internal/common/common_test.go | 302 +---------------- .../internal/common/constant.go | 35 +- .../implementation/internal/common/http.go | 311 +----------------- .../internal/common/httputil/constant.go | 40 +++ .../internal/common/httputil/endpoint.go | 107 ++++++ .../internal/common/httputil/explain.go | 22 ++ .../internal/common/httputil/httputil_test.go | 305 +++++++++++++++++ .../internal/common/httputil/query.go | 58 ++++ .../internal/common/httputil/request.go | 49 +++ .../internal/common/httputil/retry.go | 88 +++++ .../implementation/internal/common/token.go | 3 +- 11 files changed, 692 insertions(+), 628 deletions(-) create mode 100644 pkg/plugin/implementation/internal/common/httputil/constant.go create mode 100644 pkg/plugin/implementation/internal/common/httputil/endpoint.go create mode 100644 pkg/plugin/implementation/internal/common/httputil/explain.go create mode 100644 pkg/plugin/implementation/internal/common/httputil/httputil_test.go create mode 100644 pkg/plugin/implementation/internal/common/httputil/query.go create mode 100644 pkg/plugin/implementation/internal/common/httputil/request.go create mode 100644 pkg/plugin/implementation/internal/common/httputil/retry.go diff --git a/pkg/plugin/implementation/internal/common/common_test.go b/pkg/plugin/implementation/internal/common/common_test.go index 5ea5b987..2caee927 100644 --- a/pkg/plugin/implementation/internal/common/common_test.go +++ b/pkg/plugin/implementation/internal/common/common_test.go @@ -17,6 +17,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" ) const selectBody = `{ @@ -505,38 +506,6 @@ func TestRunKeepsTheProvidersBodyOffTheWire(t *testing.T) { } } -// explain still collapses whitespace, because the body it prepares now goes to -// a log line rather than an error -- an indented body would spread one failure -// over several lines either way. -func TestExplainCollapsesWhitespace(t *testing.T) { - t.Parallel() - - got := explain([]byte("{\n \"message\": \"no data available\"\n}")) - if strings.Contains(got, "\n") { - t.Errorf("explain(%q) left a newline in", got) - } - if !strings.Contains(got, "no data available") { - t.Errorf("explain = %q, want the provider's message preserved", got) - } -} - -// A body is quoted, not dumped: a provider answering with a page of HTML must -// not put all of it in a log line or a NACK. -func TestExplainTruncatesAndHandlesAnEmptyBody(t *testing.T) { - t.Parallel() - - if got := explain(nil); got != "(no body)" { - t.Errorf("explain(nil) = %q, want a marker rather than an empty string", got) - } - long := explain([]byte(strings.Repeat("x", explainLimit+50))) - if len(long) > explainLimit+len("... (truncated)") { - t.Errorf("explain kept %d characters, want it truncated near %d", len(long), explainLimit) - } - if !strings.HasSuffix(long, "(truncated)") { - t.Errorf("a truncated body should say so, got %q", long[len(long)-20:]) - } -} - // The quoted body goes through the same redaction as everything else, or a // provider that echoes the query string back would defeat it. func TestRunRedactsACredentialEchoedInABody(t *testing.T) { @@ -569,7 +538,7 @@ func TestRunRedactsACredentialEchoedInABody(t *testing.T) { // token turns up. Assert on the same expression the code logs, so moving // the body from the error to the log cannot quietly move the leak with it. echoed := `{"rejected":"token=s3cr3t"}` - logged := step.redactString(explain([]byte(echoed))) + logged := step.redactString(httputil.Explain([]byte(echoed))) if strings.Contains(logged, "s3cr3t") { t.Errorf("the credential survives into the log line: %s", logged) } @@ -1086,34 +1055,6 @@ func TestRunKeepsATrailingSlashOnThePath(t *testing.T) { } } -// The join itself: baseUrl cannot end in a slash and path must begin with one, -// so exactly one separator appears between them. Asserted so a change to either -// side cannot quietly produce a doubled or missing slash. -func TestBuildEndpointJoinsWithOneSlash(t *testing.T) { - t.Parallel() - - for _, tc := range []struct{ base, path, want string }{ - {"http://host:9100", "/get-daily", "http://host:9100/get-daily"}, - {"http://host:9100/api", "/get-daily", "http://host:9100/api/get-daily"}, - {"http://host:9100/", "/get-daily", "http://host:9100/get-daily"}, - } { - got, err := buildEndpoint(tc.base, model.ActionPlan{Method: http.MethodPost, Path: tc.path}, nil) - if err != nil { - t.Fatalf("buildEndpoint(%q, %q) returned an unexpected error: %v", tc.base, tc.path, err) - } - if got != tc.want { - t.Errorf("buildEndpoint(%q, %q) = %q, want %q", tc.base, tc.path, got, tc.want) - } - } -} - -// --- where the binding key lives ---------------------------------------------- -// -// A default, not a setting: every participant must agree where the halves of a -// binding key sit, or two adapters disagree about what a binding key is and -// requests silently fail to match. The override exists so a spec change can be -// tracked without waiting for a release, and has to be typed deliberately. - func TestNewUsesTheBecknConventionByDefault(t *testing.T) { t.Parallel() @@ -1644,155 +1585,6 @@ func TestRunRefusesAMappedQueryItCannotRender(t *testing.T) { // --- query rendering -------------------------------------------------------- -// asQueryValue carries the three JSON scalars and refuses everything else, -// naming the field so a mapping that produced one is told which. null is in the -// refused set on purpose: a parameter present but empty and a parameter absent -// mean different things to some upstreams, and a mapping says which it wants by -// omitting the field or setting "". -func TestAsQueryRefusesWhatCannotBeAQueryParameter(t *testing.T) { - t.Parallel() - - for _, tc := range []struct{ name, mapped string }{ - {"an object", `{"location":{"lat":19.9975}}`}, - {"an array", `{"days":[1,2,3]}`}, - {"null", `{"station":null}`}, - } { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - _, err := asQuery([]byte(tc.mapped)) - if err == nil { - t.Fatalf("asQuery(%s) was accepted; it cannot become a query parameter", tc.mapped) - } - // The field, so a mapping author knows which one to fix. - if !strings.Contains(err.Error(), "station") && - !strings.Contains(err.Error(), "location") && - !strings.Contains(err.Error(), "days") { - t.Errorf("error %q should name the offending field", err) - } - }) - } -} - -// An empty string IS carried: it is a scalar, and a mapping setting one is -// asking for the parameter to be present and empty. -func TestAsQueryCarriesAnEmptyString(t *testing.T) { - t.Parallel() - - got, err := asQuery([]byte(`{"token":""}`)) - if err != nil { - t.Fatalf("asQuery() returned an unexpected error: %v", err) - } - if got != "token=" { - t.Errorf("query = %q, want token= -- an empty string is a value, not an absence", got) - } -} - -func TestAsQueryRendersScalarsWithoutInventingPrecision(t *testing.T) { - t.Parallel() - - got, err := asQuery([]byte(`{"lat":19.9975,"count":3,"name":"imd","live":true}`)) - if err != nil { - t.Fatalf("asQuery() returned an unexpected error: %v", err) - } - for _, want := range []string{"lat=19.9975", "count=3", "name=imd", "live=true"} { - if !strings.Contains(got, want) { - t.Errorf("query %q is missing %q", got, want) - } - } -} - -func TestAsQueryHandlesAnEmptyMapping(t *testing.T) { - t.Parallel() - - got, err := asQuery([]byte(`{}`)) - if err != nil || got != "" { - t.Errorf("asQuery({}) = (%q, %v), want an empty query and no error", got, err) - } -} - -// Both bounds come from a registry row, so both are data. An attempt holds a -// goroutine and the inbound connection for its whole timeout, and the server's -// write timeout does not cancel the request context -- so retryMax 1000 with -// timeoutMs 60000 is one row deciding this process is busy for seventeen hours. -func TestBudgetClampsWhatTheRegistryAsksFor(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - timeoutMs int - retryMax int - wantTimeout time.Duration - wantRetries int - }{ - {"absent uses the contract's defaults", 0, 0, DefaultTimeout, DefaultRetryMax}, - {"within the ceilings is honoured", 2000, 3, 2 * time.Second, 3}, - {"exactly at the ceilings is honoured", int(MaxTimeout / time.Millisecond), MaxRetryMax, MaxTimeout, MaxRetryMax}, - {"a timeout past the ceiling is clamped", 600000, 0, MaxTimeout, DefaultRetryMax}, - {"retries past the ceiling are clamped", 0, 1000, DefaultTimeout, MaxRetryMax}, - {"both past the ceiling are clamped", 600000, 1000, MaxTimeout, MaxRetryMax}, - {"negative values fall back to the defaults", -1, -1, DefaultTimeout, DefaultRetryMax}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - gotTimeout, gotRetries := budget(model.ActionPlan{ - TimeoutMs: tt.timeoutMs, RetryMax: tt.retryMax, - }) - if gotTimeout != tt.wantTimeout { - t.Errorf("timeout = %v, want %v", gotTimeout, tt.wantTimeout) - } - if gotRetries != tt.wantRetries { - t.Errorf("retries = %d, want %d", gotRetries, tt.wantRetries) - } - }) - } -} - -// The shift this replaced overflowed int64 once it reached 38 at a 50ms base. -// The wrapped value is NEGATIVE, so it passed the ceiling check and was -// returned, and a sleep on a negative duration returns immediately -- the -// retry loop then spun as fast as the provider could refuse. Past 64 it -// yielded 0, with the same effect. Worse, it was not monotonic: attempt 40 -// wrapped back to a sane 800ms, so the symptom came and went by attempt count. -// -// The clamp on retryMax now keeps attempts to MaxRetryMax+1, which puts the -// overflow out of reach through call(). This is asserted anyway, because -// backoff is a package function and a ceiling somewhere else is not a -// property of this one. -func TestBackoffNeverReturnsANonPositiveDuration(t *testing.T) { - t.Parallel() - - for _, attempt := range []int{0, 1, 2, 3, 4, 5, 6, 37, 38, 39, 40, 63, 64, 65, 100, 1000} { - got := backoff(attempt) - if got <= 0 { - t.Errorf("backoff(%d) = %v; a non-positive wait makes the retry loop spin", attempt, got) - } - if got > RetryBackoffMax { - t.Errorf("backoff(%d) = %v, above the %v ceiling", attempt, got, RetryBackoffMax) - } - } -} - -// The doubling itself, which the overflow fix must not have changed. -func TestBackoffDoublesToTheCeiling(t *testing.T) { - t.Parallel() - - want := []time.Duration{ - 50 * time.Millisecond, // attempt 1 - 100 * time.Millisecond, // 2 - 200 * time.Millisecond, // 3 - 400 * time.Millisecond, // 4 - 800 * time.Millisecond, // 5, at the ceiling - 800 * time.Millisecond, // 6, held there - } - for i, w := range want { - if got := backoff(i + 1); got != w { - t.Errorf("backoff(%d) = %v, want %v", i+1, got, w) - } - } -} - // redact used to return errors.New(text), which reported the right thing and // broke errors.Is. The redacted value is what gets %w-wrapped into the final // 502, so under a query-string scheme -- and only then -- a caller testing for @@ -1846,34 +1638,6 @@ func TestRedactLeavesAnUnchangedErrorAlone(t *testing.T) { } } -// hasBody upper-cased privately, which made the method look case-insensitive -// when it is not: NewRequestWithContext transmits it verbatim, so a registry -// row reading `method: "post"` sent `post /path HTTP/1.1`. The body was -// attached correctly, and nginx answered 405 -- classified permanent, and -// surfacing as a 502 "provider did not answer". -func TestCanonicalMethodFixesTheCaseTheRowWasWrittenIn(t *testing.T) { - t.Parallel() - - tests := []struct{ in, want string }{ - {"post", http.MethodPost}, - {"PoSt", http.MethodPost}, - {"POST", http.MethodPost}, - {"get", http.MethodGet}, - {"delete", http.MethodDelete}, - {"patch", http.MethodPatch}, - // Left alone: upper-casing everything would restrict an upstream - // entitled to a method this list has not heard of. - {"FrobNicate", "FrobNicate"}, - // Empty stays empty; net/http documents "" as GET and substitutes it. - {"", ""}, - } - for _, tt := range tests { - if got := canonicalMethod(tt.in); got != tt.want { - t.Errorf("canonicalMethod(%q) = %q, want %q", tt.in, got, tt.want) - } - } -} - // The end of it: what the provider actually receives on the request line. func TestRunSendsTheMethodInCanonicalCase(t *testing.T) { t.Parallel() @@ -1901,64 +1665,6 @@ func TestRunSendsTheMethodInCanonicalCase(t *testing.T) { } } -// The registry publishes two urls per action and only one of them was checked. -// jsonmapper has always validated its mapping reference this way; this is the -// same check on the base url beside it. -func TestVerifyBaseURL(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - baseURL string - wantErr bool - }{ - {"http is fine", "http://provider:9100", false}, - {"https is fine", "https://provider.example.com/api", false}, - {"empty is refused", "", true}, - // The case from the review: a scheme left off. Without this check it - // failed inside NewRequestWithContext and arrived as a 502. - {"a host and port with no scheme is refused", "registry:8081", true}, - {"a bare host is refused", "provider", true}, - {"a scheme that is not http is refused", "file:///etc/passwd", true}, - {"a scheme with no host is refused", "http://", true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - err := verifyBaseURL(tt.baseURL) - if tt.wantErr && err == nil { - t.Errorf("verifyBaseURL(%q) = nil, want an error", tt.baseURL) - } - if !tt.wantErr && err != nil { - t.Errorf("verifyBaseURL(%q) = %v, want nil", tt.baseURL, err) - } - }) - } -} - -// A dot segment would be resolved by net/url, so the request that left would -// not be the request the row described. A fragment is never sent at all. -func TestVerifyPathRefusesDotSegmentsAndFragments(t *testing.T) { - t.Parallel() - - for _, path := range []string{ - "/../admin", - "/v1/../../etc", - "/v1/./get-daily", - "/get-daily#section", - } { - if err := verifyPath(path); err == nil { - t.Errorf("verifyPath(%q) = nil, want it refused", path) - } - } - // A dot inside a segment is an ordinary character and must still pass. - for _, path := range []string{"/v1/get-daily", "/v1/data.json", "/a..b"} { - if err := verifyPath(path); err != nil { - t.Errorf("verifyPath(%q) = %v, want nil", path, err) - } - } -} - // The classification is the point. A row that cannot produce a request is a // bad request, not a provider that failed to answer -- and it must not be // retried on the way to being reported. @@ -2054,7 +1760,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, tc.tweak) // The same expression the non-2xx path logs. - logged := step.redactString(explain([]byte(tc.body))) + logged := step.redactString(httputil.Explain([]byte(tc.body))) if strings.Contains(logged, tc.secret) { t.Errorf("the credential survives into the log line: %s", logged) @@ -2468,7 +2174,7 @@ func TestOAuth2RedactsTheSecretAndTheToken(t *testing.T) { for _, secret := range []string{`s3cr3t"quoted`, "the-token"} { body := fmt.Sprintf(`{"error":"rejected %s"}`, secret) - got := step.redactString(explain([]byte(body))) + got := step.redactString(httputil.Explain([]byte(body))) if strings.Contains(got, secret) { t.Errorf("%q survived redaction: %s", secret, got) } diff --git a/pkg/plugin/implementation/internal/common/constant.go b/pkg/plugin/implementation/internal/common/constant.go index 66b0623a..d86e18fe 100644 --- a/pkg/plugin/implementation/internal/common/constant.go +++ b/pkg/plugin/implementation/internal/common/constant.go @@ -6,34 +6,9 @@ import ( "time" ) -// Defaults applied when the registry or the operator leaves a setting out. -const ( - // Zero retries is deliberate: a retry on a non-idempotent action is a - // second booking, so a provider is retried only where the operator said so. - DefaultTimeout = 15 * time.Second - DefaultRetryMax = 0 - // The response is mapped in memory, so an unbounded one is an unbounded - // allocation. - DefaultMaxResponseBytes = 4 << 20 // 4 MiB - - // A ceiling on what a registry row may ask for, because the row is DATA and - // an attempt holds a goroutine and the inbound connection for its whole - // timeout. retryMax 1000 with timeoutMs 60000 would pin both for about - // seventeen hours. - // - // Clamped rather than refused: a row that overreaches is a mistake, and - // serving it with a sane budget beats failing the capability outright. - MaxTimeout = 30 * time.Second - MaxRetryMax = 5 -) - -// How long this step waits between attempts. -const ( - // The first wait, doubling up to the max. Short: an operator setting 5 - // retries asked to ride out a brief outage, not for seconds of latency. - RetryBackoffBase = 50 * time.Millisecond - RetryBackoffMax = 800 * time.Millisecond -) +// DefaultMaxResponseBytes caps what is read from a provider. The response is +// mapped in memory, so an unbounded one is an unbounded allocation. +const DefaultMaxResponseBytes = 4 << 20 // 4 MiB // redactedMarker stands in for a credential in anything logged or returned. const redactedMarker = "REDACTED" @@ -66,7 +41,3 @@ const codeUpstreamUnavailable = "NET_DOWNSTREAM_UNAVAILABLE" // the provider, so a request that passed the expiry check cannot arrive after // the token died. Costs one extra exchange per lifetime. const tokenRefreshSkew = 60 * time.Second - -// How much of a failed response is quoted: enough for the provider's message, -// not a page of HTML in a log line. -const explainLimit = 300 diff --git a/pkg/plugin/implementation/internal/common/http.go b/pkg/plugin/implementation/internal/common/http.go index 4c2c84b6..a795cd0f 100644 --- a/pkg/plugin/implementation/internal/common/http.go +++ b/pkg/plugin/implementation/internal/common/http.go @@ -3,57 +3,25 @@ package common import ( - "bytes" "context" - "encoding/json" - "errors" "fmt" "io" "net/http" - "net/url" - "strconv" - "strings" "time" "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" ) -// budget resolves how long one attempt may take and how many retries follow, -// applying the registry's values within this deployment's ceilings. -// -// Separate from call so both bounds can be asserted without a server that -// sleeps for the timeout being tested. -// -// retryMax counts retries, not attempts: the call is always made once, and an -// absent retryMax means the same as an explicit 0. -func budget(call model.ActionPlan) (time.Duration, int) { - timeout := DefaultTimeout - if call.TimeoutMs > 0 { - timeout = time.Duration(call.TimeoutMs) * time.Millisecond - } - if timeout > MaxTimeout { - timeout = MaxTimeout - } - - retries := DefaultRetryMax - if call.RetryMax > 0 { - retries = call.RetryMax - } - if retries > MaxRetryMax { - retries = MaxRetryMax - } - return timeout, retries -} - -// call makes the upstream request the plan describes, retrying within budget. +// call makes the upstream request the plan describes, retrying within httputil.Budget. func (s *Step) call(ctx context.Context, auth *authenticator, baseURL string, call model.ActionPlan, mapped []byte) ([]byte, error) { - endpoint, err := buildEndpoint(baseURL, call, mapped) + endpoint, err := httputil.BuildEndpoint(baseURL, call, mapped) if err != nil { return nil, err } - timeout, retries := budget(call) + timeout, retries := httputil.Budget(call) if d := time.Duration(call.TimeoutMs) * time.Millisecond; d > timeout { log.Warnf(ctx, "registry asks for a %v timeout; using the %v ceiling", d, timeout) } @@ -84,11 +52,11 @@ func (s *Step) call(ctx context.Context, auth *authenticator, baseURL string, ca // identically however often they are tried. Retrying the credential // case is the worst: it reports a missing environment variable as the // provider being down. - if isPermanent(err) { + if httputil.IsPermanent(err) { break } if attempt < attempts { - if err := sleep(ctx, backoff(attempt)); err != nil { + if err := httputil.Sleep(ctx, httputil.Backoff(attempt)); err != nil { break } } @@ -97,72 +65,22 @@ func (s *Step) call(ctx context.Context, auth *authenticator, baseURL string, ca fmt.Errorf("provider did not answer after %d attempts: %w", attempts, lastErr)) } -// permanentErr marks a failure no retry can fix. Kept unexported and detected -// with errors.As, so a caller of this package sees only the underlying error. -type permanentErr struct{ error } - -func (p permanentErr) Unwrap() error { return p.error } - -// doNotRetry marks err as not worth repeating. -func doNotRetry(err error) error { return permanentErr{err} } - -// isPermanent reports whether err is one no further attempt would change. -func isPermanent(err error) bool { - var permanent permanentErr - return errors.As(err, &permanent) -} - -// backoff is how long to wait before the next attempt. -// -// Exponential from a short base and capped: a briefly busy provider is worth -// waiting out, anything longer is a timeout's job. With no wait, retryMax 5 -// spends its budget in a couple of milliseconds -- the same failure six times. -func backoff(attempt int) time.Duration { - if attempt <= 1 { - return RetryBackoffBase - } - // Doubled in a loop that stops at the ceiling, not shifted then clamped. - // `base << (attempt-1)` overflows int64 around shift 38 and the wrapped - // value is NEGATIVE, so it passes the ceiling check and sleeps for no time - // -- the retry loop then spins as fast as the provider can refuse. - wait := RetryBackoffBase - for i := 1; i < attempt && wait < RetryBackoffMax; i++ { - wait *= 2 - } - if wait > RetryBackoffMax { - return RetryBackoffMax - } - return wait -} - -// sleep waits, or reports that the context ended first. -func sleep(ctx context.Context, d time.Duration) error { - timer := time.NewTimer(d) - defer timer.Stop() - select { - case <-ctx.Done(): - return ctx.Err() - case <-timer.C: - return nil - } -} - // attempt makes one upstream request. func (s *Step) attempt(ctx context.Context, auth *authenticator, call model.ActionPlan, endpoint string, mapped []byte, timeout time.Duration) ([]byte, error) { attemptCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - method := canonicalMethod(call.Method) - req, err := http.NewRequestWithContext(attemptCtx, method, endpoint, requestBody(method, mapped)) + method := httputil.CanonicalMethod(call.Method) + req, err := http.NewRequestWithContext(attemptCtx, method, endpoint, httputil.RequestBody(method, mapped)) if err != nil { - return nil, doNotRetry(fmt.Errorf("could not build the request: %w", err)) + return nil, httputil.DoNotRetry(fmt.Errorf("could not build the request: %w", err)) } - if hasBody(method) { + if httputil.HasBody(method) { req.Header.Set("Content-Type", "application/json") } if err := s.authenticate(auth, req); err != nil { // A missing or unreadable credential is configuration, not weather. - return nil, doNotRetry(err) + return nil, httputil.DoNotRetry(err) } // The URL as it went on the wire, credential removed. At info because this @@ -182,7 +100,7 @@ func (s *Step) attempt(ctx context.Context, auth *authenticator, call model.Acti log.Infof(ctx, "%s %s -> %s, %d bytes", method, requested, resp.Status, len(body)) if int64(len(body)) > s.config.MaxResponseBytes { // Asking again will not make the answer smaller. - return nil, doNotRetry(fmt.Errorf("response exceeds the %d byte limit", s.config.MaxResponseBytes)) + return nil, httputil.DoNotRetry(fmt.Errorf("response exceeds the %d byte limit", s.config.MaxResponseBytes)) } // Any 2xx, not 200 alone: 202, 204 and 201 are all legitimate answers. 3xx // does not reach here, since the client follows redirects. @@ -194,215 +112,14 @@ func (s *Step) attempt(ctx context.Context, auth *authenticator, call model.Acti // Redacted on the way to the log too: a rejected request is often // quoted back, credential and all. log.Warnf(ctx, "provider returned %s for %s %s: %s", - resp.Status, method, requested, s.redactString(explain(body))) + resp.Status, method, requested, s.redactString(httputil.Explain(body))) err := fmt.Errorf("provider returned %s", resp.Status) // 5xx and 429 ask to be tried again. Every other 4xx is a statement // about the request, which will not improve. if resp.StatusCode < http.StatusInternalServerError && resp.StatusCode != http.StatusTooManyRequests { - return nil, doNotRetry(err) + return nil, httputil.DoNotRetry(err) } return nil, err } return body, nil } - -// explain renders a failed response body for a human. -// -// The status says a call failed and nothing about why. The provider's own -// account is in the body -- Agmarknet says "no data" in the body of a 400. -func explain(body []byte) string { - text := strings.TrimSpace(string(body)) - if text == "" { - return "(no body)" - } - // Collapse whitespace: indented JSON or an HTML error page should not - // spread one failure over forty log lines. - text = strings.Join(strings.Fields(text), " ") - if len(text) > explainLimit { - return text[:explainLimit] + "... (truncated)" - } - return text -} - -// buildEndpoint joins the plan's base URL and path, carrying the mapped request -// as query parameters when the method takes no body. -func buildEndpoint(baseURL string, call model.ActionPlan, mapped []byte) (string, error) { - if err := verifyBaseURL(baseURL); err != nil { - return "", err - } - if err := verifyPath(call.Path); err != nil { - return "", err - } - - // baseUrl cannot end in a slash and path must begin with one. The trim is - // belt and braces, for a row predating that rule. - endpoint := strings.TrimSuffix(baseURL, "/") + call.Path - if hasBody(call.Method) { - return endpoint, nil - } - - query, err := asQuery(mapped) - if err != nil { - return "", err - } - if query == "" { - return endpoint, nil - } - if strings.Contains(endpoint, "?") { - return endpoint + "&" + query, nil - } - return endpoint + "?" + query, nil -} - -// verifyPath refuses a published path nobody could have meant. -// -// The registry constrains this, but it deploys separately, so a row that slips -// through must fail here with something an operator can act on rather than as -// a 404 three hops away. -// -// An empty segment is the case worth catching: "//get-daily" is never -// deliberate. A trailing slash is left alone -- "/api/" and "/api" are a -// distinction some APIs genuinely make. -func verifyPath(path string) error { - if path == "" { - return model.NewBadReqErr("", errors.New("the registry publishes no path for this action")) - } - if !strings.HasPrefix(path, "/") { - return model.NewBadReqErr("", fmt.Errorf( - "path %q does not begin with a slash, so it cannot be joined to a base url", path)) - } - if strings.Contains(path, "//") { - return model.NewBadReqErr("", fmt.Errorf( - "path %q has an empty segment; write it with single slashes", path)) - } - // A dot segment is refused, not resolved: net/url would resolve it quietly, - // and the request that left would not be the request the row described. - for _, segment := range strings.Split(path, "/") { - if segment == ".." || segment == "." { - return model.NewBadReqErr("", fmt.Errorf( - "path %q contains the %q segment; publish the path it resolves to instead", - path, segment)) - } - } - // A fragment is never sent, so a row carrying one describes a request that - // cannot be made. The transport would drop it silently and the row would - // look honoured. - if strings.Contains(path, "#") { - return model.NewBadReqErr("", fmt.Errorf( - "path %q contains a fragment, which is never sent to a server", path)) - } - return nil -} - -// verifyBaseURL checks the base url before it is joined to a path, so a row -// that cannot produce a request says so as a bad request rather than as the -// provider being unreachable. -// -// Without this, a missing scheme (`baseUrl: "registry:8081"`) failed inside -// NewRequestWithContext and arrived as a 502 blaming the provider -- retried -// on the way there. -func verifyBaseURL(baseURL string) error { - if baseURL == "" { - return model.NewBadReqErr("", errors.New("the registry publishes no base url for this provider")) - } - parsed, err := url.Parse(baseURL) - if err != nil { - return model.NewBadReqErr("", fmt.Errorf("invalid base url %q: %w", baseURL, err)) - } - if parsed.Scheme != "http" && parsed.Scheme != "https" { - return model.NewBadReqErr("", fmt.Errorf( - "base url %q must be http or https", baseURL)) - } - if parsed.Host == "" { - return model.NewBadReqErr("", fmt.Errorf("base url %q names no host", baseURL)) - } - return nil -} - -// asQuery renders a mapped request as query parameters, for a method with no -// body. Scalars only -- see asQueryValue. -func asQuery(mapped []byte) (string, error) { - if len(bytes.TrimSpace(mapped)) == 0 { - return "", nil - } - var fields map[string]any - if err := json.Unmarshal(mapped, &fields); err != nil { - return "", fmt.Errorf("mapped request is not an object, so it cannot become a query: %w", err) - } - - values := url.Values{} - for name, value := range fields { - rendered, ok := asQueryValue(value) - if !ok { - return "", fmt.Errorf("mapped field %q is not a scalar and cannot become a query parameter", name) - } - values.Set(name, rendered) - } - return values.Encode(), nil -} - -// asQueryValue renders one mapped field as a query parameter value, reporting -// false when the field cannot be one. -// -// The three JSON scalars and nothing else. An object or array has no single -// obvious encoding -- repeated keys, comma-joined, indexed are all in use -// somewhere -- so choosing here would put a convention in Go that belongs in -// the mapping. The caller turns false into an error naming the field. -// -// null is refused rather than sent as empty: present-but-empty and absent mean -// different things to some upstreams, and a mapping says which by omitting the -// field or setting "". An empty string IS carried, being a scalar. -func asQueryValue(value any) (string, bool) { - switch typed := value.(type) { - case string: - return typed, true - case bool: - return strconv.FormatBool(typed), true - case float64: - // 'g' with -1 precision round-trips without inventing trailing zeros, so - // 19.9975 stays 19.9975 rather than becoming 19.997500. - return strconv.FormatFloat(typed, 'g', -1, 64), true - default: - return "", false - } -} - -// requestBody returns the body to send, which is none for methods that take none. -func requestBody(method string, mapped []byte) io.Reader { - if !hasBody(method) { - return nil - } - return bytes.NewReader(mapped) -} - -// hasBody reports whether a method carries a request body. -func hasBody(method string) bool { - switch canonicalMethod(method) { - case http.MethodGet, http.MethodHead, http.MethodDelete, "": - return false - default: - return true - } -} - -// canonicalMethod upper-cases a known HTTP method and leaves anything else -// alone. -// -// NewRequestWithContext sends the method verbatim, so a row reading "post" put -// `post /path HTTP/1.1` on the wire and most gateways answer that with 405 -- -// which classifies permanent and surfaces as "provider did not answer". -// -// Unknown methods pass through: net/http already rejects an invalid token, and -// upper-casing everything would restrict an upstream entitled to a method this -// list has not heard of. An empty method stays empty, since net/http already -// treats "" as GET. -func canonicalMethod(method string) string { - upper := strings.ToUpper(method) - switch upper { - case http.MethodGet, http.MethodHead, http.MethodPost, http.MethodPut, - http.MethodPatch, http.MethodDelete, http.MethodConnect, - http.MethodOptions, http.MethodTrace: - return upper - } - return method -} diff --git a/pkg/plugin/implementation/internal/common/httputil/constant.go b/pkg/plugin/implementation/internal/common/httputil/constant.go new file mode 100644 index 00000000..1493ea0f --- /dev/null +++ b/pkg/plugin/implementation/internal/common/httputil/constant.go @@ -0,0 +1,40 @@ +// Constants for the call and its retry budget. +// +// Here rather than in common's constant.go because the code that reads them +// is here, and common imports this package -- so a shared file would be an +// import cycle. +package httputil + +import ( + "time" +) + +// Defaults applied when the registry or the operator leaves a setting out. +const ( + // Zero retries is deliberate: a retry on a non-idempotent action is a + // second booking, so a provider is retried only where the operator said so. + DefaultTimeout = 15 * time.Second + DefaultRetryMax = 0 + + // A ceiling on what a registry row may ask for, because the row is DATA and + // an attempt holds a goroutine and the inbound connection for its whole + // timeout. retryMax 1000 with timeoutMs 60000 would pin both for about + // seventeen hours. + // + // Clamped rather than refused: a row that overreaches is a mistake, and + // serving it with a sane budget beats failing the capability outright. + MaxTimeout = 30 * time.Second + MaxRetryMax = 5 +) + +// How long this step waits between attempts. +const ( + // The first wait, doubling up to the max. Short: an operator setting 5 + // retries asked to ride out a brief outage, not for seconds of latency. + RetryBackoffBase = 50 * time.Millisecond + RetryBackoffMax = 800 * time.Millisecond +) + +// How much of a failed response is quoted: enough for the provider's message, +// not a page of HTML in a log line. +const explainLimit = 300 diff --git a/pkg/plugin/implementation/internal/common/httputil/endpoint.go b/pkg/plugin/implementation/internal/common/httputil/endpoint.go new file mode 100644 index 00000000..be6516db --- /dev/null +++ b/pkg/plugin/implementation/internal/common/httputil/endpoint.go @@ -0,0 +1,107 @@ +// Building the URL a request goes to, and refusing one a registry row could +// not have meant. +package httputil + +import ( + "errors" + "fmt" + "net/url" + "strings" + + "github.com/beckn-one/beckn-onix/pkg/model" +) + +// BuildEndpoint joins the plan's base URL and path, carrying the mapped request +// as query parameters when the method takes no body. +func BuildEndpoint(baseURL string, call model.ActionPlan, mapped []byte) (string, error) { + if err := verifyBaseURL(baseURL); err != nil { + return "", err + } + if err := verifyPath(call.Path); err != nil { + return "", err + } + + // baseUrl cannot end in a slash and path must begin with one. The trim is + // belt and braces, for a row predating that rule. + endpoint := strings.TrimSuffix(baseURL, "/") + call.Path + if HasBody(call.Method) { + return endpoint, nil + } + + query, err := asQuery(mapped) + if err != nil { + return "", err + } + if query == "" { + return endpoint, nil + } + if strings.Contains(endpoint, "?") { + return endpoint + "&" + query, nil + } + return endpoint + "?" + query, nil +} + +// verifyPath refuses a published path nobody could have meant. +// +// The registry constrains this, but it deploys separately, so a row that slips +// through must fail here with something an operator can act on rather than as +// a 404 three hops away. +// +// An empty segment is the case worth catching: "//get-daily" is never +// deliberate. A trailing slash is left alone -- "/api/" and "/api" are a +// distinction some APIs genuinely make. +func verifyPath(path string) error { + if path == "" { + return model.NewBadReqErr("", errors.New("the registry publishes no path for this action")) + } + if !strings.HasPrefix(path, "/") { + return model.NewBadReqErr("", fmt.Errorf( + "path %q does not begin with a slash, so it cannot be joined to a base url", path)) + } + if strings.Contains(path, "//") { + return model.NewBadReqErr("", fmt.Errorf( + "path %q has an empty segment; write it with single slashes", path)) + } + // A dot segment is refused, not resolved: net/url would resolve it quietly, + // and the request that left would not be the request the row described. + for _, segment := range strings.Split(path, "/") { + if segment == ".." || segment == "." { + return model.NewBadReqErr("", fmt.Errorf( + "path %q contains the %q segment; publish the path it resolves to instead", + path, segment)) + } + } + // A fragment is never sent, so a row carrying one describes a request that + // cannot be made. The transport would drop it silently and the row would + // look honoured. + if strings.Contains(path, "#") { + return model.NewBadReqErr("", fmt.Errorf( + "path %q contains a fragment, which is never sent to a server", path)) + } + return nil +} + +// verifyBaseURL checks the base url before it is joined to a path, so a row +// that cannot produce a request says so as a bad request rather than as the +// provider being unreachable. +// +// Without this, a missing scheme (`baseUrl: "registry:8081"`) failed inside +// NewRequestWithContext and arrived as a 502 blaming the provider -- retried +// on the way there. +func verifyBaseURL(baseURL string) error { + if baseURL == "" { + return model.NewBadReqErr("", errors.New("the registry publishes no base url for this provider")) + } + parsed, err := url.Parse(baseURL) + if err != nil { + return model.NewBadReqErr("", fmt.Errorf("invalid base url %q: %w", baseURL, err)) + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return model.NewBadReqErr("", fmt.Errorf( + "base url %q must be http or https", baseURL)) + } + if parsed.Host == "" { + return model.NewBadReqErr("", fmt.Errorf("base url %q names no host", baseURL)) + } + return nil +} diff --git a/pkg/plugin/implementation/internal/common/httputil/explain.go b/pkg/plugin/implementation/internal/common/httputil/explain.go new file mode 100644 index 00000000..839d602d --- /dev/null +++ b/pkg/plugin/implementation/internal/common/httputil/explain.go @@ -0,0 +1,22 @@ +// Quoting a failed response body for a human. +package httputil + +import "strings" + +// Explain renders a failed response body for a human. +// +// The status says a call failed and nothing about why. The provider's own +// account is in the body -- Agmarknet says "no data" in the body of a 400. +func Explain(body []byte) string { + text := strings.TrimSpace(string(body)) + if text == "" { + return "(no body)" + } + // Collapse whitespace: indented JSON or an HTML error page should not + // spread one failure over forty log lines. + text = strings.Join(strings.Fields(text), " ") + if len(text) > explainLimit { + return text[:explainLimit] + "... (truncated)" + } + return text +} diff --git a/pkg/plugin/implementation/internal/common/httputil/httputil_test.go b/pkg/plugin/implementation/internal/common/httputil/httputil_test.go new file mode 100644 index 00000000..9d4b2306 --- /dev/null +++ b/pkg/plugin/implementation/internal/common/httputil/httputil_test.go @@ -0,0 +1,305 @@ +package httputil + +import ( + "net/http" + "strings" + "testing" + "time" + + "github.com/beckn-one/beckn-onix/pkg/model" +) + +// explain still collapses whitespace, because the body it prepares now goes to +// a log line rather than an error -- an indented body would spread one failure +// over several lines either way. +func TestExplainCollapsesWhitespace(t *testing.T) { + t.Parallel() + + got := Explain([]byte("{\n \"message\": \"no data available\"\n}")) + if strings.Contains(got, "\n") { + t.Errorf("Explain(%q) left a newline in", got) + } + if !strings.Contains(got, "no data available") { + t.Errorf("explain = %q, want the provider's message preserved", got) + } +} + +// A body is quoted, not dumped: a provider answering with a page of HTML must +// not put all of it in a log line or a NACK. +func TestExplainTruncatesAndHandlesAnEmptyBody(t *testing.T) { + t.Parallel() + + if got := Explain(nil); got != "(no body)" { + t.Errorf("Explain(nil) = %q, want a marker rather than an empty string", got) + } + long := Explain([]byte(strings.Repeat("x", explainLimit+50))) + if len(long) > explainLimit+len("... (truncated)") { + t.Errorf("explain kept %d characters, want it truncated near %d", len(long), explainLimit) + } + if !strings.HasSuffix(long, "(truncated)") { + t.Errorf("a truncated body should say so, got %q", long[len(long)-20:]) + } +} + +// The join itself: baseUrl cannot end in a slash and path must begin with one, +// so exactly one separator appears between them. Asserted so a change to either +// side cannot quietly produce a doubled or missing slash. +func TestBuildEndpointJoinsWithOneSlash(t *testing.T) { + t.Parallel() + + for _, tc := range []struct{ base, path, want string }{ + {"http://host:9100", "/get-daily", "http://host:9100/get-daily"}, + {"http://host:9100/api", "/get-daily", "http://host:9100/api/get-daily"}, + {"http://host:9100/", "/get-daily", "http://host:9100/get-daily"}, + } { + got, err := BuildEndpoint(tc.base, model.ActionPlan{Method: http.MethodPost, Path: tc.path}, nil) + if err != nil { + t.Fatalf("BuildEndpoint(%q, %q) returned an unexpected error: %v", tc.base, tc.path, err) + } + if got != tc.want { + t.Errorf("BuildEndpoint(%q, %q) = %q, want %q", tc.base, tc.path, got, tc.want) + } + } +} + +// --- where the binding key lives ---------------------------------------------- +// +// A default, not a setting: every participant must agree where the halves of a +// binding key sit, or two adapters disagree about what a binding key is and +// requests silently fail to match. The override exists so a spec change can be +// tracked without waiting for a release, and has to be typed deliberately. + +// asQueryValue carries the three JSON scalars and refuses everything else, +// naming the field so a mapping that produced one is told which. null is in the +// refused set on purpose: a parameter present but empty and a parameter absent +// mean different things to some upstreams, and a mapping says which it wants by +// omitting the field or setting "". +func TestAsQueryRefusesWhatCannotBeAQueryParameter(t *testing.T) { + t.Parallel() + + for _, tc := range []struct{ name, mapped string }{ + {"an object", `{"location":{"lat":19.9975}}`}, + {"an array", `{"days":[1,2,3]}`}, + {"null", `{"station":null}`}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + _, err := asQuery([]byte(tc.mapped)) + if err == nil { + t.Fatalf("asQuery(%s) was accepted; it cannot become a query parameter", tc.mapped) + } + // The field, so a mapping author knows which one to fix. + if !strings.Contains(err.Error(), "station") && + !strings.Contains(err.Error(), "location") && + !strings.Contains(err.Error(), "days") { + t.Errorf("error %q should name the offending field", err) + } + }) + } +} + +// An empty string IS carried: it is a scalar, and a mapping setting one is +// asking for the parameter to be present and empty. +func TestAsQueryCarriesAnEmptyString(t *testing.T) { + t.Parallel() + + got, err := asQuery([]byte(`{"token":""}`)) + if err != nil { + t.Fatalf("asQuery() returned an unexpected error: %v", err) + } + if got != "token=" { + t.Errorf("query = %q, want token= -- an empty string is a value, not an absence", got) + } +} + +func TestAsQueryRendersScalarsWithoutInventingPrecision(t *testing.T) { + t.Parallel() + + got, err := asQuery([]byte(`{"lat":19.9975,"count":3,"name":"imd","live":true}`)) + if err != nil { + t.Fatalf("asQuery() returned an unexpected error: %v", err) + } + for _, want := range []string{"lat=19.9975", "count=3", "name=imd", "live=true"} { + if !strings.Contains(got, want) { + t.Errorf("query %q is missing %q", got, want) + } + } +} + +func TestAsQueryHandlesAnEmptyMapping(t *testing.T) { + t.Parallel() + + got, err := asQuery([]byte(`{}`)) + if err != nil || got != "" { + t.Errorf("asQuery({}) = (%q, %v), want an empty query and no error", got, err) + } +} + +// Both bounds come from a registry row, so both are data. An attempt holds a +// goroutine and the inbound connection for its whole timeout, and the server's +// write timeout does not cancel the request context -- so retryMax 1000 with +// timeoutMs 60000 is one row deciding this process is busy for seventeen hours. +func TestBudgetClampsWhatTheRegistryAsksFor(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + timeoutMs int + retryMax int + wantTimeout time.Duration + wantRetries int + }{ + {"absent uses the contract's defaults", 0, 0, DefaultTimeout, DefaultRetryMax}, + {"within the ceilings is honoured", 2000, 3, 2 * time.Second, 3}, + {"exactly at the ceilings is honoured", int(MaxTimeout / time.Millisecond), MaxRetryMax, MaxTimeout, MaxRetryMax}, + {"a timeout past the ceiling is clamped", 600000, 0, MaxTimeout, DefaultRetryMax}, + {"retries past the ceiling are clamped", 0, 1000, DefaultTimeout, MaxRetryMax}, + {"both past the ceiling are clamped", 600000, 1000, MaxTimeout, MaxRetryMax}, + {"negative values fall back to the defaults", -1, -1, DefaultTimeout, DefaultRetryMax}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + gotTimeout, gotRetries := Budget(model.ActionPlan{ + TimeoutMs: tt.timeoutMs, RetryMax: tt.retryMax, + }) + if gotTimeout != tt.wantTimeout { + t.Errorf("timeout = %v, want %v", gotTimeout, tt.wantTimeout) + } + if gotRetries != tt.wantRetries { + t.Errorf("retries = %d, want %d", gotRetries, tt.wantRetries) + } + }) + } +} + +// The shift this replaced overflowed int64 once it reached 38 at a 50ms base. +// The wrapped value is NEGATIVE, so it passed the ceiling check and was +// returned, and a sleep on a negative duration returns immediately -- the +// retry loop then spun as fast as the provider could refuse. Past 64 it +// yielded 0, with the same effect. Worse, it was not monotonic: attempt 40 +// wrapped back to a sane 800ms, so the symptom came and went by attempt count. +// +// The clamp on retryMax now keeps attempts to MaxRetryMax+1, which puts the +// overflow out of reach through call(). This is asserted anyway, because +// backoff is a package function and a ceiling somewhere else is not a +// property of this one. +func TestBackoffNeverReturnsANonPositiveDuration(t *testing.T) { + t.Parallel() + + for _, attempt := range []int{0, 1, 2, 3, 4, 5, 6, 37, 38, 39, 40, 63, 64, 65, 100, 1000} { + got := Backoff(attempt) + if got <= 0 { + t.Errorf("Backoff(%d) = %v; a non-positive wait makes the retry loop spin", attempt, got) + } + if got > RetryBackoffMax { + t.Errorf("Backoff(%d) = %v, above the %v ceiling", attempt, got, RetryBackoffMax) + } + } +} + +// The doubling itself, which the overflow fix must not have changed. +func TestBackoffDoublesToTheCeiling(t *testing.T) { + t.Parallel() + + want := []time.Duration{ + 50 * time.Millisecond, // attempt 1 + 100 * time.Millisecond, // 2 + 200 * time.Millisecond, // 3 + 400 * time.Millisecond, // 4 + 800 * time.Millisecond, // 5, at the ceiling + 800 * time.Millisecond, // 6, held there + } + for i, w := range want { + if got := Backoff(i + 1); got != w { + t.Errorf("Backoff(%d) = %v, want %v", i+1, got, w) + } + } +} + +// hasBody upper-cased privately, which made the method look case-insensitive +// when it is not: NewRequestWithContext transmits it verbatim, so a registry +// row reading `method: "post"` sent `post /path HTTP/1.1`. The body was +// attached correctly, and nginx answered 405 -- classified permanent, and +// surfacing as a 502 "provider did not answer". +func TestCanonicalMethodFixesTheCaseTheRowWasWrittenIn(t *testing.T) { + t.Parallel() + + tests := []struct{ in, want string }{ + {"post", http.MethodPost}, + {"PoSt", http.MethodPost}, + {"POST", http.MethodPost}, + {"get", http.MethodGet}, + {"delete", http.MethodDelete}, + {"patch", http.MethodPatch}, + // Left alone: upper-casing everything would restrict an upstream + // entitled to a method this list has not heard of. + {"FrobNicate", "FrobNicate"}, + // Empty stays empty; net/http documents "" as GET and substitutes it. + {"", ""}, + } + for _, tt := range tests { + if got := CanonicalMethod(tt.in); got != tt.want { + t.Errorf("CanonicalMethod(%q) = %q, want %q", tt.in, got, tt.want) + } + } +} + +// The registry publishes two urls per action and only one of them was checked. +// jsonmapper has always validated its mapping reference this way; this is the +// same check on the base url beside it. +func TestVerifyBaseURL(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + baseURL string + wantErr bool + }{ + {"http is fine", "http://provider:9100", false}, + {"https is fine", "https://provider.example.com/api", false}, + {"empty is refused", "", true}, + // The case from the review: a scheme left off. Without this check it + // failed inside NewRequestWithContext and arrived as a 502. + {"a host and port with no scheme is refused", "registry:8081", true}, + {"a bare host is refused", "provider", true}, + {"a scheme that is not http is refused", "file:///etc/passwd", true}, + {"a scheme with no host is refused", "http://", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := verifyBaseURL(tt.baseURL) + if tt.wantErr && err == nil { + t.Errorf("verifyBaseURL(%q) = nil, want an error", tt.baseURL) + } + if !tt.wantErr && err != nil { + t.Errorf("verifyBaseURL(%q) = %v, want nil", tt.baseURL, err) + } + }) + } +} + +// A dot segment would be resolved by net/url, so the request that left would +// not be the request the row described. A fragment is never sent at all. +func TestVerifyPathRefusesDotSegmentsAndFragments(t *testing.T) { + t.Parallel() + + for _, path := range []string{ + "/../admin", + "/v1/../../etc", + "/v1/./get-daily", + "/get-daily#section", + } { + if err := verifyPath(path); err == nil { + t.Errorf("verifyPath(%q) = nil, want it refused", path) + } + } + // A dot inside a segment is an ordinary character and must still pass. + for _, path := range []string{"/v1/get-daily", "/v1/data.json", "/a..b"} { + if err := verifyPath(path); err != nil { + t.Errorf("verifyPath(%q) = %v, want nil", path, err) + } + } +} diff --git a/pkg/plugin/implementation/internal/common/httputil/query.go b/pkg/plugin/implementation/internal/common/httputil/query.go new file mode 100644 index 00000000..d5e2e381 --- /dev/null +++ b/pkg/plugin/implementation/internal/common/httputil/query.go @@ -0,0 +1,58 @@ +// Rendering a mapped request as query parameters, for methods with no body. +package httputil + +import ( + "bytes" + "encoding/json" + "fmt" + "net/url" + "strconv" +) + +// asQuery renders a mapped request as query parameters, for a method with no +// body. Scalars only -- see asQueryValue. +func asQuery(mapped []byte) (string, error) { + if len(bytes.TrimSpace(mapped)) == 0 { + return "", nil + } + var fields map[string]any + if err := json.Unmarshal(mapped, &fields); err != nil { + return "", fmt.Errorf("mapped request is not an object, so it cannot become a query: %w", err) + } + + values := url.Values{} + for name, value := range fields { + rendered, ok := asQueryValue(value) + if !ok { + return "", fmt.Errorf("mapped field %q is not a scalar and cannot become a query parameter", name) + } + values.Set(name, rendered) + } + return values.Encode(), nil +} + +// asQueryValue renders one mapped field as a query parameter value, reporting +// false when the field cannot be one. +// +// The three JSON scalars and nothing else. An object or array has no single +// obvious encoding -- repeated keys, comma-joined, indexed are all in use +// somewhere -- so choosing here would put a convention in Go that belongs in +// the mapping. The caller turns false into an error naming the field. +// +// null is refused rather than sent as empty: present-but-empty and absent mean +// different things to some upstreams, and a mapping says which by omitting the +// field or setting "". An empty string IS carried, being a scalar. +func asQueryValue(value any) (string, bool) { + switch typed := value.(type) { + case string: + return typed, true + case bool: + return strconv.FormatBool(typed), true + case float64: + // 'g' with -1 precision round-trips without inventing trailing zeros, so + // 19.9975 stays 19.9975 rather than becoming 19.997500. + return strconv.FormatFloat(typed, 'g', -1, 64), true + default: + return "", false + } +} diff --git a/pkg/plugin/implementation/internal/common/httputil/request.go b/pkg/plugin/implementation/internal/common/httputil/request.go new file mode 100644 index 00000000..d3f14f0a --- /dev/null +++ b/pkg/plugin/implementation/internal/common/httputil/request.go @@ -0,0 +1,49 @@ +// The request body and method: what to send, and in what spelling. +package httputil + +import ( + "bytes" + "io" + "net/http" + "strings" +) + +// RequestBody returns the body to send, which is none for methods that take none. +func RequestBody(method string, mapped []byte) io.Reader { + if !HasBody(method) { + return nil + } + return bytes.NewReader(mapped) +} + +// HasBody reports whether a method carries a request body. +func HasBody(method string) bool { + switch CanonicalMethod(method) { + case http.MethodGet, http.MethodHead, http.MethodDelete, "": + return false + default: + return true + } +} + +// CanonicalMethod upper-cases a known HTTP method and leaves anything else +// alone. +// +// NewRequestWithContext sends the method verbatim, so a row reading "post" put +// `post /path HTTP/1.1` on the wire and most gateways answer that with 405 -- +// which classifies permanent and surfaces as "provider did not answer". +// +// Unknown methods pass through: net/http already rejects an invalid token, and +// upper-casing everything would restrict an upstream entitled to a method this +// list has not heard of. An empty method stays empty, since net/http already +// treats "" as GET. +func CanonicalMethod(method string) string { + upper := strings.ToUpper(method) + switch upper { + case http.MethodGet, http.MethodHead, http.MethodPost, http.MethodPut, + http.MethodPatch, http.MethodDelete, http.MethodConnect, + http.MethodOptions, http.MethodTrace: + return upper + } + return method +} diff --git a/pkg/plugin/implementation/internal/common/httputil/retry.go b/pkg/plugin/implementation/internal/common/httputil/retry.go new file mode 100644 index 00000000..c11cfd32 --- /dev/null +++ b/pkg/plugin/implementation/internal/common/httputil/retry.go @@ -0,0 +1,88 @@ +// The retry budget: how long an attempt may take, how many follow it, and +// which failures are worth repeating. +package httputil + +import ( + "context" + "errors" + "time" + + "github.com/beckn-one/beckn-onix/pkg/model" +) + +// Budget resolves how long one attempt may take and how many retries follow, +// applying the registry's values within this deployment's ceilings. +// +// Separate from call so both bounds can be asserted without a server that +// sleeps for the timeout being tested. +// +// retryMax counts retries, not attempts: the call is always made once, and an +// absent retryMax means the same as an explicit 0. +func Budget(call model.ActionPlan) (time.Duration, int) { + timeout := DefaultTimeout + if call.TimeoutMs > 0 { + timeout = time.Duration(call.TimeoutMs) * time.Millisecond + } + if timeout > MaxTimeout { + timeout = MaxTimeout + } + + retries := DefaultRetryMax + if call.RetryMax > 0 { + retries = call.RetryMax + } + if retries > MaxRetryMax { + retries = MaxRetryMax + } + return timeout, retries +} + +// permanentErr marks a failure no retry can fix. Kept unexported and detected +// with errors.As, so a caller of this package sees only the underlying error. +type permanentErr struct{ error } + +func (p permanentErr) Unwrap() error { return p.error } + +// DoNotRetry marks err as not worth repeating. +func DoNotRetry(err error) error { return permanentErr{err} } + +// IsPermanent reports whether err is one no further attempt would change. +func IsPermanent(err error) bool { + var permanent permanentErr + return errors.As(err, &permanent) +} + +// Backoff is how long to wait before the next attempt. +// +// Exponential from a short base and capped: a briefly busy provider is worth +// waiting out, anything longer is a timeout's job. With no wait, retryMax 5 +// spends its Budget in a couple of milliseconds -- the same failure six times. +func Backoff(attempt int) time.Duration { + if attempt <= 1 { + return RetryBackoffBase + } + // Doubled in a loop that stops at the ceiling, not shifted then clamped. + // `base << (attempt-1)` overflows int64 around shift 38 and the wrapped + // value is NEGATIVE, so it passes the ceiling check and sleeps for no time + // -- the retry loop then spins as fast as the provider can refuse. + wait := RetryBackoffBase + for i := 1; i < attempt && wait < RetryBackoffMax; i++ { + wait *= 2 + } + if wait > RetryBackoffMax { + return RetryBackoffMax + } + return wait +} + +// Sleep waits, or reports that the context ended first. +func Sleep(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/pkg/plugin/implementation/internal/common/token.go b/pkg/plugin/implementation/internal/common/token.go index 0dbb428a..b1213ed4 100644 --- a/pkg/plugin/implementation/internal/common/token.go +++ b/pkg/plugin/implementation/internal/common/token.go @@ -14,6 +14,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" ) // bearerToken returns a token to send, exchanging one if the held token has @@ -99,7 +100,7 @@ func (s *Step) exchangeToken(ctx context.Context, auth *authenticator) (string, // The status, not the body: a failure body routinely quotes the // request back. log.Warnf(ctx, "token endpoint %s returned %s: %s", - cfg.TokenURL, resp.Status, s.redactString(explain(body))) + cfg.TokenURL, resp.Status, s.redactString(httputil.Explain(body))) return "", 0, s.tokenErr(fmt.Errorf("token endpoint returned %s", resp.Status)) } From 9107a289cc7049e51747e223d480224ed063d187 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 22:25:32 +0530 Subject: [PATCH 118/130] refactor(common): one list of credential settings, not two [#7] Each of the ten setting names was written twice in auth.go -- once as a case in ParseProviderAuth's switch, once as a key in authFields -- and the two had to stay in sync by hand. The drift had an asymmetry worth naming. A setting in the switch but not in authFields is REFUSED at startup, which is loud and fine. A setting in authFields but not in the switch parsed, validated, and was then discarded: the operator configured a credential, startup accepted it, and the provider was called with nothing. Silent, and untested. authFields is now a map from name to the field it fills, so the name and the assignment are one entry: "clientSecretEnv": func(p *AuthProfile, v string) { p.ClientSecretEnv = v }, Adding a scheme is one line and cannot half-land. Membership checks read the map the same way, so it is still the vocabulary that makes the dash split decidable. Constants for the names were the other option. They would share the strings but leave both lists, so the sync problem stays -- this removes the second list instead. Two tests, one per property the map now guarantees: every setting reaches a field on the profile, found by reflection so a new field cannot be missed either, and no setting name carries a dash. Negative control: making clientSecretEnv set nothing fails the first with `authFields["clientSecretEnv"] set nothing on the profile`. 66 packages green, vet clean. --- .../implementation/internal/common/auth.go | 63 ++++++++----------- .../internal/common/common_test.go | 47 ++++++++++++++ 2 files changed, 72 insertions(+), 38 deletions(-) diff --git a/pkg/plugin/implementation/internal/common/auth.go b/pkg/plugin/implementation/internal/common/auth.go index b524b224..ecb89af1 100644 --- a/pkg/plugin/implementation/internal/common/auth.go +++ b/pkg/plugin/implementation/internal/common/auth.go @@ -132,14 +132,15 @@ func ParseProviderAuth(config map[string]string) (map[string]*AuthProfile, error // The old step-wide form. Refused rather than ignored: dropping it // silently leaves every provider with no credential, which reads // as the provider rejecting us. - if authFields[key] { + if _, isSetting := authFields[key]; isSetting { return nil, fmt.Errorf( "%q is set for the whole step; auth is per provider now, "+ "so it belongs in a block named for the participant id", key) } continue } - if !authFields[field] { + set, isSetting := authFields[field] + if !isSetting { // Nothing else on a provider step carries a dash, so this is a // misspelling rather than a setting to pass through. return nil, fmt.Errorf("%q is not a credential setting", key) @@ -153,46 +154,32 @@ func ParseProviderAuth(config map[string]string) (map[string]*AuthProfile, error profile = &AuthProfile{Provider: provider} profiles[provider] = profile } - value := config[key] - switch field { - case "authScheme": - profile.Scheme = value - case "usernameEnv": - profile.UsernameEnv = value - case "passwordEnv": - profile.PasswordEnv = value - case "headerName": - profile.HeaderName = value - case "headerValueEnv": - profile.HeaderValueEnv = value - case "queryName": - profile.QueryName = value - case "queryValueEnv": - profile.QueryValueEnv = value - case "tokenUrl": - profile.TokenURL = value - case "clientIdEnv": - profile.ClientIDEnv = value - case "clientSecretEnv": - profile.ClientSecretEnv = value - } + set(profile, config[key]) } return profiles, nil } -// authFields is the closed set of per-provider settings. Closed on purpose: it -// makes the dash split decidable and turns a misspelling into a startup error. -var authFields = map[string]bool{ - "authScheme": true, - "usernameEnv": true, - "passwordEnv": true, - "headerName": true, - "headerValueEnv": true, - "queryName": true, - "queryValueEnv": true, - "tokenUrl": true, - "clientIdEnv": true, - "clientSecretEnv": true, +// authFields maps each per-provider setting to the field it fills. +// +// ONE list, not two. The name and the assignment used to sit apart -- a switch +// in ParseProviderAuth and a set of valid names here -- and the two had to stay +// in sync by hand. Forgetting the switch was the silent direction: the setting +// parsed, validated, and was discarded, leaving a provider on a credential the +// operator thought they had configured. +// +// Being the whole vocabulary, this is also what makes the dash split decidable +// and turns a misspelled setting into a startup error. +var authFields = map[string]func(*AuthProfile, string){ + "authScheme": func(p *AuthProfile, v string) { p.Scheme = v }, + "usernameEnv": func(p *AuthProfile, v string) { p.UsernameEnv = v }, + "passwordEnv": func(p *AuthProfile, v string) { p.PasswordEnv = v }, + "headerName": func(p *AuthProfile, v string) { p.HeaderName = v }, + "headerValueEnv": func(p *AuthProfile, v string) { p.HeaderValueEnv = v }, + "queryName": func(p *AuthProfile, v string) { p.QueryName = v }, + "queryValueEnv": func(p *AuthProfile, v string) { p.QueryValueEnv = v }, + "tokenUrl": func(p *AuthProfile, v string) { p.TokenURL = v }, + "clientIdEnv": func(p *AuthProfile, v string) { p.ClientIDEnv = v }, + "clientSecretEnv": func(p *AuthProfile, v string) { p.ClientSecretEnv = v }, } // providerIDFrom returns the provider half of "|". diff --git a/pkg/plugin/implementation/internal/common/common_test.go b/pkg/plugin/implementation/internal/common/common_test.go index 2caee927..c44bda0d 100644 --- a/pkg/plugin/implementation/internal/common/common_test.go +++ b/pkg/plugin/implementation/internal/common/common_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "reflect" "slices" "sort" "strings" @@ -2209,3 +2210,49 @@ func TestOAuth2ReportsAMissingCredential(t *testing.T) { t.Error("the token endpoint was called with no credentials") } } + +// Every setting in authFields must actually reach a field on the profile. +// +// This is the check the old shape could not make. The name and the assignment +// lived apart -- a set of valid names, and a switch that filled the fields -- +// so a setting present in the set but missing from the switch parsed, validated +// and was then discarded, leaving a provider on a credential the operator +// thought they had configured. One map of setters makes that impossible by +// construction; this test pins it anyway, since the map is what a new scheme +// gets added to. +func TestEveryAuthFieldReachesTheProfile(t *testing.T) { + t.Parallel() + + const sentinel = "SENTINEL_VALUE" + for name, set := range authFields { + var profile AuthProfile + set(&profile, sentinel) + + // Reflect over the profile rather than naming the fields, so a field + // added to AuthProfile cannot be missed here either. + found := false + value := reflect.ValueOf(profile) + for i := 0; i < value.NumField(); i++ { + if value.Field(i).Kind() == reflect.String && + value.Field(i).String() == sentinel { + found = true + break + } + } + if !found { + t.Errorf("authFields[%q] set nothing on the profile", name) + } + } +} + +// The vocabulary is what makes the dash split decidable, so no setting name may +// contain a dash -- it would flatten to - ambiguously. +func TestNoAuthFieldNameCarriesADash(t *testing.T) { + t.Parallel() + + for name := range authFields { + if strings.Contains(name, "-") { + t.Errorf("setting %q carries a dash, which makes the split ambiguous", name) + } + } +} From bd40a7eec78ad2f12263be3b4fb22529302660aa Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 22:32:38 +0530 Subject: [PATCH 119/130] refactor(common): all sixteen constants in one file again [#7] The httputil move had split constant.go in two, which contradicted the one-constants-file decision. Only one direction could put them back together: common imports httputil, so a constant read by httputil cannot live in common without an import cycle. All sixteen are now in httputil/constant.go and common/constant.go is gone. Four that were package-private are now exported, since common reads them across the boundary: redactedMarker -> httputil.RedactedMarker codeUpstreamUnavailable -> httputil.CodeUpstreamUnavailable tokenRefreshSkew -> httputil.TokenRefreshSkew explainLimit -> httputil.ExplainLimit THE COST IS THE PACKAGE NAME. httputil now holds the auth schemes and the redaction marker, which are not HTTP plumbing, so its name undersells the contents. The file says so. One constants file that can be found beat two that were each in the right place -- a deliberate trade, not an oversight. All 16 verified present with identical values, compared name by name against both files before the merge. The three capability plugins' tests referenced common.AuthSchemeNone and now take it from httputil. 66 packages green, vet clean, no races. --- .../KnowledgeAdvisory/mappings_test.go | 3 +- .../MandiPrice/mappings_test.go | 7 +- .../WeatherObservation/mappings_test.go | 3 +- .../implementation/internal/common/auth.go | 19 ++--- .../implementation/internal/common/common.go | 3 +- .../internal/common/common_test.go | 80 +++++++++---------- .../internal/common/constant.go | 43 ---------- .../internal/common/dispatch_test.go | 3 +- .../implementation/internal/common/http.go | 2 +- .../internal/common/httputil/constant.go | 51 ++++++++++-- .../internal/common/httputil/explain.go | 4 +- .../internal/common/httputil/httputil_test.go | 6 +- .../internal/common/perprovider_test.go | 17 ++-- .../implementation/internal/common/redact.go | 12 +-- .../implementation/internal/common/token.go | 4 +- 15 files changed, 132 insertions(+), 125 deletions(-) delete mode 100644 pkg/plugin/implementation/internal/common/constant.go diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go index 9c06c18e..2df36246 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go @@ -23,6 +23,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/KnowledgeAdvisory" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" ) @@ -213,7 +214,7 @@ func runShippedWith(t *testing.T, request, providerBody string) (map[string]any, // Auth is per provider; these upstreams are stubs needing // no credential, and that is declared rather than defaulted. AuthByProvider: map[string]*common.AuthProfile{ - strings.Split(shippedBindingKey, "|")[0]: {Scheme: common.AuthSchemeNone}, + strings.Split(shippedBindingKey, "|")[0]: {Scheme: httputil.AuthSchemeNone}, }, }) if err != nil { diff --git a/pkg/plugin/implementation/MandiPrice/mappings_test.go b/pkg/plugin/implementation/MandiPrice/mappings_test.go index 75299068..0cd1618c 100644 --- a/pkg/plugin/implementation/MandiPrice/mappings_test.go +++ b/pkg/plugin/implementation/MandiPrice/mappings_test.go @@ -24,6 +24,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/MandiPrice" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" ) @@ -198,7 +199,7 @@ func runShippedWith(t *testing.T, request, providerBody string) (url.Values, map // Auth is per provider; these upstreams are stubs needing // no credential, and that is declared rather than defaulted. AuthByProvider: map[string]*common.AuthProfile{ - strings.Split(shippedBindingKey, "|")[0]: {Scheme: common.AuthSchemeNone}, + strings.Split(shippedBindingKey, "|")[0]: {Scheme: httputil.AuthSchemeNone}, }, }) if err != nil { @@ -445,7 +446,7 @@ func TestShippedMappingRefusesWhatItCannotServe(t *testing.T) { // Auth is per provider; these upstreams are stubs needing // no credential, and that is declared rather than defaulted. AuthByProvider: map[string]*common.AuthProfile{ - strings.Split(shippedBindingKey, "|")[0]: {Scheme: common.AuthSchemeNone}, + strings.Split(shippedBindingKey, "|")[0]: {Scheme: httputil.AuthSchemeNone}, }, }) if err != nil { @@ -761,7 +762,7 @@ func TestShippedMappingRefusesPayloadsItCannotAnswer(t *testing.T) { // Auth is per provider; these upstreams are stubs needing // no credential, and that is declared rather than defaulted. AuthByProvider: map[string]*common.AuthProfile{ - strings.Split(shippedBindingKey, "|")[0]: {Scheme: common.AuthSchemeNone}, + strings.Split(shippedBindingKey, "|")[0]: {Scheme: httputil.AuthSchemeNone}, }, }) if err != nil { diff --git a/pkg/plugin/implementation/WeatherObservation/mappings_test.go b/pkg/plugin/implementation/WeatherObservation/mappings_test.go index 0f98413f..23afb39a 100644 --- a/pkg/plugin/implementation/WeatherObservation/mappings_test.go +++ b/pkg/plugin/implementation/WeatherObservation/mappings_test.go @@ -24,6 +24,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/WeatherObservation" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" ) @@ -149,7 +150,7 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { // Auth is per provider; these upstreams are stubs needing // no credential, and that is declared rather than defaulted. AuthByProvider: map[string]*common.AuthProfile{ - strings.Split(shippedBindingKey, "|")[0]: {Scheme: common.AuthSchemeNone}, + strings.Split(shippedBindingKey, "|")[0]: {Scheme: httputil.AuthSchemeNone}, }, }) if err != nil { diff --git a/pkg/plugin/implementation/internal/common/auth.go b/pkg/plugin/implementation/internal/common/auth.go index ecb89af1..145d8cc2 100644 --- a/pkg/plugin/implementation/internal/common/auth.go +++ b/pkg/plugin/implementation/internal/common/auth.go @@ -13,6 +13,7 @@ import ( "sync/atomic" "github.com/beckn-one/beckn-onix/pkg/log" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" ) // AuthProfile is how ONE provider's credentials are presented upstream. @@ -68,23 +69,23 @@ type authenticator struct { // names the provider, since several profiles share one step. func (a *AuthProfile) validate() error { switch a.Scheme { - case AuthSchemeNone: - case AuthSchemeBasic: + case httputil.AuthSchemeNone: + case httputil.AuthSchemeBasic: if a.UsernameEnv == "" || a.PasswordEnv == "" { return fmt.Errorf( "%s: authScheme basic requires usernameEnv and passwordEnv", a.Provider) } - case AuthSchemeHeader: + case httputil.AuthSchemeHeader: if a.HeaderName == "" || a.HeaderValueEnv == "" { return fmt.Errorf( "%s: authScheme header requires headerName and headerValueEnv", a.Provider) } - case AuthSchemeQuery: + case httputil.AuthSchemeQuery: if a.QueryName == "" || a.QueryValueEnv == "" { return fmt.Errorf( "%s: authScheme query requires queryName and queryValueEnv", a.Provider) } - case AuthSchemeOAuth2: + case httputil.AuthSchemeOAuth2: if a.TokenURL == "" || a.ClientIDEnv == "" || a.ClientSecretEnv == "" { return fmt.Errorf( "%s: authScheme oauth2 requires tokenUrl, clientIdEnv and clientSecretEnv", @@ -207,20 +208,20 @@ func (s *Step) missingCredential(ctx context.Context, provider, scheme, envNames func (s *Step) authenticate(auth *authenticator, req *http.Request) error { cfg := auth.cfg switch cfg.Scheme { - case AuthSchemeBasic: + case httputil.AuthSchemeBasic: username, password := os.Getenv(cfg.UsernameEnv), os.Getenv(cfg.PasswordEnv) if username == "" || password == "" { return s.missingCredential(req.Context(), cfg.Provider, "basic", cfg.UsernameEnv+" and "+cfg.PasswordEnv) } req.SetBasicAuth(username, password) - case AuthSchemeHeader: + case httputil.AuthSchemeHeader: value := os.Getenv(cfg.HeaderValueEnv) if value == "" { return s.missingCredential(req.Context(), cfg.Provider, "header", cfg.HeaderValueEnv) } req.Header.Set(cfg.HeaderName, value) - case AuthSchemeQuery: + case httputil.AuthSchemeQuery: value := os.Getenv(cfg.QueryValueEnv) if value == "" { return s.missingCredential(req.Context(), cfg.Provider, "query", cfg.QueryValueEnv) @@ -230,7 +231,7 @@ func (s *Step) authenticate(auth *authenticator, req *http.Request) error { query := req.URL.Query() query.Set(cfg.QueryName, value) req.URL.RawQuery = query.Encode() - case AuthSchemeOAuth2: + case httputil.AuthSchemeOAuth2: token, err := s.bearerToken(req.Context(), auth) if err != nil { return err diff --git a/pkg/plugin/implementation/internal/common/common.go b/pkg/plugin/implementation/internal/common/common.go index f9f50f2f..96e1dfe8 100644 --- a/pkg/plugin/implementation/internal/common/common.go +++ b/pkg/plugin/implementation/internal/common/common.go @@ -23,6 +23,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" ) // Prerequisites are the values a capability needs that its payload does not @@ -169,7 +170,7 @@ func applyDefaults(cfg *Config) error { } } if cfg.MaxResponseBytes <= 0 { - cfg.MaxResponseBytes = DefaultMaxResponseBytes + cfg.MaxResponseBytes = httputil.DefaultMaxResponseBytes } if cfg.AuthByProvider == nil { cfg.AuthByProvider = map[string]*AuthProfile{} diff --git a/pkg/plugin/implementation/internal/common/common_test.go b/pkg/plugin/implementation/internal/common/common_test.go index c44bda0d..a46493f6 100644 --- a/pkg/plugin/implementation/internal/common/common_test.go +++ b/pkg/plugin/implementation/internal/common/common_test.go @@ -149,7 +149,7 @@ func newStep(t *testing.T, registry definition.ProviderRecordLookup, mapper defi // refused at startup. Filled in after the tweaks, because a tweak may // replace BindingKeys and the profile has to follow. if len(cfg.AuthByProvider) == 0 { - cfg.setProviderAuth(AuthProfile{Scheme: AuthSchemeNone}) + cfg.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeNone}) } step, closer, err := New(context.Background(), registry, mapper, nil, cfg) if err != nil { @@ -186,26 +186,26 @@ func TestNewValidatesTheAuthScheme(t *testing.T) { config *Config valid bool }{ - {"none, declared", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeNone})}, true}, + {"none, declared", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeNone})}, true}, {"no profile at all for a served provider", &Config{}, false}, {"an empty scheme", &Config{AuthByProvider: authForTestProvider(AuthProfile{})}, false}, - {"basic with both variables", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeBasic, UsernameEnv: "U", PasswordEnv: "P"})}, true}, - {"basic missing the password variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeBasic, UsernameEnv: "U"})}, false}, - {"basic missing the username variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeBasic, PasswordEnv: "P"})}, false}, - {"header with both settings", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeHeader, HeaderName: "X-Key", HeaderValueEnv: "V"})}, true}, - {"header missing the value variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeHeader, HeaderName: "X-Key"})}, false}, - {"query with both settings", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "t", QueryValueEnv: "V"})}, true}, + {"basic with both variables", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeBasic, UsernameEnv: "U", PasswordEnv: "P"})}, true}, + {"basic missing the password variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeBasic, UsernameEnv: "U"})}, false}, + {"basic missing the username variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeBasic, PasswordEnv: "P"})}, false}, + {"header with both settings", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeHeader, HeaderName: "X-Key", HeaderValueEnv: "V"})}, true}, + {"header missing the value variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeHeader, HeaderName: "X-Key"})}, false}, + {"query with both settings", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "t", QueryValueEnv: "V"})}, true}, {"an unknown scheme", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: "oauth"})}, false}, - {"oauth2 with all three settings", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeOAuth2, + {"oauth2 with all three settings", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeOAuth2, TokenURL: "https://issuer.invalid/token", ClientIDEnv: "ID", ClientSecretEnv: "SECRET"})}, true}, - {"oauth2 missing the token url", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeOAuth2, + {"oauth2 missing the token url", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeOAuth2, ClientIDEnv: "ID", ClientSecretEnv: "SECRET"})}, false}, - {"oauth2 missing the client id variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeOAuth2, + {"oauth2 missing the client id variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeOAuth2, TokenURL: "https://issuer.invalid/token", ClientSecretEnv: "SECRET"})}, false}, - {"oauth2 missing the client secret variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeOAuth2, + {"oauth2 missing the client secret variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeOAuth2, TokenURL: "https://issuer.invalid/token", ClientIDEnv: "ID"})}, false}, {"a profile for a provider not in bindingKeys", &Config{ - AuthByProvider: map[string]*AuthProfile{"someone-else": {Scheme: AuthSchemeNone}}}, false}, + AuthByProvider: map[string]*AuthProfile{"someone-else": {Scheme: httputil.AuthSchemeNone}}}, false}, } for _, tc := range testCases { @@ -339,7 +339,7 @@ func TestRunDoesNotRetryAMissingCredential(t *testing.T) { } mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} step := newStep(t, &stubRegistry{plan: plan}, mapper, func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_ABSENT_USER_FOR_RETRY", PasswordEnv: "TEST_ABSENT_PASS_FOR_RETRY"}) + c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeBasic, UsernameEnv: "TEST_ABSENT_USER_FOR_RETRY", PasswordEnv: "TEST_ABSENT_PASS_FOR_RETRY"}) }) _, err := runStep(t, step, selectBody) @@ -523,7 +523,7 @@ func TestRunRedactsACredentialEchoedInABody(t *testing.T) { mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper, func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_ECHO_TOKEN"}) + c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_ECHO_TOKEN"}) }) _, err := runStep(t, step, selectBody) @@ -567,7 +567,7 @@ func TestRunSendsTheCredentialAsAQueryParameter(t *testing.T) { mapper := &stubMapper{requestResult: []byte(`{"statecode":"CG"}`), responseResult: []byte(`{}`)} step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper, func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) + c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) }) if _, err := runStep(t, step, selectBody); err != nil { @@ -597,7 +597,7 @@ func TestRunRedactsAQueryCredentialFromAnError(t *testing.T) { } mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} step := newStep(t, &stubRegistry{plan: plan}, mapper, func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) + c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) }) _, err := runStep(t, step, selectBody) @@ -619,7 +619,7 @@ func TestRedactStringRemovesTheCredentialFromTheURL(t *testing.T) { // No t.Parallel: t.Setenv forbids it. t.Setenv("TEST_MANDI_TOKEN", "s3cr3t") - step := stepWithProviderAuth(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) + step := stepWithProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) got := step.redactString("http://host/v1/x?statecode=CG&token=s3cr3t") if strings.Contains(got, "s3cr3t") { t.Errorf("the credential survived redaction: %s", got) @@ -629,7 +629,7 @@ func TestRedactStringRemovesTheCredentialFromTheURL(t *testing.T) { } // Any other scheme has nothing to hide in a URL, so the text is untouched. - plain := stepWithProviderAuth(AuthProfile{Scheme: AuthSchemeNone}) + plain := stepWithProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeNone}) if out := plain.redactString("http://host/v1/x?statecode=CG"); out != "http://host/v1/x?statecode=CG" { t.Errorf("a url with no credential must pass through unchanged, got %q", out) } @@ -645,7 +645,7 @@ func TestRedactStringRemovesThePercentEncodedCredential(t *testing.T) { const token = "a+b/c=d e" // every character Encode treats specially t.Setenv("TEST_MANDI_TOKEN", token) - step := stepWithProviderAuth(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) + step := stepWithProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) // Exactly how the credential appears once authenticate has run: Encode // escapes it, so this is the string a transport error quotes. @@ -671,7 +671,7 @@ func TestRedactStringStillRemovesAnUnescapedCredential(t *testing.T) { // No t.Parallel: t.Setenv forbids it. t.Setenv("TEST_MANDI_TOKEN", "plaintoken123") - step := stepWithProviderAuth(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) + step := stepWithProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) got := step.redactString("http://host/v1/x?token=plaintoken123") if strings.Contains(got, "plaintoken123") { t.Errorf("the credential survived redaction: %s", got) @@ -688,9 +688,9 @@ func TestNewRefusesAHalfConfiguredQueryScheme(t *testing.T) { cfg *Config }{ {"no queryName", &Config{BindingKeys: []string{testBindingKey}, - AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeQuery, QueryValueEnv: "TEST_MANDI_TOKEN"})}}, + AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryValueEnv: "TEST_MANDI_TOKEN"})}}, {"no queryValueEnv", &Config{BindingKeys: []string{testBindingKey}, - AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token"})}}, + AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token"})}}, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -1101,7 +1101,7 @@ func TestNewRefusesAHalfConfiguredOverride(t *testing.T) { _, _, err := New(context.Background(), &stubRegistry{}, &stubMapper{}, nil, &Config{ BindingKeys: []string{testBindingKey}, ProviderIDAt: "who.provider", - AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeNone}), + AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeNone}), }) if err == nil { t.Fatal("expected one path without the other to be refused") @@ -1350,7 +1350,7 @@ func TestRunPresentsBasicCredentialsFromTheEnvironment(t *testing.T) { step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_MAUSAMGRAM_USER", PasswordEnv: "TEST_MAUSAMGRAM_KEY"}) + c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeBasic, UsernameEnv: "TEST_MAUSAMGRAM_USER", PasswordEnv: "TEST_MAUSAMGRAM_KEY"}) }) if _, err := runStep(t, step, selectBody); err != nil { @@ -1374,7 +1374,7 @@ func TestRunPresentsAHeaderCredentialFromTheEnvironment(t *testing.T) { step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: AuthSchemeHeader, HeaderName: "X-Api-Key", HeaderValueEnv: "TEST_MAUSAMGRAM_TOKEN"}) + c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeHeader, HeaderName: "X-Api-Key", HeaderValueEnv: "TEST_MAUSAMGRAM_TOKEN"}) }) if _, err := runStep(t, step, selectBody); err != nil { @@ -1396,7 +1396,7 @@ func TestRunFailsWhenAConfiguredCredentialIsAbsent(t *testing.T) { step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_MAUSAMGRAM_ABSENT_USER", PasswordEnv: "TEST_MAUSAMGRAM_ABSENT_KEY"}) + c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeBasic, UsernameEnv: "TEST_MAUSAMGRAM_ABSENT_USER", PasswordEnv: "TEST_MAUSAMGRAM_ABSENT_KEY"}) }) if _, err := runStep(t, step, selectBody); err == nil { @@ -1595,7 +1595,7 @@ func TestRedactKeepsTheErrorChainMatchable(t *testing.T) { // No t.Parallel: t.Setenv forbids it. t.Setenv("TEST_CHAIN_TOKEN", "s3cr3t") - step := stepWithProviderAuth(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_CHAIN_TOKEN"}) + step := stepWithProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_CHAIN_TOKEN"}) // The shape net/http produces: the cause wrapped behind text that quotes // the whole URL, credential and all. @@ -1629,7 +1629,7 @@ func TestRedactLeavesAnUnchangedErrorAlone(t *testing.T) { // No t.Parallel: t.Setenv forbids it. t.Setenv("TEST_CHAIN_TOKEN_2", "s3cr3t") - step := stepWithProviderAuth(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_CHAIN_TOKEN_2"}) + step := stepWithProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_CHAIN_TOKEN_2"}) original := errors.New("nothing sensitive here") if got := step.redact(original); got != original { t.Errorf("redact returned a different error for text it did not change: %v", got) @@ -1718,7 +1718,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { { name: "basic, the wire form a gateway echoes", tweak: func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_USER", PasswordEnv: "TEST_PASS"}) + c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeBasic, UsernameEnv: "TEST_USER", PasswordEnv: "TEST_PASS"}) }, body: `{"error":"invalid Authorization: Basic ` + wire + `"}`, secret: wire, @@ -1726,7 +1726,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { { name: "basic, the password quoted raw", tweak: func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_USER", PasswordEnv: "TEST_PASS"}) + c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeBasic, UsernameEnv: "TEST_USER", PasswordEnv: "TEST_PASS"}) }, body: `{"error":"bad password s3cr3t"}`, secret: "s3cr3t", @@ -1734,7 +1734,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { { name: "header, the value as sent", tweak: func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: AuthSchemeHeader, HeaderName: "X-API-Key", HeaderValueEnv: "TEST_HDR"}) + c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeHeader, HeaderName: "X-API-Key", HeaderValueEnv: "TEST_HDR"}) }, body: `{"error":"bad X-API-Key: hdr-k3y"}`, secret: "hdr-k3y", @@ -1742,7 +1742,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { { name: "query, still covered, raw form", tweak: func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_QRY"}) + c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_QRY"}) }, body: `{"rejected":"token=a+b/c="}`, secret: "a+b/c=", @@ -1750,7 +1750,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { { name: "query, still covered, percent-encoded form", tweak: func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_QRY"}) + c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_QRY"}) }, body: `{"rejected":"token=` + url.QueryEscape("a+b/c=") + `"}`, secret: url.QueryEscape("a+b/c="), @@ -1766,7 +1766,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { if strings.Contains(logged, tc.secret) { t.Errorf("the credential survives into the log line: %s", logged) } - if !strings.Contains(logged, redactedMarker) { + if !strings.Contains(logged, httputil.RedactedMarker) { t.Errorf("logged = %q, want the credential replaced", logged) } }) @@ -1784,7 +1784,7 @@ func TestRedactStringLeavesTheBasicUsername(t *testing.T) { step := newStep(t, &stubRegistry{plan: testPlan("http://provider.invalid", http.MethodGet)}, &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: AuthSchemeBasic, UsernameEnv: "TEST_USER", PasswordEnv: "TEST_PASS"}) + c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeBasic, UsernameEnv: "TEST_USER", PasswordEnv: "TEST_PASS"}) }) got := step.redactString(`user mausam failed to authenticate with s3cr3t`) @@ -1799,7 +1799,7 @@ func TestRedactStringLeavesTheBasicUsername(t *testing.T) { // Nothing configured, nothing to hide: an unset credential must not turn every // empty string in the text into a redaction marker. func TestRedactStringWithNoCredentialConfigured(t *testing.T) { - for _, scheme := range []string{AuthSchemeNone, AuthSchemeBasic, AuthSchemeHeader, AuthSchemeQuery} { + for _, scheme := range []string{httputil.AuthSchemeNone, httputil.AuthSchemeBasic, httputil.AuthSchemeHeader, httputil.AuthSchemeQuery} { t.Run(scheme, func(t *testing.T) { const text = `{"error":"provider said no"}` step := newStep(t, &stubRegistry{plan: testPlan("http://provider.invalid", http.MethodGet)}, @@ -1847,7 +1847,7 @@ func TestRunHandsResolvedPrerequisitesToTheMappingAsLocal(t *testing.T) { step, closer, err := New(context.Background(), &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper, prerequisites, &Config{BindingKeys: []string{testBindingKey}, - AuthByProvider: authForTestProvider(AuthProfile{Scheme: AuthSchemeNone})}) + AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeNone})}) if err != nil { t.Fatalf("New() returned an unexpected error: %v", err) } @@ -2014,7 +2014,7 @@ func oauth2Step(t *testing.T, tokenURL string, seen *[]string) *Step { return newStep(t, &stubRegistry{plan: testPlan(provider.URL, http.MethodGet)}, &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{"a":1}`)}, func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: AuthSchemeOAuth2, TokenURL: tokenURL, ClientIDEnv: "TEST_OAUTH_ID", ClientSecretEnv: "TEST_OAUTH_SECRET"}) + c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeOAuth2, TokenURL: tokenURL, ClientIDEnv: "TEST_OAUTH_ID", ClientSecretEnv: "TEST_OAUTH_SECRET"}) }) } @@ -2179,7 +2179,7 @@ func TestOAuth2RedactsTheSecretAndTheToken(t *testing.T) { if strings.Contains(got, secret) { t.Errorf("%q survived redaction: %s", secret, got) } - if !strings.Contains(got, redactedMarker) { + if !strings.Contains(got, httputil.RedactedMarker) { t.Errorf("nothing was redacted from %q", got) } } diff --git a/pkg/plugin/implementation/internal/common/constant.go b/pkg/plugin/implementation/internal/common/constant.go deleted file mode 100644 index d86e18fe..00000000 --- a/pkg/plugin/implementation/internal/common/constant.go +++ /dev/null @@ -1,43 +0,0 @@ -// Constants only. authFields is not here: it is a var, since Go has no constant -// maps, and it sits beside the parser in auth.go that reads it. -package common - -import ( - "time" -) - -// DefaultMaxResponseBytes caps what is read from a provider. The response is -// mapped in memory, so an unbounded one is an unbounded allocation. -const DefaultMaxResponseBytes = 4 << 20 // 4 MiB - -// redactedMarker stands in for a credential in anything logged or returned. -const redactedMarker = "REDACTED" - -// Auth schemes a step can present. Credentials are never in config or in the -// registry: config names the environment variable to read. -const ( - AuthSchemeNone = "none" - AuthSchemeBasic = "basic" - AuthSchemeHeader = "header" - // The credential goes in the query string, which some upstreams require. - // The least safe scheme -- proxies log query strings and transport errors - // quote them -- so the value is redacted. See redact. - AuthSchemeQuery = "query" - - // A client id and secret are exchanged for a short-lived bearer token. For - // an upstream where a static token is not an option: the live one this was - // written against expires every ten hours. - // - // Only client_credentials. That grant has no user to redirect and no - // refresh token, so the other flows would be dead code. - AuthSchemeOAuth2 = "oauth2" -) - -// codeUpstreamUnavailable: the provider could not be reached or failed. Not -// this adapter's fault and not the caller's. -const codeUpstreamUnavailable = "NET_DOWNSTREAM_UNAVAILABLE" - -// How early an oauth2 token stops being trusted. Must exceed the round trip to -// the provider, so a request that passed the expiry check cannot arrive after -// the token died. Costs one extra exchange per lifetime. -const tokenRefreshSkew = 60 * time.Second diff --git a/pkg/plugin/implementation/internal/common/dispatch_test.go b/pkg/plugin/implementation/internal/common/dispatch_test.go index d2eea389..230265f3 100644 --- a/pkg/plugin/implementation/internal/common/dispatch_test.go +++ b/pkg/plugin/implementation/internal/common/dispatch_test.go @@ -11,6 +11,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" ) // dispatchMappingRef stands in for the one reference an action carries. This @@ -84,7 +85,7 @@ func TestTwoProviderStepsDispatchByBindingKey(t *testing.T) { // Auth is per provider, so each step declares its own even // when neither upstream needs a credential. AuthByProvider: map[string]*common.AuthProfile{ - strings.Split(bindingKey, "|")[0]: {Scheme: common.AuthSchemeNone}, + strings.Split(bindingKey, "|")[0]: {Scheme: httputil.AuthSchemeNone}, }, }) if err != nil { diff --git a/pkg/plugin/implementation/internal/common/http.go b/pkg/plugin/implementation/internal/common/http.go index a795cd0f..6742431b 100644 --- a/pkg/plugin/implementation/internal/common/http.go +++ b/pkg/plugin/implementation/internal/common/http.go @@ -61,7 +61,7 @@ func (s *Step) call(ctx context.Context, auth *authenticator, baseURL string, ca } } } - return nil, model.NewCodedErr(http.StatusBadGateway, codeUpstreamUnavailable, + return nil, model.NewCodedErr(http.StatusBadGateway, httputil.CodeUpstreamUnavailable, fmt.Errorf("provider did not answer after %d attempts: %w", attempts, lastErr)) } diff --git a/pkg/plugin/implementation/internal/common/httputil/constant.go b/pkg/plugin/implementation/internal/common/httputil/constant.go index 1493ea0f..c4e95158 100644 --- a/pkg/plugin/implementation/internal/common/httputil/constant.go +++ b/pkg/plugin/implementation/internal/common/httputil/constant.go @@ -1,8 +1,13 @@ -// Constants for the call and its retry budget. +// Every constant this machinery uses, in one file. // -// Here rather than in common's constant.go because the code that reads them -// is here, and common imports this package -- so a shared file would be an -// import cycle. +// Here rather than in common because common imports this package: a constant +// read by anything under httputil could not live in common without an import +// cycle, and splitting them across two files put the auth schemes in one place +// and the retry budget in another. +// +// The consequence is that the package name undersells the contents -- the auth +// schemes and the redaction marker are not about HTTP plumbing. One file that +// can be found beats two that are each in the right place. package httputil import ( @@ -37,4 +42,40 @@ const ( // How much of a failed response is quoted: enough for the provider's message, // not a page of HTML in a log line. -const explainLimit = 300 +const ExplainLimit = 300 + +// DefaultMaxResponseBytes caps what is read from a provider. The response is +// mapped in memory, so an unbounded one is an unbounded allocation. +const DefaultMaxResponseBytes = 4 << 20 // 4 MiB + +// RedactedMarker stands in for a credential in anything logged or returned. +const RedactedMarker = "REDACTED" + +// Auth schemes a step can present. Credentials are never in config or in the +// registry: config names the environment variable to read. +const ( + AuthSchemeNone = "none" + AuthSchemeBasic = "basic" + AuthSchemeHeader = "header" + // The credential goes in the query string, which some upstreams require. + // The least safe scheme -- proxies log query strings and transport errors + // quote them -- so the value is redacted. See redact. + AuthSchemeQuery = "query" + + // A client id and secret are exchanged for a short-lived bearer token. For + // an upstream where a static token is not an option: the live one this was + // written against expires every ten hours. + // + // Only client_credentials. That grant has no user to redirect and no + // refresh token, so the other flows would be dead code. + AuthSchemeOAuth2 = "oauth2" +) + +// CodeUpstreamUnavailable: the provider could not be reached or failed. Not +// this adapter's fault and not the caller's. +const CodeUpstreamUnavailable = "NET_DOWNSTREAM_UNAVAILABLE" + +// How early an oauth2 token stops being trusted. Must exceed the round trip to +// the provider, so a request that passed the expiry check cannot arrive after +// the token died. Costs one extra exchange per lifetime. +const TokenRefreshSkew = 60 * time.Second diff --git a/pkg/plugin/implementation/internal/common/httputil/explain.go b/pkg/plugin/implementation/internal/common/httputil/explain.go index 839d602d..d65884e0 100644 --- a/pkg/plugin/implementation/internal/common/httputil/explain.go +++ b/pkg/plugin/implementation/internal/common/httputil/explain.go @@ -15,8 +15,8 @@ func Explain(body []byte) string { // Collapse whitespace: indented JSON or an HTML error page should not // spread one failure over forty log lines. text = strings.Join(strings.Fields(text), " ") - if len(text) > explainLimit { - return text[:explainLimit] + "... (truncated)" + if len(text) > ExplainLimit { + return text[:ExplainLimit] + "... (truncated)" } return text } diff --git a/pkg/plugin/implementation/internal/common/httputil/httputil_test.go b/pkg/plugin/implementation/internal/common/httputil/httputil_test.go index 9d4b2306..a77c2dfb 100644 --- a/pkg/plugin/implementation/internal/common/httputil/httputil_test.go +++ b/pkg/plugin/implementation/internal/common/httputil/httputil_test.go @@ -32,9 +32,9 @@ func TestExplainTruncatesAndHandlesAnEmptyBody(t *testing.T) { if got := Explain(nil); got != "(no body)" { t.Errorf("Explain(nil) = %q, want a marker rather than an empty string", got) } - long := Explain([]byte(strings.Repeat("x", explainLimit+50))) - if len(long) > explainLimit+len("... (truncated)") { - t.Errorf("explain kept %d characters, want it truncated near %d", len(long), explainLimit) + long := Explain([]byte(strings.Repeat("x", ExplainLimit+50))) + if len(long) > ExplainLimit+len("... (truncated)") { + t.Errorf("explain kept %d characters, want it truncated near %d", len(long), ExplainLimit) } if !strings.HasSuffix(long, "(truncated)") { t.Errorf("a truncated body should say so, got %q", long[len(long)-20:]) diff --git a/pkg/plugin/implementation/internal/common/perprovider_test.go b/pkg/plugin/implementation/internal/common/perprovider_test.go index 8f32b6da..865752d4 100644 --- a/pkg/plugin/implementation/internal/common/perprovider_test.go +++ b/pkg/plugin/implementation/internal/common/perprovider_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" ) // bodyFor builds a select naming a particular provider, so one step serving @@ -80,8 +81,8 @@ func TestTwoProvidersOnOneStepAuthenticateDifferently(t *testing.T) { cfg := &Config{ BindingKeys: []string{"alpha|" + capability, "beta|" + capability}, AuthByProvider: map[string]*AuthProfile{ - "alpha": {Scheme: AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_PP_QUERY_TOKEN"}, - "beta": {Scheme: AuthSchemeHeader, HeaderName: "X-Api-Key", HeaderValueEnv: "TEST_PP_HEADER_TOKEN"}, + "alpha": {Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_PP_QUERY_TOKEN"}, + "beta": {Scheme: httputil.AuthSchemeHeader, HeaderName: "X-Api-Key", HeaderValueEnv: "TEST_PP_HEADER_TOKEN"}, }, } registry := &routingRegistry{plans: map[string]*model.ProviderRecord{ @@ -155,9 +156,9 @@ func TestTwoOAuth2ProvidersDoNotShareAToken(t *testing.T) { cfg := &Config{ BindingKeys: []string{"alpha|" + capability, "beta|" + capability}, AuthByProvider: map[string]*AuthProfile{ - "alpha": {Scheme: AuthSchemeOAuth2, TokenURL: issuerA.URL, + "alpha": {Scheme: httputil.AuthSchemeOAuth2, TokenURL: issuerA.URL, ClientIDEnv: "TEST_PP_ID_A", ClientSecretEnv: "TEST_PP_SECRET_A"}, - "beta": {Scheme: AuthSchemeOAuth2, TokenURL: issuerB.URL, + "beta": {Scheme: httputil.AuthSchemeOAuth2, TokenURL: issuerB.URL, ClientIDEnv: "TEST_PP_ID_B", ClientSecretEnv: "TEST_PP_SECRET_B"}, }, } @@ -214,8 +215,8 @@ func TestRedactionCoversEveryProvidersSecret(t *testing.T) { cfg := &Config{ BindingKeys: []string{"alpha|" + capability, "beta|" + capability}, AuthByProvider: map[string]*AuthProfile{ - "alpha": {Scheme: AuthSchemeHeader, HeaderName: "X-A", HeaderValueEnv: "TEST_PP_RED_A"}, - "beta": {Scheme: AuthSchemeHeader, HeaderName: "X-B", HeaderValueEnv: "TEST_PP_RED_B"}, + "alpha": {Scheme: httputil.AuthSchemeHeader, HeaderName: "X-A", HeaderValueEnv: "TEST_PP_RED_A"}, + "beta": {Scheme: httputil.AuthSchemeHeader, HeaderName: "X-B", HeaderValueEnv: "TEST_PP_RED_B"}, }, } step, closer, err := New(context.Background(), &routingRegistry{}, &stubMapper{}, nil, cfg) @@ -247,8 +248,8 @@ func TestRedactionSortsAcrossProvidersNotWithinOne(t *testing.T) { cfg := &Config{ BindingKeys: []string{"shorty|" + capability, "longy|" + capability}, AuthByProvider: map[string]*AuthProfile{ - "shorty": {Scheme: AuthSchemeHeader, HeaderName: "X-S", HeaderValueEnv: "TEST_PP_SHORT"}, - "longy": {Scheme: AuthSchemeHeader, HeaderName: "X-L", HeaderValueEnv: "TEST_PP_LONG"}, + "shorty": {Scheme: httputil.AuthSchemeHeader, HeaderName: "X-S", HeaderValueEnv: "TEST_PP_SHORT"}, + "longy": {Scheme: httputil.AuthSchemeHeader, HeaderName: "X-L", HeaderValueEnv: "TEST_PP_LONG"}, }, } step, closer, err := New(context.Background(), &routingRegistry{}, &stubMapper{}, nil, cfg) diff --git a/pkg/plugin/implementation/internal/common/redact.go b/pkg/plugin/implementation/internal/common/redact.go index fded2101..c4121652 100644 --- a/pkg/plugin/implementation/internal/common/redact.go +++ b/pkg/plugin/implementation/internal/common/redact.go @@ -7,6 +7,8 @@ import ( "os" "sort" "strings" + + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" ) // redact removes a credential from an error's text. @@ -48,7 +50,7 @@ func (e redactedErr) Unwrap() error { return e.err } // so a credential reaches a message under any scheme. func (s *Step) redactString(text string) string { for _, secret := range s.secretForms() { - text = strings.ReplaceAll(text, secret, redactedMarker) + text = strings.ReplaceAll(text, secret, httputil.RedactedMarker) } return text } @@ -81,7 +83,7 @@ func (s *Step) secretForms() []string { // request. func (a *authenticator) secretForms() []string { switch a.cfg.Scheme { - case AuthSchemeBasic: + case httputil.AuthSchemeBasic: username, password := os.Getenv(a.cfg.UsernameEnv), os.Getenv(a.cfg.PasswordEnv) if password == "" { return nil @@ -96,13 +98,13 @@ func (a *authenticator) secretForms() []string { // authenticates, and words like "user" or "admin" would eat // unrelated text. return forms - case AuthSchemeHeader: + case httputil.AuthSchemeHeader: value := os.Getenv(a.cfg.HeaderValueEnv) if value == "" { return nil } return []string{value} - case AuthSchemeOAuth2: + case httputil.AuthSchemeOAuth2: // Both halves: the secret we send the issuer, and the token it gave // back. The token is what reaches the provider. var forms []string @@ -117,7 +119,7 @@ func (a *authenticator) secretForms() []string { // The client id is NOT redacted: it identifies, it does not // authenticate. return forms - case AuthSchemeQuery: + case httputil.AuthSchemeQuery: value := os.Getenv(a.cfg.QueryValueEnv) if value == "" { return nil diff --git a/pkg/plugin/implementation/internal/common/token.go b/pkg/plugin/implementation/internal/common/token.go index b1213ed4..c61cc10d 100644 --- a/pkg/plugin/implementation/internal/common/token.go +++ b/pkg/plugin/implementation/internal/common/token.go @@ -113,7 +113,7 @@ func (s *Step) exchangeToken(ctx context.Context, auth *authenticator) (string, return "", 0, s.tokenErr(fmt.Errorf("token response from %s carries no access_token", cfg.TokenURL)) } - lifetime, ok := tokenLifetime(parsed.ExpiresIn, tokenRefreshSkew) + lifetime, ok := tokenLifetime(parsed.ExpiresIn, httputil.TokenRefreshSkew) if !ok { return "", 0, s.tokenErr(fmt.Errorf( "token response from %s carries no usable expires_in, so its lifetime is unknown", @@ -125,7 +125,7 @@ func (s *Step) exchangeToken(ctx context.Context, auth *authenticator) (string, // tokenErr classifies a failed exchange. Always 502: the failure is the // issuer's, never the caller's. func (s *Step) tokenErr(err error) error { - return model.NewCodedErr(http.StatusBadGateway, codeUpstreamUnavailable, + return model.NewCodedErr(http.StatusBadGateway, httputil.CodeUpstreamUnavailable, fmt.Errorf("oauth2 token exchange failed: %w", err)) } From ec3649e24237d74fb98259ae33ebb13421803559 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 22:42:46 +0530 Subject: [PATCH 120/130] refactor(common): rename httputil to util [#7] The package stopped being about HTTP when all sixteen constants moved into it: it holds AuthSchemeOAuth2 and RedactedMarker alongside the retry budget, and "httputil" told a reader those were not there. util is plainer and does not promise a subject the contents do not have. 11 files, 93 references, 19 symbols. `util` collides with nothing in the repo. common/util/constant.go every constant, in one file common/util/retry.go Budget, Backoff, Sleep, DoNotRetry, IsPermanent common/util/endpoint.go BuildEndpoint, verifyPath, verifyBaseURL common/util/query.go asQuery, asQueryValue common/util/request.go RequestBody, HasBody, CanonicalMethod common/util/explain.go Explain The package now has a real doc comment, on retry.go, saying what it is and what it is not: a helper belongs beside the code that uses it unless a receiver forbids it. A package called util invites the opposite, so it is written down. CAUGHT BY READING THE DIFF, not by the compiler: the blanket rename rewrote two comments in core/module/handler/stdHandler.go that refer to Go's OWN net/http/httputil -- ReverseProxy and BufferPool. Reverted; that file is untouched and its four httputil references are the standard library's. 66 packages green, vet clean, no races. --- .../KnowledgeAdvisory/mappings_test.go | 4 +- .../MandiPrice/mappings_test.go | 8 +- .../WeatherObservation/mappings_test.go | 4 +- .../implementation/internal/common/auth.go | 20 ++--- .../implementation/internal/common/common.go | 4 +- .../internal/common/common_test.go | 88 +++++++++---------- .../internal/common/dispatch_test.go | 4 +- .../implementation/internal/common/http.go | 30 +++---- .../internal/common/perprovider_test.go | 18 ++-- .../implementation/internal/common/redact.go | 12 +-- .../implementation/internal/common/token.go | 8 +- .../common/{httputil => util}/constant.go | 14 +-- .../common/{httputil => util}/endpoint.go | 2 +- .../common/{httputil => util}/explain.go | 2 +- .../common/{httputil => util}/query.go | 2 +- .../common/{httputil => util}/request.go | 2 +- .../common/{httputil => util}/retry.go | 14 ++- .../httputil_test.go => util/util_test.go} | 2 +- 18 files changed, 120 insertions(+), 118 deletions(-) rename pkg/plugin/implementation/internal/common/{httputil => util}/constant.go (85%) rename pkg/plugin/implementation/internal/common/{httputil => util}/endpoint.go (99%) rename pkg/plugin/implementation/internal/common/{httputil => util}/explain.go (97%) rename pkg/plugin/implementation/internal/common/{httputil => util}/query.go (99%) rename pkg/plugin/implementation/internal/common/{httputil => util}/request.go (98%) rename pkg/plugin/implementation/internal/common/{httputil => util}/retry.go (80%) rename pkg/plugin/implementation/internal/common/{httputil/httputil_test.go => util/util_test.go} (99%) diff --git a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go index 2df36246..a31c8eb3 100644 --- a/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go +++ b/pkg/plugin/implementation/KnowledgeAdvisory/mappings_test.go @@ -23,7 +23,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/KnowledgeAdvisory" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/util" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" ) @@ -214,7 +214,7 @@ func runShippedWith(t *testing.T, request, providerBody string) (map[string]any, // Auth is per provider; these upstreams are stubs needing // no credential, and that is declared rather than defaulted. AuthByProvider: map[string]*common.AuthProfile{ - strings.Split(shippedBindingKey, "|")[0]: {Scheme: httputil.AuthSchemeNone}, + strings.Split(shippedBindingKey, "|")[0]: {Scheme: util.AuthSchemeNone}, }, }) if err != nil { diff --git a/pkg/plugin/implementation/MandiPrice/mappings_test.go b/pkg/plugin/implementation/MandiPrice/mappings_test.go index 0cd1618c..a13dfbf1 100644 --- a/pkg/plugin/implementation/MandiPrice/mappings_test.go +++ b/pkg/plugin/implementation/MandiPrice/mappings_test.go @@ -24,7 +24,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/MandiPrice" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/util" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" ) @@ -199,7 +199,7 @@ func runShippedWith(t *testing.T, request, providerBody string) (url.Values, map // Auth is per provider; these upstreams are stubs needing // no credential, and that is declared rather than defaulted. AuthByProvider: map[string]*common.AuthProfile{ - strings.Split(shippedBindingKey, "|")[0]: {Scheme: httputil.AuthSchemeNone}, + strings.Split(shippedBindingKey, "|")[0]: {Scheme: util.AuthSchemeNone}, }, }) if err != nil { @@ -446,7 +446,7 @@ func TestShippedMappingRefusesWhatItCannotServe(t *testing.T) { // Auth is per provider; these upstreams are stubs needing // no credential, and that is declared rather than defaulted. AuthByProvider: map[string]*common.AuthProfile{ - strings.Split(shippedBindingKey, "|")[0]: {Scheme: httputil.AuthSchemeNone}, + strings.Split(shippedBindingKey, "|")[0]: {Scheme: util.AuthSchemeNone}, }, }) if err != nil { @@ -762,7 +762,7 @@ func TestShippedMappingRefusesPayloadsItCannotAnswer(t *testing.T) { // Auth is per provider; these upstreams are stubs needing // no credential, and that is declared rather than defaulted. AuthByProvider: map[string]*common.AuthProfile{ - strings.Split(shippedBindingKey, "|")[0]: {Scheme: httputil.AuthSchemeNone}, + strings.Split(shippedBindingKey, "|")[0]: {Scheme: util.AuthSchemeNone}, }, }) if err != nil { diff --git a/pkg/plugin/implementation/WeatherObservation/mappings_test.go b/pkg/plugin/implementation/WeatherObservation/mappings_test.go index 23afb39a..1101aa83 100644 --- a/pkg/plugin/implementation/WeatherObservation/mappings_test.go +++ b/pkg/plugin/implementation/WeatherObservation/mappings_test.go @@ -24,7 +24,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/WeatherObservation" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/util" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" ) @@ -150,7 +150,7 @@ func TestShippedMappingsServeARealSelect(t *testing.T) { // Auth is per provider; these upstreams are stubs needing // no credential, and that is declared rather than defaulted. AuthByProvider: map[string]*common.AuthProfile{ - strings.Split(shippedBindingKey, "|")[0]: {Scheme: httputil.AuthSchemeNone}, + strings.Split(shippedBindingKey, "|")[0]: {Scheme: util.AuthSchemeNone}, }, }) if err != nil { diff --git a/pkg/plugin/implementation/internal/common/auth.go b/pkg/plugin/implementation/internal/common/auth.go index 145d8cc2..c013a9f3 100644 --- a/pkg/plugin/implementation/internal/common/auth.go +++ b/pkg/plugin/implementation/internal/common/auth.go @@ -13,7 +13,7 @@ import ( "sync/atomic" "github.com/beckn-one/beckn-onix/pkg/log" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/util" ) // AuthProfile is how ONE provider's credentials are presented upstream. @@ -69,23 +69,23 @@ type authenticator struct { // names the provider, since several profiles share one step. func (a *AuthProfile) validate() error { switch a.Scheme { - case httputil.AuthSchemeNone: - case httputil.AuthSchemeBasic: + case util.AuthSchemeNone: + case util.AuthSchemeBasic: if a.UsernameEnv == "" || a.PasswordEnv == "" { return fmt.Errorf( "%s: authScheme basic requires usernameEnv and passwordEnv", a.Provider) } - case httputil.AuthSchemeHeader: + case util.AuthSchemeHeader: if a.HeaderName == "" || a.HeaderValueEnv == "" { return fmt.Errorf( "%s: authScheme header requires headerName and headerValueEnv", a.Provider) } - case httputil.AuthSchemeQuery: + case util.AuthSchemeQuery: if a.QueryName == "" || a.QueryValueEnv == "" { return fmt.Errorf( "%s: authScheme query requires queryName and queryValueEnv", a.Provider) } - case httputil.AuthSchemeOAuth2: + case util.AuthSchemeOAuth2: if a.TokenURL == "" || a.ClientIDEnv == "" || a.ClientSecretEnv == "" { return fmt.Errorf( "%s: authScheme oauth2 requires tokenUrl, clientIdEnv and clientSecretEnv", @@ -208,20 +208,20 @@ func (s *Step) missingCredential(ctx context.Context, provider, scheme, envNames func (s *Step) authenticate(auth *authenticator, req *http.Request) error { cfg := auth.cfg switch cfg.Scheme { - case httputil.AuthSchemeBasic: + case util.AuthSchemeBasic: username, password := os.Getenv(cfg.UsernameEnv), os.Getenv(cfg.PasswordEnv) if username == "" || password == "" { return s.missingCredential(req.Context(), cfg.Provider, "basic", cfg.UsernameEnv+" and "+cfg.PasswordEnv) } req.SetBasicAuth(username, password) - case httputil.AuthSchemeHeader: + case util.AuthSchemeHeader: value := os.Getenv(cfg.HeaderValueEnv) if value == "" { return s.missingCredential(req.Context(), cfg.Provider, "header", cfg.HeaderValueEnv) } req.Header.Set(cfg.HeaderName, value) - case httputil.AuthSchemeQuery: + case util.AuthSchemeQuery: value := os.Getenv(cfg.QueryValueEnv) if value == "" { return s.missingCredential(req.Context(), cfg.Provider, "query", cfg.QueryValueEnv) @@ -231,7 +231,7 @@ func (s *Step) authenticate(auth *authenticator, req *http.Request) error { query := req.URL.Query() query.Set(cfg.QueryName, value) req.URL.RawQuery = query.Encode() - case httputil.AuthSchemeOAuth2: + case util.AuthSchemeOAuth2: token, err := s.bearerToken(req.Context(), auth) if err != nil { return err diff --git a/pkg/plugin/implementation/internal/common/common.go b/pkg/plugin/implementation/internal/common/common.go index 96e1dfe8..bbf7ff98 100644 --- a/pkg/plugin/implementation/internal/common/common.go +++ b/pkg/plugin/implementation/internal/common/common.go @@ -23,7 +23,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/util" ) // Prerequisites are the values a capability needs that its payload does not @@ -170,7 +170,7 @@ func applyDefaults(cfg *Config) error { } } if cfg.MaxResponseBytes <= 0 { - cfg.MaxResponseBytes = httputil.DefaultMaxResponseBytes + cfg.MaxResponseBytes = util.DefaultMaxResponseBytes } if cfg.AuthByProvider == nil { cfg.AuthByProvider = map[string]*AuthProfile{} diff --git a/pkg/plugin/implementation/internal/common/common_test.go b/pkg/plugin/implementation/internal/common/common_test.go index a46493f6..61fcc9b9 100644 --- a/pkg/plugin/implementation/internal/common/common_test.go +++ b/pkg/plugin/implementation/internal/common/common_test.go @@ -18,7 +18,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/util" ) const selectBody = `{ @@ -149,7 +149,7 @@ func newStep(t *testing.T, registry definition.ProviderRecordLookup, mapper defi // refused at startup. Filled in after the tweaks, because a tweak may // replace BindingKeys and the profile has to follow. if len(cfg.AuthByProvider) == 0 { - cfg.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeNone}) + cfg.setProviderAuth(AuthProfile{Scheme: util.AuthSchemeNone}) } step, closer, err := New(context.Background(), registry, mapper, nil, cfg) if err != nil { @@ -186,26 +186,26 @@ func TestNewValidatesTheAuthScheme(t *testing.T) { config *Config valid bool }{ - {"none, declared", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeNone})}, true}, + {"none, declared", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: util.AuthSchemeNone})}, true}, {"no profile at all for a served provider", &Config{}, false}, {"an empty scheme", &Config{AuthByProvider: authForTestProvider(AuthProfile{})}, false}, - {"basic with both variables", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeBasic, UsernameEnv: "U", PasswordEnv: "P"})}, true}, - {"basic missing the password variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeBasic, UsernameEnv: "U"})}, false}, - {"basic missing the username variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeBasic, PasswordEnv: "P"})}, false}, - {"header with both settings", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeHeader, HeaderName: "X-Key", HeaderValueEnv: "V"})}, true}, - {"header missing the value variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeHeader, HeaderName: "X-Key"})}, false}, - {"query with both settings", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "t", QueryValueEnv: "V"})}, true}, + {"basic with both variables", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: util.AuthSchemeBasic, UsernameEnv: "U", PasswordEnv: "P"})}, true}, + {"basic missing the password variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: util.AuthSchemeBasic, UsernameEnv: "U"})}, false}, + {"basic missing the username variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: util.AuthSchemeBasic, PasswordEnv: "P"})}, false}, + {"header with both settings", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: util.AuthSchemeHeader, HeaderName: "X-Key", HeaderValueEnv: "V"})}, true}, + {"header missing the value variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: util.AuthSchemeHeader, HeaderName: "X-Key"})}, false}, + {"query with both settings", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: util.AuthSchemeQuery, QueryName: "t", QueryValueEnv: "V"})}, true}, {"an unknown scheme", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: "oauth"})}, false}, - {"oauth2 with all three settings", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeOAuth2, + {"oauth2 with all three settings", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: util.AuthSchemeOAuth2, TokenURL: "https://issuer.invalid/token", ClientIDEnv: "ID", ClientSecretEnv: "SECRET"})}, true}, - {"oauth2 missing the token url", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeOAuth2, + {"oauth2 missing the token url", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: util.AuthSchemeOAuth2, ClientIDEnv: "ID", ClientSecretEnv: "SECRET"})}, false}, - {"oauth2 missing the client id variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeOAuth2, + {"oauth2 missing the client id variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: util.AuthSchemeOAuth2, TokenURL: "https://issuer.invalid/token", ClientSecretEnv: "SECRET"})}, false}, - {"oauth2 missing the client secret variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeOAuth2, + {"oauth2 missing the client secret variable", &Config{AuthByProvider: authForTestProvider(AuthProfile{Scheme: util.AuthSchemeOAuth2, TokenURL: "https://issuer.invalid/token", ClientIDEnv: "ID"})}, false}, {"a profile for a provider not in bindingKeys", &Config{ - AuthByProvider: map[string]*AuthProfile{"someone-else": {Scheme: httputil.AuthSchemeNone}}}, false}, + AuthByProvider: map[string]*AuthProfile{"someone-else": {Scheme: util.AuthSchemeNone}}}, false}, } for _, tc := range testCases { @@ -339,7 +339,7 @@ func TestRunDoesNotRetryAMissingCredential(t *testing.T) { } mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} step := newStep(t, &stubRegistry{plan: plan}, mapper, func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeBasic, UsernameEnv: "TEST_ABSENT_USER_FOR_RETRY", PasswordEnv: "TEST_ABSENT_PASS_FOR_RETRY"}) + c.setProviderAuth(AuthProfile{Scheme: util.AuthSchemeBasic, UsernameEnv: "TEST_ABSENT_USER_FOR_RETRY", PasswordEnv: "TEST_ABSENT_PASS_FOR_RETRY"}) }) _, err := runStep(t, step, selectBody) @@ -523,7 +523,7 @@ func TestRunRedactsACredentialEchoedInABody(t *testing.T) { mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper, func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_ECHO_TOKEN"}) + c.setProviderAuth(AuthProfile{Scheme: util.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_ECHO_TOKEN"}) }) _, err := runStep(t, step, selectBody) @@ -539,7 +539,7 @@ func TestRunRedactsACredentialEchoedInABody(t *testing.T) { // token turns up. Assert on the same expression the code logs, so moving // the body from the error to the log cannot quietly move the leak with it. echoed := `{"rejected":"token=s3cr3t"}` - logged := step.redactString(httputil.Explain([]byte(echoed))) + logged := step.redactString(util.Explain([]byte(echoed))) if strings.Contains(logged, "s3cr3t") { t.Errorf("the credential survives into the log line: %s", logged) } @@ -567,7 +567,7 @@ func TestRunSendsTheCredentialAsAQueryParameter(t *testing.T) { mapper := &stubMapper{requestResult: []byte(`{"statecode":"CG"}`), responseResult: []byte(`{}`)} step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper, func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) + c.setProviderAuth(AuthProfile{Scheme: util.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) }) if _, err := runStep(t, step, selectBody); err != nil { @@ -597,7 +597,7 @@ func TestRunRedactsAQueryCredentialFromAnError(t *testing.T) { } mapper := &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)} step := newStep(t, &stubRegistry{plan: plan}, mapper, func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) + c.setProviderAuth(AuthProfile{Scheme: util.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) }) _, err := runStep(t, step, selectBody) @@ -619,7 +619,7 @@ func TestRedactStringRemovesTheCredentialFromTheURL(t *testing.T) { // No t.Parallel: t.Setenv forbids it. t.Setenv("TEST_MANDI_TOKEN", "s3cr3t") - step := stepWithProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) + step := stepWithProviderAuth(AuthProfile{Scheme: util.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) got := step.redactString("http://host/v1/x?statecode=CG&token=s3cr3t") if strings.Contains(got, "s3cr3t") { t.Errorf("the credential survived redaction: %s", got) @@ -629,7 +629,7 @@ func TestRedactStringRemovesTheCredentialFromTheURL(t *testing.T) { } // Any other scheme has nothing to hide in a URL, so the text is untouched. - plain := stepWithProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeNone}) + plain := stepWithProviderAuth(AuthProfile{Scheme: util.AuthSchemeNone}) if out := plain.redactString("http://host/v1/x?statecode=CG"); out != "http://host/v1/x?statecode=CG" { t.Errorf("a url with no credential must pass through unchanged, got %q", out) } @@ -645,7 +645,7 @@ func TestRedactStringRemovesThePercentEncodedCredential(t *testing.T) { const token = "a+b/c=d e" // every character Encode treats specially t.Setenv("TEST_MANDI_TOKEN", token) - step := stepWithProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) + step := stepWithProviderAuth(AuthProfile{Scheme: util.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) // Exactly how the credential appears once authenticate has run: Encode // escapes it, so this is the string a transport error quotes. @@ -671,7 +671,7 @@ func TestRedactStringStillRemovesAnUnescapedCredential(t *testing.T) { // No t.Parallel: t.Setenv forbids it. t.Setenv("TEST_MANDI_TOKEN", "plaintoken123") - step := stepWithProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) + step := stepWithProviderAuth(AuthProfile{Scheme: util.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_MANDI_TOKEN"}) got := step.redactString("http://host/v1/x?token=plaintoken123") if strings.Contains(got, "plaintoken123") { t.Errorf("the credential survived redaction: %s", got) @@ -688,9 +688,9 @@ func TestNewRefusesAHalfConfiguredQueryScheme(t *testing.T) { cfg *Config }{ {"no queryName", &Config{BindingKeys: []string{testBindingKey}, - AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryValueEnv: "TEST_MANDI_TOKEN"})}}, + AuthByProvider: authForTestProvider(AuthProfile{Scheme: util.AuthSchemeQuery, QueryValueEnv: "TEST_MANDI_TOKEN"})}}, {"no queryValueEnv", &Config{BindingKeys: []string{testBindingKey}, - AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token"})}}, + AuthByProvider: authForTestProvider(AuthProfile{Scheme: util.AuthSchemeQuery, QueryName: "token"})}}, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -1101,7 +1101,7 @@ func TestNewRefusesAHalfConfiguredOverride(t *testing.T) { _, _, err := New(context.Background(), &stubRegistry{}, &stubMapper{}, nil, &Config{ BindingKeys: []string{testBindingKey}, ProviderIDAt: "who.provider", - AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeNone}), + AuthByProvider: authForTestProvider(AuthProfile{Scheme: util.AuthSchemeNone}), }) if err == nil { t.Fatal("expected one path without the other to be refused") @@ -1350,7 +1350,7 @@ func TestRunPresentsBasicCredentialsFromTheEnvironment(t *testing.T) { step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeBasic, UsernameEnv: "TEST_MAUSAMGRAM_USER", PasswordEnv: "TEST_MAUSAMGRAM_KEY"}) + c.setProviderAuth(AuthProfile{Scheme: util.AuthSchemeBasic, UsernameEnv: "TEST_MAUSAMGRAM_USER", PasswordEnv: "TEST_MAUSAMGRAM_KEY"}) }) if _, err := runStep(t, step, selectBody); err != nil { @@ -1374,7 +1374,7 @@ func TestRunPresentsAHeaderCredentialFromTheEnvironment(t *testing.T) { step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeHeader, HeaderName: "X-Api-Key", HeaderValueEnv: "TEST_MAUSAMGRAM_TOKEN"}) + c.setProviderAuth(AuthProfile{Scheme: util.AuthSchemeHeader, HeaderName: "X-Api-Key", HeaderValueEnv: "TEST_MAUSAMGRAM_TOKEN"}) }) if _, err := runStep(t, step, selectBody); err != nil { @@ -1396,7 +1396,7 @@ func TestRunFailsWhenAConfiguredCredentialIsAbsent(t *testing.T) { step := newStep(t, &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeBasic, UsernameEnv: "TEST_MAUSAMGRAM_ABSENT_USER", PasswordEnv: "TEST_MAUSAMGRAM_ABSENT_KEY"}) + c.setProviderAuth(AuthProfile{Scheme: util.AuthSchemeBasic, UsernameEnv: "TEST_MAUSAMGRAM_ABSENT_USER", PasswordEnv: "TEST_MAUSAMGRAM_ABSENT_KEY"}) }) if _, err := runStep(t, step, selectBody); err == nil { @@ -1595,7 +1595,7 @@ func TestRedactKeepsTheErrorChainMatchable(t *testing.T) { // No t.Parallel: t.Setenv forbids it. t.Setenv("TEST_CHAIN_TOKEN", "s3cr3t") - step := stepWithProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_CHAIN_TOKEN"}) + step := stepWithProviderAuth(AuthProfile{Scheme: util.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_CHAIN_TOKEN"}) // The shape net/http produces: the cause wrapped behind text that quotes // the whole URL, credential and all. @@ -1629,7 +1629,7 @@ func TestRedactLeavesAnUnchangedErrorAlone(t *testing.T) { // No t.Parallel: t.Setenv forbids it. t.Setenv("TEST_CHAIN_TOKEN_2", "s3cr3t") - step := stepWithProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_CHAIN_TOKEN_2"}) + step := stepWithProviderAuth(AuthProfile{Scheme: util.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_CHAIN_TOKEN_2"}) original := errors.New("nothing sensitive here") if got := step.redact(original); got != original { t.Errorf("redact returned a different error for text it did not change: %v", got) @@ -1718,7 +1718,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { { name: "basic, the wire form a gateway echoes", tweak: func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeBasic, UsernameEnv: "TEST_USER", PasswordEnv: "TEST_PASS"}) + c.setProviderAuth(AuthProfile{Scheme: util.AuthSchemeBasic, UsernameEnv: "TEST_USER", PasswordEnv: "TEST_PASS"}) }, body: `{"error":"invalid Authorization: Basic ` + wire + `"}`, secret: wire, @@ -1726,7 +1726,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { { name: "basic, the password quoted raw", tweak: func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeBasic, UsernameEnv: "TEST_USER", PasswordEnv: "TEST_PASS"}) + c.setProviderAuth(AuthProfile{Scheme: util.AuthSchemeBasic, UsernameEnv: "TEST_USER", PasswordEnv: "TEST_PASS"}) }, body: `{"error":"bad password s3cr3t"}`, secret: "s3cr3t", @@ -1734,7 +1734,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { { name: "header, the value as sent", tweak: func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeHeader, HeaderName: "X-API-Key", HeaderValueEnv: "TEST_HDR"}) + c.setProviderAuth(AuthProfile{Scheme: util.AuthSchemeHeader, HeaderName: "X-API-Key", HeaderValueEnv: "TEST_HDR"}) }, body: `{"error":"bad X-API-Key: hdr-k3y"}`, secret: "hdr-k3y", @@ -1742,7 +1742,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { { name: "query, still covered, raw form", tweak: func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_QRY"}) + c.setProviderAuth(AuthProfile{Scheme: util.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_QRY"}) }, body: `{"rejected":"token=a+b/c="}`, secret: "a+b/c=", @@ -1750,7 +1750,7 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { { name: "query, still covered, percent-encoded form", tweak: func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_QRY"}) + c.setProviderAuth(AuthProfile{Scheme: util.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_QRY"}) }, body: `{"rejected":"token=` + url.QueryEscape("a+b/c=") + `"}`, secret: url.QueryEscape("a+b/c="), @@ -1761,12 +1761,12 @@ func TestRedactStringCoversEveryScheme(t *testing.T) { &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, tc.tweak) // The same expression the non-2xx path logs. - logged := step.redactString(httputil.Explain([]byte(tc.body))) + logged := step.redactString(util.Explain([]byte(tc.body))) if strings.Contains(logged, tc.secret) { t.Errorf("the credential survives into the log line: %s", logged) } - if !strings.Contains(logged, httputil.RedactedMarker) { + if !strings.Contains(logged, util.RedactedMarker) { t.Errorf("logged = %q, want the credential replaced", logged) } }) @@ -1784,7 +1784,7 @@ func TestRedactStringLeavesTheBasicUsername(t *testing.T) { step := newStep(t, &stubRegistry{plan: testPlan("http://provider.invalid", http.MethodGet)}, &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeBasic, UsernameEnv: "TEST_USER", PasswordEnv: "TEST_PASS"}) + c.setProviderAuth(AuthProfile{Scheme: util.AuthSchemeBasic, UsernameEnv: "TEST_USER", PasswordEnv: "TEST_PASS"}) }) got := step.redactString(`user mausam failed to authenticate with s3cr3t`) @@ -1799,7 +1799,7 @@ func TestRedactStringLeavesTheBasicUsername(t *testing.T) { // Nothing configured, nothing to hide: an unset credential must not turn every // empty string in the text into a redaction marker. func TestRedactStringWithNoCredentialConfigured(t *testing.T) { - for _, scheme := range []string{httputil.AuthSchemeNone, httputil.AuthSchemeBasic, httputil.AuthSchemeHeader, httputil.AuthSchemeQuery} { + for _, scheme := range []string{util.AuthSchemeNone, util.AuthSchemeBasic, util.AuthSchemeHeader, util.AuthSchemeQuery} { t.Run(scheme, func(t *testing.T) { const text = `{"error":"provider said no"}` step := newStep(t, &stubRegistry{plan: testPlan("http://provider.invalid", http.MethodGet)}, @@ -1847,7 +1847,7 @@ func TestRunHandsResolvedPrerequisitesToTheMappingAsLocal(t *testing.T) { step, closer, err := New(context.Background(), &stubRegistry{plan: testPlan(upstream.URL, http.MethodGet)}, mapper, prerequisites, &Config{BindingKeys: []string{testBindingKey}, - AuthByProvider: authForTestProvider(AuthProfile{Scheme: httputil.AuthSchemeNone})}) + AuthByProvider: authForTestProvider(AuthProfile{Scheme: util.AuthSchemeNone})}) if err != nil { t.Fatalf("New() returned an unexpected error: %v", err) } @@ -2014,7 +2014,7 @@ func oauth2Step(t *testing.T, tokenURL string, seen *[]string) *Step { return newStep(t, &stubRegistry{plan: testPlan(provider.URL, http.MethodGet)}, &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{"a":1}`)}, func(c *Config) { - c.setProviderAuth(AuthProfile{Scheme: httputil.AuthSchemeOAuth2, TokenURL: tokenURL, ClientIDEnv: "TEST_OAUTH_ID", ClientSecretEnv: "TEST_OAUTH_SECRET"}) + c.setProviderAuth(AuthProfile{Scheme: util.AuthSchemeOAuth2, TokenURL: tokenURL, ClientIDEnv: "TEST_OAUTH_ID", ClientSecretEnv: "TEST_OAUTH_SECRET"}) }) } @@ -2175,11 +2175,11 @@ func TestOAuth2RedactsTheSecretAndTheToken(t *testing.T) { for _, secret := range []string{`s3cr3t"quoted`, "the-token"} { body := fmt.Sprintf(`{"error":"rejected %s"}`, secret) - got := step.redactString(httputil.Explain([]byte(body))) + got := step.redactString(util.Explain([]byte(body))) if strings.Contains(got, secret) { t.Errorf("%q survived redaction: %s", secret, got) } - if !strings.Contains(got, httputil.RedactedMarker) { + if !strings.Contains(got, util.RedactedMarker) { t.Errorf("nothing was redacted from %q", got) } } diff --git a/pkg/plugin/implementation/internal/common/dispatch_test.go b/pkg/plugin/implementation/internal/common/dispatch_test.go index 230265f3..72728801 100644 --- a/pkg/plugin/implementation/internal/common/dispatch_test.go +++ b/pkg/plugin/implementation/internal/common/dispatch_test.go @@ -11,7 +11,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/util" ) // dispatchMappingRef stands in for the one reference an action carries. This @@ -85,7 +85,7 @@ func TestTwoProviderStepsDispatchByBindingKey(t *testing.T) { // Auth is per provider, so each step declares its own even // when neither upstream needs a credential. AuthByProvider: map[string]*common.AuthProfile{ - strings.Split(bindingKey, "|")[0]: {Scheme: httputil.AuthSchemeNone}, + strings.Split(bindingKey, "|")[0]: {Scheme: util.AuthSchemeNone}, }, }) if err != nil { diff --git a/pkg/plugin/implementation/internal/common/http.go b/pkg/plugin/implementation/internal/common/http.go index 6742431b..6d4090f6 100644 --- a/pkg/plugin/implementation/internal/common/http.go +++ b/pkg/plugin/implementation/internal/common/http.go @@ -11,17 +11,17 @@ import ( "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/model" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/util" ) -// call makes the upstream request the plan describes, retrying within httputil.Budget. +// call makes the upstream request the plan describes, retrying within util.Budget. func (s *Step) call(ctx context.Context, auth *authenticator, baseURL string, call model.ActionPlan, mapped []byte) ([]byte, error) { - endpoint, err := httputil.BuildEndpoint(baseURL, call, mapped) + endpoint, err := util.BuildEndpoint(baseURL, call, mapped) if err != nil { return nil, err } - timeout, retries := httputil.Budget(call) + timeout, retries := util.Budget(call) if d := time.Duration(call.TimeoutMs) * time.Millisecond; d > timeout { log.Warnf(ctx, "registry asks for a %v timeout; using the %v ceiling", d, timeout) } @@ -52,16 +52,16 @@ func (s *Step) call(ctx context.Context, auth *authenticator, baseURL string, ca // identically however often they are tried. Retrying the credential // case is the worst: it reports a missing environment variable as the // provider being down. - if httputil.IsPermanent(err) { + if util.IsPermanent(err) { break } if attempt < attempts { - if err := httputil.Sleep(ctx, httputil.Backoff(attempt)); err != nil { + if err := util.Sleep(ctx, util.Backoff(attempt)); err != nil { break } } } - return nil, model.NewCodedErr(http.StatusBadGateway, httputil.CodeUpstreamUnavailable, + return nil, model.NewCodedErr(http.StatusBadGateway, util.CodeUpstreamUnavailable, fmt.Errorf("provider did not answer after %d attempts: %w", attempts, lastErr)) } @@ -70,17 +70,17 @@ func (s *Step) attempt(ctx context.Context, auth *authenticator, call model.Acti attemptCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - method := httputil.CanonicalMethod(call.Method) - req, err := http.NewRequestWithContext(attemptCtx, method, endpoint, httputil.RequestBody(method, mapped)) + method := util.CanonicalMethod(call.Method) + req, err := http.NewRequestWithContext(attemptCtx, method, endpoint, util.RequestBody(method, mapped)) if err != nil { - return nil, httputil.DoNotRetry(fmt.Errorf("could not build the request: %w", err)) + return nil, util.DoNotRetry(fmt.Errorf("could not build the request: %w", err)) } - if httputil.HasBody(method) { + if util.HasBody(method) { req.Header.Set("Content-Type", "application/json") } if err := s.authenticate(auth, req); err != nil { // A missing or unreadable credential is configuration, not weather. - return nil, httputil.DoNotRetry(err) + return nil, util.DoNotRetry(err) } // The URL as it went on the wire, credential removed. At info because this @@ -100,7 +100,7 @@ func (s *Step) attempt(ctx context.Context, auth *authenticator, call model.Acti log.Infof(ctx, "%s %s -> %s, %d bytes", method, requested, resp.Status, len(body)) if int64(len(body)) > s.config.MaxResponseBytes { // Asking again will not make the answer smaller. - return nil, httputil.DoNotRetry(fmt.Errorf("response exceeds the %d byte limit", s.config.MaxResponseBytes)) + return nil, util.DoNotRetry(fmt.Errorf("response exceeds the %d byte limit", s.config.MaxResponseBytes)) } // Any 2xx, not 200 alone: 202, 204 and 201 are all legitimate answers. 3xx // does not reach here, since the client follows redirects. @@ -112,12 +112,12 @@ func (s *Step) attempt(ctx context.Context, auth *authenticator, call model.Acti // Redacted on the way to the log too: a rejected request is often // quoted back, credential and all. log.Warnf(ctx, "provider returned %s for %s %s: %s", - resp.Status, method, requested, s.redactString(httputil.Explain(body))) + resp.Status, method, requested, s.redactString(util.Explain(body))) err := fmt.Errorf("provider returned %s", resp.Status) // 5xx and 429 ask to be tried again. Every other 4xx is a statement // about the request, which will not improve. if resp.StatusCode < http.StatusInternalServerError && resp.StatusCode != http.StatusTooManyRequests { - return nil, httputil.DoNotRetry(err) + return nil, util.DoNotRetry(err) } return nil, err } diff --git a/pkg/plugin/implementation/internal/common/perprovider_test.go b/pkg/plugin/implementation/internal/common/perprovider_test.go index 865752d4..f9e14e0d 100644 --- a/pkg/plugin/implementation/internal/common/perprovider_test.go +++ b/pkg/plugin/implementation/internal/common/perprovider_test.go @@ -11,7 +11,7 @@ import ( "testing" "github.com/beckn-one/beckn-onix/pkg/model" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/util" ) // bodyFor builds a select naming a particular provider, so one step serving @@ -81,8 +81,8 @@ func TestTwoProvidersOnOneStepAuthenticateDifferently(t *testing.T) { cfg := &Config{ BindingKeys: []string{"alpha|" + capability, "beta|" + capability}, AuthByProvider: map[string]*AuthProfile{ - "alpha": {Scheme: httputil.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_PP_QUERY_TOKEN"}, - "beta": {Scheme: httputil.AuthSchemeHeader, HeaderName: "X-Api-Key", HeaderValueEnv: "TEST_PP_HEADER_TOKEN"}, + "alpha": {Scheme: util.AuthSchemeQuery, QueryName: "token", QueryValueEnv: "TEST_PP_QUERY_TOKEN"}, + "beta": {Scheme: util.AuthSchemeHeader, HeaderName: "X-Api-Key", HeaderValueEnv: "TEST_PP_HEADER_TOKEN"}, }, } registry := &routingRegistry{plans: map[string]*model.ProviderRecord{ @@ -156,9 +156,9 @@ func TestTwoOAuth2ProvidersDoNotShareAToken(t *testing.T) { cfg := &Config{ BindingKeys: []string{"alpha|" + capability, "beta|" + capability}, AuthByProvider: map[string]*AuthProfile{ - "alpha": {Scheme: httputil.AuthSchemeOAuth2, TokenURL: issuerA.URL, + "alpha": {Scheme: util.AuthSchemeOAuth2, TokenURL: issuerA.URL, ClientIDEnv: "TEST_PP_ID_A", ClientSecretEnv: "TEST_PP_SECRET_A"}, - "beta": {Scheme: httputil.AuthSchemeOAuth2, TokenURL: issuerB.URL, + "beta": {Scheme: util.AuthSchemeOAuth2, TokenURL: issuerB.URL, ClientIDEnv: "TEST_PP_ID_B", ClientSecretEnv: "TEST_PP_SECRET_B"}, }, } @@ -215,8 +215,8 @@ func TestRedactionCoversEveryProvidersSecret(t *testing.T) { cfg := &Config{ BindingKeys: []string{"alpha|" + capability, "beta|" + capability}, AuthByProvider: map[string]*AuthProfile{ - "alpha": {Scheme: httputil.AuthSchemeHeader, HeaderName: "X-A", HeaderValueEnv: "TEST_PP_RED_A"}, - "beta": {Scheme: httputil.AuthSchemeHeader, HeaderName: "X-B", HeaderValueEnv: "TEST_PP_RED_B"}, + "alpha": {Scheme: util.AuthSchemeHeader, HeaderName: "X-A", HeaderValueEnv: "TEST_PP_RED_A"}, + "beta": {Scheme: util.AuthSchemeHeader, HeaderName: "X-B", HeaderValueEnv: "TEST_PP_RED_B"}, }, } step, closer, err := New(context.Background(), &routingRegistry{}, &stubMapper{}, nil, cfg) @@ -248,8 +248,8 @@ func TestRedactionSortsAcrossProvidersNotWithinOne(t *testing.T) { cfg := &Config{ BindingKeys: []string{"shorty|" + capability, "longy|" + capability}, AuthByProvider: map[string]*AuthProfile{ - "shorty": {Scheme: httputil.AuthSchemeHeader, HeaderName: "X-S", HeaderValueEnv: "TEST_PP_SHORT"}, - "longy": {Scheme: httputil.AuthSchemeHeader, HeaderName: "X-L", HeaderValueEnv: "TEST_PP_LONG"}, + "shorty": {Scheme: util.AuthSchemeHeader, HeaderName: "X-S", HeaderValueEnv: "TEST_PP_SHORT"}, + "longy": {Scheme: util.AuthSchemeHeader, HeaderName: "X-L", HeaderValueEnv: "TEST_PP_LONG"}, }, } step, closer, err := New(context.Background(), &routingRegistry{}, &stubMapper{}, nil, cfg) diff --git a/pkg/plugin/implementation/internal/common/redact.go b/pkg/plugin/implementation/internal/common/redact.go index c4121652..155c2d28 100644 --- a/pkg/plugin/implementation/internal/common/redact.go +++ b/pkg/plugin/implementation/internal/common/redact.go @@ -8,7 +8,7 @@ import ( "sort" "strings" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/util" ) // redact removes a credential from an error's text. @@ -50,7 +50,7 @@ func (e redactedErr) Unwrap() error { return e.err } // so a credential reaches a message under any scheme. func (s *Step) redactString(text string) string { for _, secret := range s.secretForms() { - text = strings.ReplaceAll(text, secret, httputil.RedactedMarker) + text = strings.ReplaceAll(text, secret, util.RedactedMarker) } return text } @@ -83,7 +83,7 @@ func (s *Step) secretForms() []string { // request. func (a *authenticator) secretForms() []string { switch a.cfg.Scheme { - case httputil.AuthSchemeBasic: + case util.AuthSchemeBasic: username, password := os.Getenv(a.cfg.UsernameEnv), os.Getenv(a.cfg.PasswordEnv) if password == "" { return nil @@ -98,13 +98,13 @@ func (a *authenticator) secretForms() []string { // authenticates, and words like "user" or "admin" would eat // unrelated text. return forms - case httputil.AuthSchemeHeader: + case util.AuthSchemeHeader: value := os.Getenv(a.cfg.HeaderValueEnv) if value == "" { return nil } return []string{value} - case httputil.AuthSchemeOAuth2: + case util.AuthSchemeOAuth2: // Both halves: the secret we send the issuer, and the token it gave // back. The token is what reaches the provider. var forms []string @@ -119,7 +119,7 @@ func (a *authenticator) secretForms() []string { // The client id is NOT redacted: it identifies, it does not // authenticate. return forms - case httputil.AuthSchemeQuery: + case util.AuthSchemeQuery: value := os.Getenv(a.cfg.QueryValueEnv) if value == "" { return nil diff --git a/pkg/plugin/implementation/internal/common/token.go b/pkg/plugin/implementation/internal/common/token.go index c61cc10d..6124dbda 100644 --- a/pkg/plugin/implementation/internal/common/token.go +++ b/pkg/plugin/implementation/internal/common/token.go @@ -14,7 +14,7 @@ import ( "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/model" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/httputil" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/util" ) // bearerToken returns a token to send, exchanging one if the held token has @@ -100,7 +100,7 @@ func (s *Step) exchangeToken(ctx context.Context, auth *authenticator) (string, // The status, not the body: a failure body routinely quotes the // request back. log.Warnf(ctx, "token endpoint %s returned %s: %s", - cfg.TokenURL, resp.Status, s.redactString(httputil.Explain(body))) + cfg.TokenURL, resp.Status, s.redactString(util.Explain(body))) return "", 0, s.tokenErr(fmt.Errorf("token endpoint returned %s", resp.Status)) } @@ -113,7 +113,7 @@ func (s *Step) exchangeToken(ctx context.Context, auth *authenticator) (string, return "", 0, s.tokenErr(fmt.Errorf("token response from %s carries no access_token", cfg.TokenURL)) } - lifetime, ok := tokenLifetime(parsed.ExpiresIn, httputil.TokenRefreshSkew) + lifetime, ok := tokenLifetime(parsed.ExpiresIn, util.TokenRefreshSkew) if !ok { return "", 0, s.tokenErr(fmt.Errorf( "token response from %s carries no usable expires_in, so its lifetime is unknown", @@ -125,7 +125,7 @@ func (s *Step) exchangeToken(ctx context.Context, auth *authenticator) (string, // tokenErr classifies a failed exchange. Always 502: the failure is the // issuer's, never the caller's. func (s *Step) tokenErr(err error) error { - return model.NewCodedErr(http.StatusBadGateway, httputil.CodeUpstreamUnavailable, + return model.NewCodedErr(http.StatusBadGateway, util.CodeUpstreamUnavailable, fmt.Errorf("oauth2 token exchange failed: %w", err)) } diff --git a/pkg/plugin/implementation/internal/common/httputil/constant.go b/pkg/plugin/implementation/internal/common/util/constant.go similarity index 85% rename from pkg/plugin/implementation/internal/common/httputil/constant.go rename to pkg/plugin/implementation/internal/common/util/constant.go index c4e95158..56fa7e8b 100644 --- a/pkg/plugin/implementation/internal/common/httputil/constant.go +++ b/pkg/plugin/implementation/internal/common/util/constant.go @@ -1,14 +1,8 @@ -// Every constant this machinery uses, in one file. +// Every constant the package uses, in one file. // -// Here rather than in common because common imports this package: a constant -// read by anything under httputil could not live in common without an import -// cycle, and splitting them across two files put the auth schemes in one place -// and the retry budget in another. -// -// The consequence is that the package name undersells the contents -- the auth -// schemes and the redaction marker are not about HTTP plumbing. One file that -// can be found beats two that are each in the right place. -package httputil +// Here rather than in common because common imports this package, so a +// constant read on this side could not live there without an import cycle. +package util import ( "time" diff --git a/pkg/plugin/implementation/internal/common/httputil/endpoint.go b/pkg/plugin/implementation/internal/common/util/endpoint.go similarity index 99% rename from pkg/plugin/implementation/internal/common/httputil/endpoint.go rename to pkg/plugin/implementation/internal/common/util/endpoint.go index be6516db..fce988d8 100644 --- a/pkg/plugin/implementation/internal/common/httputil/endpoint.go +++ b/pkg/plugin/implementation/internal/common/util/endpoint.go @@ -1,6 +1,6 @@ // Building the URL a request goes to, and refusing one a registry row could // not have meant. -package httputil +package util import ( "errors" diff --git a/pkg/plugin/implementation/internal/common/httputil/explain.go b/pkg/plugin/implementation/internal/common/util/explain.go similarity index 97% rename from pkg/plugin/implementation/internal/common/httputil/explain.go rename to pkg/plugin/implementation/internal/common/util/explain.go index d65884e0..ae5e0fe6 100644 --- a/pkg/plugin/implementation/internal/common/httputil/explain.go +++ b/pkg/plugin/implementation/internal/common/util/explain.go @@ -1,5 +1,5 @@ // Quoting a failed response body for a human. -package httputil +package util import "strings" diff --git a/pkg/plugin/implementation/internal/common/httputil/query.go b/pkg/plugin/implementation/internal/common/util/query.go similarity index 99% rename from pkg/plugin/implementation/internal/common/httputil/query.go rename to pkg/plugin/implementation/internal/common/util/query.go index d5e2e381..825ff024 100644 --- a/pkg/plugin/implementation/internal/common/httputil/query.go +++ b/pkg/plugin/implementation/internal/common/util/query.go @@ -1,5 +1,5 @@ // Rendering a mapped request as query parameters, for methods with no body. -package httputil +package util import ( "bytes" diff --git a/pkg/plugin/implementation/internal/common/httputil/request.go b/pkg/plugin/implementation/internal/common/util/request.go similarity index 98% rename from pkg/plugin/implementation/internal/common/httputil/request.go rename to pkg/plugin/implementation/internal/common/util/request.go index d3f14f0a..d34a07e8 100644 --- a/pkg/plugin/implementation/internal/common/httputil/request.go +++ b/pkg/plugin/implementation/internal/common/util/request.go @@ -1,5 +1,5 @@ // The request body and method: what to send, and in what spelling. -package httputil +package util import ( "bytes" diff --git a/pkg/plugin/implementation/internal/common/httputil/retry.go b/pkg/plugin/implementation/internal/common/util/retry.go similarity index 80% rename from pkg/plugin/implementation/internal/common/httputil/retry.go rename to pkg/plugin/implementation/internal/common/util/retry.go index c11cfd32..8e6161d9 100644 --- a/pkg/plugin/implementation/internal/common/httputil/retry.go +++ b/pkg/plugin/implementation/internal/common/util/retry.go @@ -1,6 +1,14 @@ -// The retry budget: how long an attempt may take, how many follow it, and -// which failures are worth repeating. -package httputil +// Package util holds what the capability machinery needs but does not own: the +// retry budget, the endpoint and body of a request, and every constant. +// +// It exists because these have no receiver on common's types, so they can sit +// in their own package -- and because common imports it, the constants live +// here too. Named plainly, since the contents are a mix rather than one +// subject: the retry budget, the query rendering, the auth scheme names. +// +// Not a home for anything that will fit elsewhere. A helper belongs beside the +// code that uses it unless a receiver forbids it. +package util import ( "context" diff --git a/pkg/plugin/implementation/internal/common/httputil/httputil_test.go b/pkg/plugin/implementation/internal/common/util/util_test.go similarity index 99% rename from pkg/plugin/implementation/internal/common/httputil/httputil_test.go rename to pkg/plugin/implementation/internal/common/util/util_test.go index a77c2dfb..5bd9dcfe 100644 --- a/pkg/plugin/implementation/internal/common/httputil/httputil_test.go +++ b/pkg/plugin/implementation/internal/common/util/util_test.go @@ -1,4 +1,4 @@ -package httputil +package util import ( "net/http" From 2d296b65f7646e46df6245bb5873a99195442db9 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 23:18:31 +0530 Subject: [PATCH 121/130] fix(deps): bump grpc to 1.83.2 for CVE-2026-84445 [#7] The Security Scan gate has been red on every PR since the CVE entered Trivy's database: CVE-2026-84445 google.golang.org/grpc v1.83.1 HIGH fixed in 1.83.2 NOT INTRODUCED HERE. development carries the same version and its own scan passed, having run before the CVE was published. The bump is in this branch because that is where the red gate is; it applies equally to development. golang.org/x/net moves 0.57.0 -> 0.58.0 with it, indirect. go mod tidy leaves the diff at exactly those two lines and four go.sum entries. Verified rather than assumed: trivy fs over the tree now reports ZERO findings at CRITICAL,HIGH,MEDIUM,LOW, where it reported this one before. grpc here is the OTLP exporter's transport, so otelsetup was run separately -- green. 66 packages green with -count=1, vet clean. --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 5715fa56..b09734c0 100644 --- a/go.mod +++ b/go.mod @@ -80,7 +80,7 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/net v0.57.0 // indirect + golang.org/x/net v0.58.0 // indirect golang.org/x/sys v0.47.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect @@ -120,7 +120,7 @@ require ( go.opentelemetry.io/otel/trace v1.44.0 go.uber.org/automaxprocs v1.6.0 golang.org/x/sync v0.22.0 - google.golang.org/grpc v1.83.1 + google.golang.org/grpc v1.83.2 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 35da526b..2906ed33 100644 --- a/go.sum +++ b/go.sum @@ -263,8 +263,8 @@ golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= @@ -281,8 +281,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1: google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= -google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 7ffeafecd53cd126c384f9434d165e625e7195b1 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 9 Sep 2026 23:30:22 +0530 Subject: [PATCH 122/130] fix(common): retry a transient oauth2 token exchange [#7] A reviewer on #24 found this, and the diagnosis was exact. attempt wrapped every authenticate error in DoNotRetry, with the comment "a missing or unreadable credential is configuration, not weather". That was true before this branch, when authenticate could only fail by reading an unset environment variable. oauth2 made the same path do network I/O to a third-party issuer, so the blanket wrap then swallowed a transient failure: an unreachable token endpoint or a 5xx aborted the request after one attempt, while the IDENTICAL 5xx from the provider is retried twenty lines below. The branch's own test proved it and asserted it as correct: TestOAuth2DoesNotCacheAFailedExchange budgeted two attempts, pointed the issuer at a permanent 500, and checked for exactly one call per request. Classified at the source instead, so attempt no longer has to guess: retried endpoint unreachable, response unreadable, 5xx, 429 not retried tokenUrl that will not parse, 4xx other than 429, a 2xx that is not JSON, no access_token, no usable expires_in not retried an unset credential -- marked in missingCredential, which is shared with basic, header and query and unchanged for them The 5xx-and-429 rule is copied from the provider-response path in the same function rather than invented, so the two agree. TestOAuth2DoesNotCacheAFailedExchange now expects 6 calls for 3 requests rather than 3. Its subject is unchanged -- a cached failure would hold the count at 2 however many requests arrive. Two new tests cover both directions, which is what was missing: 500, 503 and 429 retry; 401 and 400 do not; an unset credential never dials the issuer. Negative control: reinstating the blanket wrap fails exactly the three transient cases and leaves the two permanent ones passing. 66 packages green, vet clean, no races. --- .../implementation/internal/common/auth.go | 6 +- .../internal/common/common_test.go | 78 ++++++++++++++++++- .../implementation/internal/common/http.go | 7 +- .../implementation/internal/common/token.go | 35 +++++++-- 4 files changed, 115 insertions(+), 11 deletions(-) diff --git a/pkg/plugin/implementation/internal/common/auth.go b/pkg/plugin/implementation/internal/common/auth.go index c013a9f3..e53eb4e6 100644 --- a/pkg/plugin/implementation/internal/common/auth.go +++ b/pkg/plugin/implementation/internal/common/auth.go @@ -200,7 +200,11 @@ func (s *Step) missingCredential(ctx context.Context, provider, scheme, envNames err := fmt.Errorf("the %s credential for %s is not configured", scheme, provider) log.Errorf(ctx, err, "%s auth is configured for %s but %s is not set", scheme, provider, envNames) - return err + // Marked here rather than at the call site in attempt, which cannot tell + // this apart from an oauth2 exchange that failed on the network. An unset + // environment variable is configuration: retrying reports an operator's + // missing variable as the provider being down. + return util.DoNotRetry(err) } // authenticate presents this provider's credentials, read from the environment diff --git a/pkg/plugin/implementation/internal/common/common_test.go b/pkg/plugin/implementation/internal/common/common_test.go index 61fcc9b9..a59b2f44 100644 --- a/pkg/plugin/implementation/internal/common/common_test.go +++ b/pkg/plugin/implementation/internal/common/common_test.go @@ -2152,8 +2152,13 @@ func TestOAuth2DoesNotCacheAFailedExchange(t *testing.T) { t.Fatalf("request %d unexpectedly succeeded", i) } } - if got := ts.calls.Load(); got != 3 { - t.Errorf("token endpoint called %d times for 3 failing requests, want 3 -- a failure was cached", got) + // Two per request, not one: RetryMax is 1, and a 500 from the issuer is + // weather, so the retry budget covers the exchange. What matters here is + // that the count RISES with the requests -- a cached failure would leave it + // at 2 for all three. + if got := ts.calls.Load(); got != 6 { + t.Errorf("token endpoint called %d times for 3 failing requests, want 6 "+ + "(2 attempts each) -- fewer means a failure was cached", got) } } @@ -2256,3 +2261,72 @@ func TestNoAuthFieldNameCarriesADash(t *testing.T) { } } } + +// A 5xx from the token issuer is weather, so the retry budget covers it -- the +// same rule the provider's own 5xx gets a few lines below in attempt. +// +// This was the defect a reviewer found on #24: authenticate's error was wrapped +// in DoNotRetry wholesale, which was right when the only way to fail was an +// unset environment variable. oauth2 made that path do network I/O to a third +// party, and the blanket wrap then swallowed a transient issuer failure. +func TestOAuth2RetriesATransientIssuerFailure(t *testing.T) { + t.Setenv("TEST_OAUTH_ID", "svc-client") + t.Setenv("TEST_OAUTH_SECRET", "svc-secret") + + for _, tc := range []struct { + name string + status int + want int32 + }{ + {"500 is retried", http.StatusInternalServerError, 2}, + {"503 is retried", http.StatusServiceUnavailable, 2}, + {"429 is retried", http.StatusTooManyRequests, 2}, + // A 4xx that is not 429 says the client credentials are wrong, which + // another attempt will not change. + {"401 is not retried", http.StatusUnauthorized, 1}, + {"400 is not retried", http.StatusBadRequest, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + ts := &tokenServer{status: tc.status, body: `{"error":"no"}`} + var seen []string + step := oauth2Step(t, ts.start(t), &seen) + + if _, err := runStep(t, step, selectBody); err == nil { + t.Fatal("expected the request to fail") + } + if got := ts.calls.Load(); got != tc.want { + t.Errorf("token endpoint called %d times, want %d", got, tc.want) + } + }) + } +} + +// An unset credential is configuration, not weather: retrying reports an +// operator's missing environment variable as the provider being down. +func TestOAuth2DoesNotRetryAMissingCredential(t *testing.T) { + // Deliberately unset, so authenticate fails before any network I/O. + t.Setenv("TEST_OAUTH_ABSENT_ID", "") + t.Setenv("TEST_OAUTH_ABSENT_SECRET", "") + + ts := &tokenServer{status: http.StatusOK, body: `{"access_token":"t","expires_in":3600}`} + tokenURL := ts.start(t) + step := newStep(t, &stubRegistry{plan: testPlan("http://provider.invalid", http.MethodGet)}, + &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{}`)}, + func(c *Config) { + c.setProviderAuth(AuthProfile{ + Scheme: util.AuthSchemeOAuth2, + TokenURL: tokenURL, + ClientIDEnv: "TEST_OAUTH_ABSENT_ID", + ClientSecretEnv: "TEST_OAUTH_ABSENT_SECRET", + }) + }) + + if _, err := runStep(t, step, selectBody); err == nil { + t.Fatal("expected an unset credential to fail") + } + // Never dialled: the credential is missing before the exchange, and the + // failure must not be repeated. + if got := ts.calls.Load(); got != 0 { + t.Errorf("token endpoint called %d times, want 0", got) + } +} diff --git a/pkg/plugin/implementation/internal/common/http.go b/pkg/plugin/implementation/internal/common/http.go index 6d4090f6..56304098 100644 --- a/pkg/plugin/implementation/internal/common/http.go +++ b/pkg/plugin/implementation/internal/common/http.go @@ -79,8 +79,11 @@ func (s *Step) attempt(ctx context.Context, auth *authenticator, call model.Acti req.Header.Set("Content-Type", "application/json") } if err := s.authenticate(auth, req); err != nil { - // A missing or unreadable credential is configuration, not weather. - return nil, util.DoNotRetry(err) + // Already classified at the source. A missing environment variable is + // configuration and marked permanent there; an oauth2 exchange that + // could not reach the issuer is weather and is not, so the retry budget + // covers the token endpoint exactly as it covers the provider. + return nil, err } // The URL as it went on the wire, credential removed. At info because this diff --git a/pkg/plugin/implementation/internal/common/token.go b/pkg/plugin/implementation/internal/common/token.go index 6124dbda..93ded54b 100644 --- a/pkg/plugin/implementation/internal/common/token.go +++ b/pkg/plugin/implementation/internal/common/token.go @@ -80,7 +80,9 @@ func (s *Step) exchangeToken(ctx context.Context, auth *authenticator) (string, req, err := http.NewRequestWithContext(ctx, http.MethodPost, cfg.TokenURL, strings.NewReader(form.Encode())) if err != nil { - return "", 0, s.tokenErr(fmt.Errorf("token request could not be built: %w", err)) + // A tokenUrl that will not parse is configuration. + return "", 0, s.permanentTokenErr( + fmt.Errorf("token request could not be built: %w", err)) } req.Header.Set("Content-Type", "application/x-www-form-urlencoded") @@ -101,21 +103,31 @@ func (s *Step) exchangeToken(ctx context.Context, auth *authenticator) (string, // request back. log.Warnf(ctx, "token endpoint %s returned %s: %s", cfg.TokenURL, resp.Status, s.redactString(util.Explain(body))) - return "", 0, s.tokenErr(fmt.Errorf("token endpoint returned %s", resp.Status)) + err := fmt.Errorf("token endpoint returned %s", resp.Status) + // The same rule the provider's own status gets: 5xx and 429 ask to be + // tried again, and every other 4xx is a statement about the request -- + // here, usually that the client credentials are wrong. + if resp.StatusCode < http.StatusInternalServerError && + resp.StatusCode != http.StatusTooManyRequests { + return "", 0, s.permanentTokenErr(err) + } + return "", 0, s.tokenErr(err) } var parsed tokenResponse if err := json.Unmarshal(body, &parsed); err != nil { - return "", 0, s.tokenErr(fmt.Errorf("token response from %s is not JSON: %w", + // A 2xx that is not JSON is the issuer misbehaving in a way another + // attempt will not change. + return "", 0, s.permanentTokenErr(fmt.Errorf("token response from %s is not JSON: %w", cfg.TokenURL, err)) } if parsed.AccessToken == "" { - return "", 0, s.tokenErr(fmt.Errorf("token response from %s carries no access_token", - cfg.TokenURL)) + return "", 0, s.permanentTokenErr( + fmt.Errorf("token response from %s carries no access_token", cfg.TokenURL)) } lifetime, ok := tokenLifetime(parsed.ExpiresIn, util.TokenRefreshSkew) if !ok { - return "", 0, s.tokenErr(fmt.Errorf( + return "", 0, s.permanentTokenErr(fmt.Errorf( "token response from %s carries no usable expires_in, so its lifetime is unknown", cfg.TokenURL)) } @@ -124,11 +136,22 @@ func (s *Step) exchangeToken(ctx context.Context, auth *authenticator) (string, // tokenErr classifies a failed exchange. Always 502: the failure is the // issuer's, never the caller's. +// +// Retryable unless the caller says otherwise. The exchange is network I/O to a +// third party, so the same 5xx and 429 that earn a retry against the provider +// earn one here; permanent causes are marked with permanentTokenErr instead. func (s *Step) tokenErr(err error) error { return model.NewCodedErr(http.StatusBadGateway, util.CodeUpstreamUnavailable, fmt.Errorf("oauth2 token exchange failed: %w", err)) } +// permanentTokenErr is tokenErr for a cause no further attempt would change: a +// token URL that cannot be built, a 4xx that is not 429, or a response the +// issuer will keep sending in the same shape. +func (s *Step) permanentTokenErr(err error) error { + return util.DoNotRetry(s.tokenErr(err)) +} + // tokenLifetime turns expires_in into how long the token may be held, or // reports that it may not be held at all. // From cb30dc92a3f9f2122add28c9cb57bbbca96e4171 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Thu, 10 Sep 2026 00:16:34 +0530 Subject: [PATCH 123/130] feat(AgricultureFacility): add the POCRA facility plugin, on latest development [#91] PR #22's work, replayed onto development and adapted to it. Squashed from 22 commits: 12 of them modified internal/upstream/upstream.go, which development split into seven files, so each would have conflicted as a modify/delete. The net change to the shared package across all 22 was one exported function; the rest was churn -- fan-out added then removed, BindingPaths renamed three times. WHAT THE ADAPTATION NEEDED. development has since renamed internal/upstream to internal/common, folded capabilitybinding into it, and made auth per provider: internal/upstream -> internal/common capabilitybinding.From -> common.BindingFrom, exported for this plugin bindingPaths -> common.BindingPaths, exported for this plugin Config's flat auth fields -> Config.AuthByProvider, one profile per provider authScheme: none -> a pocra: block, nested under the participant id Two of those are exports the plugin needs and this branch adds, with the reason recorded at each: it answers "is this payload mine?" for a multi-type search, and it must answer it the same way serve does rather than reading the payload a second way. The config entry and the steps list gain AgricultureFacility alongside the three that were already there; build-plugins.sh gains it too, rather than replacing KnowledgeAdvisory as the original diff did -- that branch predated the knowledge plugin. Their tests needed the same adaptation. Every helper that builds a Config now declares pocra's auth, since a served provider without a block is refused at startup -- which is the new validation working, not a defect. 69 packages green, vet clean, no races. The other three capabilities were run separately to confirm the two new exports changed nothing for them. Original commits are preserved on feat/91-AgricultualFacility(Plugin) and on PR #22, which is untouched. --- .gitignore | 6 +- .../pocra/agriculture-facility.select.yaml | 514 +++++ config/provider-adapter.yaml | 28 + install/build-plugins.sh | 1 + .../AgricultureFacility.go | 104 + .../AgricultureFacility/README.md | 187 ++ .../AgricultureFacility/cmd/plugin.go | 120 ++ .../AgricultureFacility/cmd/plugin_test.go | 199 ++ .../AgricultureFacility/conformance_test.go | 462 +++++ .../AgricultureFacility/live_test.go | 399 ++++ .../AgricultureFacility/mappings_test.go | 1762 +++++++++++++++++ .../AgricultureFacility/payload.go | 134 ++ .../AgricultureFacility/payload_test.go | 133 ++ .../AgricultureFacility/prerequisites.go | 17 + .../AgricultureFacility/schemacache_test.go | 309 +++ .../AgricultureFacility/search.go | 398 ++++ .../AgricultureFacility/search_test.go | 249 +++ .../AgricultureFacility/testdata/.gitignore | 4 + .../implementation/internal/common/binding.go | 7 +- .../internal/common/binding_test.go | 30 +- .../implementation/internal/common/common.go | 11 +- .../implementation/internal/common/serve.go | 4 +- .../internal/concurrent/concurrent.go | 116 ++ .../internal/concurrent/concurrent_test.go | 288 +++ 24 files changed, 5459 insertions(+), 23 deletions(-) create mode 100644 config/mappings/pocra/agriculture-facility.select.yaml create mode 100644 pkg/plugin/implementation/AgricultureFacility/AgricultureFacility.go create mode 100644 pkg/plugin/implementation/AgricultureFacility/README.md create mode 100644 pkg/plugin/implementation/AgricultureFacility/cmd/plugin.go create mode 100644 pkg/plugin/implementation/AgricultureFacility/cmd/plugin_test.go create mode 100644 pkg/plugin/implementation/AgricultureFacility/conformance_test.go create mode 100644 pkg/plugin/implementation/AgricultureFacility/live_test.go create mode 100644 pkg/plugin/implementation/AgricultureFacility/mappings_test.go create mode 100644 pkg/plugin/implementation/AgricultureFacility/payload.go create mode 100644 pkg/plugin/implementation/AgricultureFacility/payload_test.go create mode 100644 pkg/plugin/implementation/AgricultureFacility/prerequisites.go create mode 100644 pkg/plugin/implementation/AgricultureFacility/schemacache_test.go create mode 100644 pkg/plugin/implementation/AgricultureFacility/search.go create mode 100644 pkg/plugin/implementation/AgricultureFacility/search_test.go create mode 100644 pkg/plugin/implementation/AgricultureFacility/testdata/.gitignore create mode 100644 pkg/plugin/implementation/internal/concurrent/concurrent.go create mode 100644 pkg/plugin/implementation/internal/concurrent/concurrent_test.go diff --git a/.gitignore b/.gitignore index 6292bb58..1134973c 100644 --- a/.gitignore +++ b/.gitignore @@ -176,4 +176,8 @@ test_request.json .claude # catalogPublish handler's local output root (config/local-beckn-one-bap.yaml) -/catalog/ \ No newline at end of file +/catalog/ + +# Local, non-shipped developer notes (review summaries, scratch docs) +/dev_docs/ +/docs/ \ No newline at end of file diff --git a/config/mappings/pocra/agriculture-facility.select.yaml b/config/mappings/pocra/agriculture-facility.select.yaml new file mode 100644 index 00000000..44aee0fe --- /dev/null +++ b/config/mappings/pocra/agriculture-facility.select.yaml @@ -0,0 +1,514 @@ +# POCRA, openagrinet:AgricultureFacility, select. Both directions, one file. +# +# One file per binding-action rather than one per direction, because both legs of +# an exchange are one contract: the response has to answer the request that was +# sent, and splitting them lets one change without the other. +# +# The registry entry pointing here decides which action this serves, so nothing +# in the file names it. The filename's action segment must match that entry -- +# a mismatch would apply a correct mapping to the wrong call, silently. +# +# Both halves read: +# beckn the inbound Beckn payload +# and the response half additionally reads: +# response POCRA's answer, in its own shape +# +# WHERE THE QUERY LIVES, and why it is not in resourceAttributes. +# +# An inbound query resource is informationMode OnDemand. The search origin is +# read from the Beckn fulfillment stop and the requested type from +# supportedFacilityTypes, rather than from location/address/facilityType on +# resourceAttributes -- a convention this file chooses to keep, not a schema +# requirement: the AgricultureFacility v0.1 pack no longer forbids those +# fields under OnDemand (it did until network-specs commit b76c9ad8a5 on +# schema-packs-v0.1 quietly dropped that constraint; see +# dev_docs/schema-onDemand-forbid-removed.md). Nothing here depends on the +# constraint being back -- the search origin still has nowhere verified to go +# in resourceAttributes, since POCRA returns no per-facility coordinate for +# the searched point. +# +# PROVISIONAL. This convention was chosen on the design and has not been +# confirmed against a payload captured from the network. Confirm it before the +# registry row goes live. + +# What this capability cannot serve, refused before the provider is called. +# +# A predicate that is false refuses the request with the message beside it, so +# the caller is told what is wrong with their payload rather than that an +# expression somewhere returned false. Each check is its own expression and binds +# for itself; the first failure is the one reported. +# +# Sending an unservable payload anyway earns POCRA's own schema NACK, whose +# instanceLocation names a JSON pointer into a request the caller never saw. +required: + - check: | + ( + $geo := beckn.message.contract.commitments[0].fulfillment.stops[0].location.geo; + $exists($geo) and $geo.type = "Point" and $count($geo.coordinates) = 2 + ) + message: "this capability needs a fulfillment stop with a Point location; POCRA searches around one point at a time" + - check: | + ( + $requested := [beckn.message.contract.commitments[0].resources[0].resourceAttributes.supportedFacilityTypes]; + $governed := ["CustomHiringCentre", "KrishiVigyanKendra", "Warehouse", "SoilTestingFacility"]; + $count($requested) > 0 and $count($requested[$ in $governed]) = $count($requested) + ) + message: "every entry in supportedFacilityTypes must name a governed facility type: CustomHiringCentre, KrishiVigyanKendra, Warehouse or SoilTestingFacility" + - check: | + ( + $requested := [beckn.message.contract.commitments[0].resources[0].resourceAttributes.supportedFacilityTypes]; + $count($requested) = $count($distinct($requested)) + ) + message: "supportedFacilityTypes must not repeat a facility type" + +# ONE call per payload, and this file is written for exactly that. +# +# POCRA's search takes exactly ONE category code: a comma-separated pair +# ("kvk,warehouse") answers 200 with no providers at all, and a category array +# is refused with "Schema validation failed" -- both verified against the live +# API. So a payload asking for two facility types cannot be served by one call, +# however this file is written. +# +# The plugin handles that before this file is reached. It reads +# supportedFacilityTypes out of the payload, splits a multi-type search into +# one single-type payload per type, runs each of them concurrently and merges +# the answers -- see AgricultureFacility/search.go. So what arrives here always +# names exactly one facility type, and both halves below read it straight off +# the payload with no variable injected from Go. +# +# The cost of that, for whoever edits this file next: the required: checks +# above read supportedFacilityTypes in JSONata, and search.go reads the same +# path in Go. If the Beckn payload moves that field, BOTH change, and only +# this one is a config edit. + +# The upstream is a POST, so this object becomes the request body. +# +# This is where the extraction lives, deliberately. When POCRA wants another +# parameter it is an edit here and nothing else: no Go, no rebuild, live on the +# next cache expiry. +# +# $codes is the ONLY place POCRA's private vocabulary appears. Adding a facility +# type is one line here, one line in the inverse table below, and one more entry +# in the precondition's list. +# +# GeoJSON is [lon, lat] -- longitude first -- and POCRA's gps is "lat,lon". +# Reading them the same way round gives a point in the wrong place that is still +# a valid request, so it fails as wrong data rather than as an error. +# +# context.timestamp is epoch SECONDS as a string. Every other timestamp in both +# protocols is ISO 8601, and POCRA's schema refuses one here. +# +# domain, version, bap_id and bap_uri are POCRA's request contract rather than +# this adapter's identity. Identity on the wire is the adapter's own: it signs +# what it answers with, using the key the registry publishes for it. +request: | + ( + $commitment := beckn.message.contract.commitments[0]; + $stop := $commitment.fulfillment.stops[0]; + /* This call's own facility type. The payload names exactly one, because + the plugin split a multi-type search into one payload per type before + this ran -- reading [0] of a multi-type list here is what made such a + search return one type and silently drop the rest. */ + $requested := [beckn.message.contract.commitments[0].resources[0].resourceAttributes.supportedFacilityTypes][0]; + $codes := { + "CustomHiringCentre": "chc", + "KrishiVigyanKendra": "kvk", + "Warehouse": "warehouse", + "SoilTestingFacility": "soil_lab" + }; + { + "context": { + "domain": "advisory:mh-vistaar", + "location": { "country": { "name": "IND" } }, + "action": "search", + "version": "1.1.0", + "bap_id": "bap.mahapocra.gov.in", + "bap_uri": "https://middleware.mahapocra.gov.in/bap/", + /* This call's own id. POCRA keeps a message_id's answers for ten + minutes and returns the UNION of everything asked for under it, so + two calls sharing an id would each return the other's facilities + too. + + Read straight off the payload: the plugin stamps a fresh UUID on + context.messageId of every single-type payload it splits out, for + exactly this reason, so each call already carries its own. POCRA's + schema refuses a message_id that is not a UUID. */ + "message_id": beckn.context.messageId, + "transaction_id": beckn.context.transactionId, + "timestamp": $string($floor($millis() / 1000)) + }, + "message": { + "intent": { + "category": { "descriptor": { "code": $lookup($codes, $requested) } }, + "item": { "descriptor": { "name": "service-locations" } }, + "fulfillment": { + "stops": [ + { + "location": { + "gps": $string($stop.location.geo.coordinates[1]) & "," & $string($stop.location.geo.coordinates[0]) + }, + "time": { + "range": { + "start": $exists($stop.time.range.start) ? $stop.time.range.start : $now() + } + } + } + ] + } + } + } + } + ) + +# Keyed by direction, not by the action it produces: a select is answered by an +# on_select over the same HTTP round trip, so the callback is this half rather +# than an action of its own. +# +# POCRA answers with responses[], one entry per BPP that replied, each carrying +# its own catalog. Every facility is an item inside one of them, so the path +# flattens all four levels at once. +response: | + ( + $selected := beckn.message.contract.commitments[0]; + + /* The caller's own @context, echoed back rather than restated here. + Hardcoding the pack URL meant this file had to know which one is current, + and could contradict what the request actually declared. Not enforced by + a required check above -- an absent @context here means an absent one on + the way out too, which degrades the JSON-LD rather than breaking pack + conformance (conformance_test.go treats @context as conventional, not a + schema property). + + Backticks because @ is an operator in JSONata. */ + $ctx := $selected.resources[0].resourceAttributes.`@context`; + $resourceId := function($item) { "res:pocra:facility:" & $item.id }; + + /* The inverse of the request half's $codes. Read from the item's OWN + category tag rather than echoed from the request, so an answer carrying + more than one category still labels each facility correctly. */ + $types := { + "chc": "CustomHiringCentre", + "kvk": "KrishiVigyanKendra", + "warehouse": "Warehouse", + "soil_lab": "SoilTestingFacility", + /* The warehouse BPP labels its fulfillment GSW rather than "warehouse", + and its items carry no category tag at all, so this is the only place + a warehouse says what it is. */ + "GSW": "Warehouse" + }; + + /* EVERY type the payload asked for, as a list. The answer is filtered to + this set rather than to its first entry -- filtering to [0] is what made a + two-type search drop a type. */ + $requestedTypes := [$selected.resources[0].resourceAttributes.supportedFacilityTypes]; + + /* POCRA states a provider's facility type in the provider's own id: + COMMON_PROVIDER_KVK, COMMON_PROVIDER_CHC, COMMON_PROVIDER_SOIL_LAB. That + is the outermost and most reliable statement of what a group of items is, + because POCRA groups items under one provider per category. + + $contains first because $substringAfter returns the WHOLE string when the + separator is absent, so "WAREHOUSE001" would otherwise become the code + "warehouse001" and look like a deliberate answer rather than no answer. + Warehouse providers are named that way and state their type only in their + fulfillment, so they fall through to the sources below. */ + $providerCode := function($id) { + $contains($id, "COMMON_PROVIDER_") + ? $lowercase($substringAfter($id, "COMMON_PROVIDER_")) + }; + + /* Every facility POCRA returned, each carrying the type it actually is. + Walked per PROVIDER rather than flattened straight to items, because both + of the reliable statements of type are the provider's -- a warehouse item + carries no category tag of its own at all. + + Three sources, narrowest-trusted first: the provider's id, then the + item's own category tag, then the provider's fulfillment category. The + provider id leads because it is what POCRA groups by; the item tag is + kept as a second source so a provider that ever mixed categories still + labels each facility correctly; the fulfillment is what a warehouse has + instead of either. + + Nothing falls back to what was REQUESTED. Doing that relabels a leaked + facility as the thing being searched for, which is how a warehouse comes + back as a Krishi Vigyan Kendra. An item none of the three can type stays + untyped and is dropped by the filter below -- POCRA also returns + administrative and mandi providers, which are not facilities at all. */ + /* POCRA writes "Unknown", "N/A" and "000000" where it has no value. + Publishing them would put junk on the network under a governed schema, so + a value that is one of them is treated as absent. A ternary with no else + yields nothing, which is how a field is omitted rather than nulled. */ + $clean := function($value) { + ($exists($value) and $value != "" and $value != "Unknown" + and $value != "N/A" and $value != "000000" and $value != "-") ? $value + }; + + /* $number($substringBefore(...)) THROWS on anything it cannot parse, and + that throw is not local: it propagates out of the whole response + mapping, so one placeholder on one item's distance or capacity failed + the ENTIRE search, good facilities included. $exists() cannot guard + it -- $exists($number(...)) evaluates its argument first, so the throw + fires before $exists is ever reached. + + $clean strips the placeholders this file already knows POCRA writes + ("Unknown", "N/A", "000000", "-"); $type/$match then confirms what is + left actually looks like "" before $number ever + sees it. Same shape as $priced in agmarknet/mandi-price.select.yaml, + and for the same reason: $match() itself throws T0410 on a non-string, + so the string check has to come first rather than be folded into the + regex. + + Returns nothing when the value cannot be read as a number -- absent, + not zero, so a caller cannot mistake "not reported" for "reported as + none". */ + $numericPrefix := function($tagged) { + ( + $head := $clean($substringBefore($tagged, " ")); + $exists($head) and $type($head) = "string" and $match($head, /^[0-9]+(\.[0-9]+)?$/) + ? $number($head) + ) + }; + + $labelled := response.responses.message.catalog.providers.( + $provider := $; + $fromProvider := $lookup($types, $providerCode($provider.id)); + /* NOT a single $lookup($types, ....descriptor.code): a provider with more + than one fulfillment, or a fulfillment with more than one category, + makes descriptor.code a SEQUENCE rather than a string, and $lookup + throws T0410 on anything but a scalar key. One bad provider shape then + took the whole response down, all items included -- not just the ones + with more than one category. + + Each code is looked up on its own instead, and the first that resolves + to a governed type wins. A provider that never mixes categories still + gets exactly the $lookup($types, code) it had before, since a + one-element sequence maps to a one-element result. */ + $fromFulfillment := ( + $resolved := [$provider.fulfillments.categories.descriptor.code].($lookup($types, $)); + $resolved[$exists($)][0] + ); + $map($provider.items, function($item) { + { + "item": $item, + "type": ( + /* Same T0410 hazard as $fromFulfillment above: an item carrying + more than one category tag makes .value a SEQUENCE, and $lookup + throws on anything but a scalar key. Same fix -- look up each + code on its own and take the first that resolves. */ + $tagged := ( + $resolved := [$item.tags.list[descriptor.code = "category"].value].($lookup($types, $)); + $resolved[$exists($)][0] + ); + $exists($fromProvider) ? $fromProvider + : $exists($tagged) ? $tagged + : $fromFulfillment + ) + } + }) + ); + + /* A search is answered with the types it asked for and nothing else, + whatever POCRA chose to include. + + It includes more than was asked for, verified against the live API: it + caches per message_id for PT10M and ACCUMULATES across searches sharing + one, so a kvk search under a reused message_id comes back carrying the chc + facilities of an earlier one. The step now sends a fresh message_id per + call, which stops this adapter causing it; the filter stays because a + provider deciding what to include is not something a mapping should + trust either way. */ + $wanted := $labelled[type in $requestedTypes]; + + /* The same facility appears in more than one responses[] entry, so grouping + by id keeps one of each; [0] takes the first where a key collected + several. A repeated id in the answer would be a dangling reference the + moment anything resolved it. */ + $unique := $each($wanted{ item.id: $ }, function($grouped) { $grouped[0] }); + + /* POCRA ranks by distance and reports it only as a string with a unit -- + "165 Km". It is parsed to sort by and then dropped: the pack states that + query-relative distance is not an intrinsic facility attribute, and + belongs in result metadata rather than on the facility. + + Sorting the strings would put "165 Km" before "98 Km", which nothing + downstream could detect. An item with no distance sorts last, which is + what an absent ranking should mean. */ + $reading := function($item) { + ( + /* A duplicated distance tag makes .value a sequence, and + $numericPrefix's $substringBefore throws on anything but a string + before the $type guard inside it is ever reached. Take the first. */ + $values := [$item.tags.list[descriptor.code = "distance"].value]; + $numericPrefix($values[0]) + ) + }; + $sorted := $map($unique, function($entry) { + ( + $distance := $reading($entry.item); + { + "facility": $entry.item, + "type": $entry.type, + "distance": $exists($distance) ? $distance : 1e9 + } + ) + })^(distance); + + /* POCRA reports a warehouse's capacity as one string with a unit -- + "500 tons" -- and reports it for no other facility type. The pack has a + home for it, so dropping it would discard the most useful thing a + warehouse search returns. + + No basis: the pack's example says "total storage", but POCRA does not say + which it means and guessing would put an unverified claim under a governed + field. */ + $capacity := function($item) { + ( + /* A duplicated capacity_estimate tag makes .value a sequence -- same + T0410 hazard as distance, above. Take the first. */ + $values := [$item.tags.list[descriptor.code = "capacity_estimate"].value]; + $reported := $values[0]; + /* value guarded the same way distance is, above -- "Unknown" or any + other placeholder must not reach $number. unit is left unguarded + deliberately: it is only ever read when value parsed, and at that + point substringAfter on the same string that just matched the + numeric-prefix regex cannot itself throw. */ + $value := $numericPrefix($reported); + $exists($value) ? { + "value": $value, + "unit": $substringAfter($reported, " ") + } + ) + }; + + /* A Beckn Address. streetAddress is POCRA's one real address line; district, + taluka and village go to extendedAddress, which is where the pack's own + examples put them. addressRegion is Maharashtra because POCRA is that + state's aggregator and publishes nothing outside it. */ + $address := function($item) { + ( + $parts := [$clean($item.address.district), $clean($item.address.taluka), $clean($item.address.vilage)]; + { + "streetAddress": $clean($item.address.address), + "extendedAddress": $count($parts) > 0 ? $join($parts, ", "), + "postalCode": $clean($item.address.pinCode), + "addressRegion": "Maharashtra", + "addressCountry": "IN" + } + ) + }; + + /* Organisational contact only. The pack restricts publicContact to contact + data approved for catalog publication, and POCRA's person field is an + office rather than an individual. Omitted entirely when there is nothing + usable, rather than emitted empty. */ + $contact := function($item) { + ( + $company := $clean($item.contact.person); + $phone := $clean($item.contact.phone); + $email := $clean($item.contact.email); + ($exists($company) or $exists($phone) or $exists($email)) ? { + "company": $company, + "phone": $phone, + "email": $email + } + ) + }; + + $attributes := function($item, $facilityType) { + { + "@context": $ctx, + "@type": "openagrinet:AgricultureFacility", + "informationMode": "Direct", + /* REQUIRED by AgricultureResource v0.1, and the AgricultureFacility + pack additionally requires this list to CONTAIN "Facility". It is + the pack's broad discovery category, deliberately coarse: the + concrete kind is facilityType below. + + "Facility" alone. The pack permits further values to name the + agricultural domains a facility serves (Crop, Livestock, ...), and + POCRA states nothing about that for any item -- inventing one would + publish a guess as a governed discovery category. */ + "subjectCategories": ["Facility"], + "facilityType": $facilityType, + "address": $address($item), + "capacity": $capacity($item), + /* sourceUri rather than website: contact.webUrl is POCRA's own site + repeated on every item, so it identifies the source and not the + facility. */ + "source": { + "sourceId": "pocra", + "sourceName": "PoCRA Provider Aggregator Services", + "sourceUri": $clean($item.contact.webUrl) + }, + "publicContact": $contact($item) + } + }; + { + /* Correlation only: the ids that tie this answer to the request that + asked for it, and nothing that asserts who anybody is. + + bapId, bapUri, bppId and bppUri are deliberately absent. A mapping is a + payload transformation -- it has no business asserting network identity, + and the two Uri fields it could copy are whatever the caller happened to + send, which in a deployed stack is a container-internal address that + means nothing to anyone outside it. */ + "context": { + "version": beckn.context.version, + "action": "on_select", + "networkId": beckn.context.networkId, + "transactionId": beckn.context.transactionId, + "messageId": beckn.context.messageId, + "timestamp": $now() + }, + "message": { + "contract": { + "commitments": [ + { + /* DRAFT, not QUOTED. The Beckn v2 status enum is DRAFT, ACTIVE and + CLOSED, and a quote is still a draft: nothing is committed until + init and confirm. QUOTED read better and validated nowhere. */ + "status": { + "descriptor": { "code": "DRAFT", "name": "Draft" } + }, + /* The offer is echoed, but its references are not: the request + named an abstract facility search, and the answer returns the + concrete facilities that satisfy it. Leaving resourceIds as they + arrived would point the offer at an id that appears nowhere in + the answer. + + $merge keeps everything else the request offered -- the id, the + descriptor, the provider -- and replaces one key. */ + "offer": $merge([ + $selected.offer, + { "resourceIds": [$map($sorted, function($entry) { $resourceId($entry.facility) })] } + ]), + /* Wrapped in [] because JSONata collapses a one-element sequence + to a bare value, so a single facility would answer with an + object where every other count answers with a list. */ + "resources": [$map($sorted, function($entry) { + { + "id": $resourceId($entry.facility), + /* Required by Commitment.resources in the spec, which defines + no quantity property and no Quantity schema anywhere -- a + defect upstream. One resource is one facility, so one. */ + "quantity": 1, + /* Direct mode requires @type, informationMode, facilityType + and source, plus one of location or address. + + location is deliberately absent: POCRA returns no verified + per-facility coordinate, and the pack forbids substituting + the search origin for one. + + services, capacity, website and lastUpdatedAt are absent for + the same reason -- POCRA supplies none of them, and deriving + them from the facility type, or lastUpdatedAt from the time + of the fetch, would assert something nobody verified. */ + "resourceAttributes": $attributes($entry.facility, $entry.type) + } + })] + } + ] + } + } + } + ) diff --git a/config/provider-adapter.yaml b/config/provider-adapter.yaml index f6c798c9..8d1c7450 100644 --- a/config/provider-adapter.yaml +++ b/config/provider-adapter.yaml @@ -435,6 +435,33 @@ modules: # something that no longer parses. maxResponseBytes: 1048576 + - id: AgricultureFacility + config: + # One capability for all four governed facility types. Which one a + # request wants is a value in its payload -- supportedFacilityTypes + # -- not part of the binding key, so there is one row in the + # registry and one mapping however many types are in use. + bindingKeys: "pocra|openagrinet:AgricultureFacility" + + # POCRA's search takes no credential, which is also why its + # baseUrl can be published as it stands. Auth is per provider, in + # a block named for the participant id -- declared even when the + # scheme is none, because a served provider without a block is + # refused at startup. + pocra: + authScheme: none + + # internal/common's default is 1 (sequential), deliberately: + # POCRA's failure mode when pushed is a 200 with an EMPTY catalog, + # indistinguishable from "no results", so a parallel search can + # silently lose a facility type with no error anywhere, with + # nothing in the payload recording that a type was asked for and + # lost. Kept at 1 here until that's verified against the live + # API -- raise it in a follow-up once confirmed safe, to at most + # 4, which is every governed type at once and so full + # concurrency for this capability. + searchConcurrency: 1 + # Declaring a step above is not enough: THIS list is what runs. A step # that appears under providerSteps but not here never executes, and the # request falls through to the 404 above -- which looks like a registry @@ -445,6 +472,7 @@ modules: - WeatherObservation # its binding key, or pass through - MandiPrice # its binding key, or pass through - KnowledgeAdvisory # its binding key, or pass through + - AgricultureFacility # its binding key, or pass through - signAck # signs whatever the step answered with # ---------------------------------------------------------------------------- diff --git a/install/build-plugins.sh b/install/build-plugins.sh index 59e5b817..4e505cc0 100755 --- a/install/build-plugins.sh +++ b/install/build-plugins.sh @@ -35,6 +35,7 @@ plugins=( "WeatherObservation" "MandiPrice" "KnowledgeAdvisory" + "AgricultureFacility" "manifestloader" "reqpreprocessor" "otelsetup" diff --git a/pkg/plugin/implementation/AgricultureFacility/AgricultureFacility.go b/pkg/plugin/implementation/AgricultureFacility/AgricultureFacility.go new file mode 100644 index 00000000..17f1e8c8 --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/AgricultureFacility.go @@ -0,0 +1,104 @@ +// Package AgricultureFacility serves the openagrinet:AgricultureFacility +// capability. +// +// One package per capability, named for the capability it serves, so which +// plugin owns one is readable from its binding key. +// +// Named for the capability and not for POCRA, deliberately. A provider is a +// registry row, and more than one could serve this same capability -- a second +// state aggregator would be another row and another mapping, not another +// package. +// +// Almost nothing lives here, and that is the point. Recognising a capability, +// resolving the call plan, authenticating, calling with the registry's budget +// and translating in both directions are all internal/upstream's, because none +// of them differ by domain. What this package owns is its name, and +// prerequisites -- the work a mapping cannot express, which is domain knowledge +// by definition. +// +// The upstream this was written against is POCRA's aggregator, whose search +// takes a category code and a point, both of which an AgricultureFacility +// payload carries. So the package is a name and nothing else: see +// prerequisites.go for why that is worth stating. +package AgricultureFacility + +import ( + "context" + + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" +) + +// Config carries everything common.Config does, plus this capability's own +// search concurrency -- which common.Config has no field for, because how +// many calls one payload becomes is not something that package knows about at +// all. A flat struct with common.Config's fields repeated rather than an +// alias (which this package used to be, and MandiPrice and WeatherObservation +// still are) or an embedded common.Config (which would break every existing +// flat struct literal, `&Config{BindingKeys: ..., AuthScheme: ...}`, since +// Go's composite literal syntax does not promote an embedded struct's fields +// the way a selector expression does). +type Config struct { + BindingKeys []string `yaml:"bindingKeys" json:"bindingKeys"` + ProviderIDAt string `yaml:"providerIdAt" json:"providerIdAt"` + CapabilityCodeAt string `yaml:"capabilityCodeAt" json:"capabilityCodeAt"` + MaxResponseBytes int64 `yaml:"maxResponseBytes" json:"maxResponseBytes"` + + // AuthByProvider carries one credential profile per provider, keyed by + // participant id. Passed through to the inner step, which is where the + // schemes are defined and validated -- see common.AuthProfile. + AuthByProvider map[string]*common.AuthProfile `yaml:"-" json:"-"` + + // SearchConcurrency is how many of a multi-type search's calls may be in + // flight at once. See search.go's DefaultSearchConcurrency and + // MaxFacilityTypes for what absent and too-large mean. + SearchConcurrency int `yaml:"searchConcurrency" json:"searchConcurrency"` +} + +// New creates the agriculture facility step. +// +// Which capabilities it answers to is configuration, with no default: a package +// serving a family cannot guess which of them a deployment has providers for. +// +// Two steps, one returned. The inner one is internal/upstream's, which serves +// one payload with one call and knows nothing about facility types. The outer +// one is this package's own (see search.go): it splits a multi-type search +// into one single-type payload per type, runs the inner step over each of them +// concurrently, and merges the answers. Everything POCRA-specific about that +// is in the outer step, which is why the inner one is the same step +// MandiPrice and WeatherObservation use unchanged. +func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper definition.Mapper, + cfg *Config) (definition.Step, func() error, error) { + if cfg == nil { + cfg = &Config{} + } + + upstreamCfg := &common.Config{ + BindingKeys: cfg.BindingKeys, + ProviderIDAt: cfg.ProviderIDAt, + CapabilityCodeAt: cfg.CapabilityCodeAt, + MaxResponseBytes: cfg.MaxResponseBytes, + AuthByProvider: cfg.AuthByProvider, + } + + one, closer, err := common.New(ctx, registry, mapper, prerequisites, upstreamCfg) + if err != nil { + return nil, nil, err + } + + // The same paths the inner step resolved from the same config, so both + // answer "is this payload mine?" identically. Resolved through upstream + // rather than duplicated here: a second reading of the same two config + // fields could drift from the first. + paths, err := common.BindingPaths(upstreamCfg) + if err != nil { + return nil, nil, err + } + + return &Step{ + inner: one, + paths: paths, + bindingKeys: cfg.BindingKeys, + concurrency: searchConcurrency(cfg.SearchConcurrency), + }, closer, nil +} diff --git a/pkg/plugin/implementation/AgricultureFacility/README.md b/pkg/plugin/implementation/AgricultureFacility/README.md new file mode 100644 index 00000000..f1541eb2 --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/README.md @@ -0,0 +1,187 @@ +# Agriculture Facility Plugin + +A **provider step plugin** that serves `openagrinet:AgricultureFacility` by +calling an ordinary HTTP API that has never heard of Beckn. + +Today that API is POCRA's aggregator, which answers +`pocra|openagrinet:AgricultureFacility` for the `select` action. + +## What lives here + +Almost nothing. Recognising a capability, resolving the call plan, +authenticating, calling with the registry's budget and translating in both +directions are all `internal/upstream`'s. This package owns its name and its +prerequisites — which are empty, because a facility search names the point and +the facility type it wants, and POCRA's search takes exactly those. + +Named for the schema-pack family rather than for POCRA: a provider is a registry +row, and a second state aggregator would be another row and another mapping, not +another package. + +## Configuration + +```yaml +providerSteps: + - id: AgricultureFacility + config: + bindingKeys: "pocra|openagrinet:AgricultureFacility" + authScheme: none + searchConcurrency: 4 +``` + +## How a multi-type search is served + +POCRA's search takes exactly ONE category code: a comma-separated pair answers +200 with no providers at all, and a category array is refused outright, both +verified against the live API. A Beckn payload asking for three facility types +therefore has to become three calls. + +That is this package's own job, and all of it lives here: + +- `search.go` is the step the adapter runs. It reads the facility types out of + the payload, splits one payload into one single-type payload per type -- each + with a fresh `context.messageId`, because POCRA returns the union of + everything asked for under one id -- runs the ordinary upstream step over + each part concurrently, and merges the answers into one. +- `payload.go` is the one path read this package does in Go rather than in a + mapping, and the one place `supportedFacilityTypes` is located. Its own + comment says what that costs. +- `internal/upstream` serves one payload with one call and knows none of this. + `jsonmapper` compiles the two halves every mapping has and no third thing. + `internal/concurrent` runs N of anything, bounded and ordered, and has never + heard of Beckn. + +**Ordering, worth knowing:** within one facility type the mapping ranks by +POCRA's distance, and that ranking survives the merge. ACROSS types the answer +is type-blocked -- every KrishiVigyanKendra, then every Warehouse -- rather +than globally nearest-first, because the schema pack says query-relative +distance is not a facility attribute, so the mapping drops it before the merge +could sort on it. + +| Parameter | Required | Description | Default | +|-----------|----------|-------------|---------| +| `bindingKeys` | **Yes** | Comma-separated capabilities this step answers to. No default is possible: a package serving a family cannot guess which of them a deployment has providers for. | — | +| `providerIdAt` | No | Path override for where the provider-id half of a binding key sits in a payload. Beckn v2 convention if absent. | Beckn v2 convention | +| `capabilityCodeAt` | No | Path override for the capability-code half. Must be given together with `providerIdAt`. | Beckn v2 convention | +| `authScheme` | No | `none`, `basic`, `header` or `query`. POCRA needs none. | `none` | +| `maxResponseBytes` | No | Cap on what is read from the provider. | 4 MiB | +| `searchConcurrency` | No | How many of a multi-type search's calls run at once, up to `MaxFacilityTypes` (8, this package's own constant -- see `search.go`). 4 is every governed type at once -- full concurrency for this capability. **Trade-off:** defaults to 1 (sequential) because POCRA's failure mode when pushed is a 200 with an *empty* catalog, indistinguishable from "no results" -- a parallel search can silently drop a facility type with no error. Verify against the live API before raising it in production. | 1 (sequential) | + +This package's `Config` (`AgricultureFacility.go`, a flat struct of its own because it carries `searchConcurrency`, which `upstream.Config` has no field for) also has the `basic`/`header`/`query` auth credential pairs (`usernameEnv`/`passwordEnv`, `headerName`/`headerValueEnv`, `queryName`/`queryValueEnv`). This plugin's `parseConfig` does not wire them through -- POCRA needs none of them. A second provider on `openagrinet:AgricultureFacility` (see "What lives here") that needs one adds the corresponding line to `parseConfig`, mirroring `maxResponseBytes`. + +The id must also appear in the module's `steps:` list, and must be unique across +`steps` and `providerSteps` — a repeat is refused at startup, because both land +in one id-keyed map and one capability would otherwise be lost silently. + +## Registry rows + +Two, joined on `participantId`. + +```json +{ "participantId": "pocra", "name": "PoCRA Provider Aggregator", + "type": "upstream", "status": "active", + "baseUrl": "https://middleware-bap-client.mahapocra.gov.in" } +``` +```json +{ "bindingKey": "pocra|openagrinet:AgricultureFacility", + "participantId": "pocra", + "capabilityCode": "openagrinet:AgricultureFacility", + "status": "active", + "actions": [ { "action": "select", "method": "POST", "path": "/search", + "mappings": "/agriculture-facility.select.yaml", + "timeoutMs": 30000, "retryMax": 2, "status": "active" } ] } +``` + +`retryMax` is 2. The step marks a 4xx other than 429 as permanent and stops +retrying it, so a schema NACK caused by our own malformed request costs one +attempt rather than three. What the budget buys is resilience against a 5xx, a +429 or a transport failure, backing off exponentially from 50ms. + +## Facility types + +The governed enum maps one-to-one onto POCRA's category codes. The translation +lives in the mapping and nowhere else. + +| `FacilityType` | POCRA code | +|---|---| +| `CustomHiringCentre` | `chc` | +| `KrishiVigyanKendra` | `kvk` | +| `Warehouse` | `warehouse` | +| `SoilTestingFacility` | `soil_lab` | + +Adding a type is three lines in +`config/mappings/pocra/agriculture-facility.select.yaml` — the forward table in +the request half, the inverse in the response half, and the governed value in the +precondition's list. No rebuild. + +## Where the query lives + +An inbound query resource is `informationMode: OnDemand`. The search origin is +read from `message.contract.commitments[].fulfillment.stops[].location.geo` +and the requested type from `resourceAttributes.supportedFacilityTypes`, +rather than from `location`/`address`/`facilityType` on `resourceAttributes` -- +a convention this plugin keeps, not a schema requirement. The pack forbade +those fields under OnDemand until `network-specs` commit `b76c9ad8a5` on +`schema-packs-v0.1` dropped that constraint (see +`dev_docs/schema-onDemand-forbid-removed.md`); the plugin's behavior did not +change when that happened, since POCRA has no verified per-facility +coordinate to put there anyway. + +**This convention is provisional.** It was chosen on design and has not been +confirmed against a payload captured from the network. + +## What the answer deliberately omits + +`location`, because POCRA returns no verified per-facility coordinate — the one +`gps` in its response is a fixed stub unrelated to the point that was asked for — +and the pack forbids substituting the search origin. + +`services`, `capacity`, `website` and `lastUpdatedAt`, because POCRA supplies +none of them. Deriving services from the facility type, or `lastUpdatedAt` from +the time of the fetch, would assert something nobody verified. + +Distance, which POCRA does supply, is used to order the resources nearest-first +and then dropped: the pack states that query-relative distance is not an +intrinsic facility attribute and belongs in result metadata. + +`"Unknown"`, `"N/A"`, `"000000"` and `"-"` are treated as absent wherever POCRA +sends them, so a field is omitted rather than published as a placeholder. The +warehouse BPP uses `"-"` for a phone it does not have; the other three use +`"N/A"`. + +`@context` is echoed from the request rather than restated here, so this mapping +does not have to know which pack identifier is current and cannot contradict +what the caller declared. Which matters: the identifier the packs name in their +own `x-jsonld`, `https://schemas.openagrinet.global/…`, has no DNS record, so a +deployment tracking the published ref sends the `raw.githubusercontent.com` pack +URL instead. Both are exercised. + +## Testing + +```sh +go test ./pkg/plugin/implementation/AgricultureFacility/... +``` + +25 pass, 2 skip. The two skips are live tests against the real POCRA API, opted +into with `POCRA_LIVE=1`. + +`mappings_test.go` runs the shipped mapping through the real mapper and the real +step against a fake POCRA. It reads from `config/mappings/pocra/` rather than +from a fixture, so it breaks when what is deployed breaks. + +`conformance_test.go` validates the answer against +`openagrinet:AgricultureFacility v0.1` with a real JSON Schema validator. It +also validates the pack's own published examples — if those fail, the +compilation is wrong and nothing else in that file means anything — and covers +the two things the schema is silent on: fields the pack does not declare, and +the README's prose mapping rules. + +The schemas are not vendored. `schemacache_test.go` compiles the pack under the +URL that publishes it, + + https://github.com/OpenAgriNet/network-specs/tree/schema-packs-v0.1/schema/AgricultureFacility/v0.1 + +lets the validator resolve the pack's own `$ref`s, and caches each document +under `testdata/schema-cache/` (gitignored). A cold run needs the network; every +run after it is offline. With an empty cache **and** no network the schema tests +skip rather than fail. To refresh, delete the cache directory and re-run. diff --git a/pkg/plugin/implementation/AgricultureFacility/cmd/plugin.go b/pkg/plugin/implementation/AgricultureFacility/cmd/plugin.go new file mode 100644 index 00000000..25a13ef3 --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/cmd/plugin.go @@ -0,0 +1,120 @@ +// Command plugin builds the agriculture facility provider step as a loadable +// plugin. +// +// The filename of the built .so is the id a deployment names in providerSteps, +// so this package is AgricultureFacility's whole public surface: a config map +// in, a step out. +package main + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/beckn-one/beckn-onix/pkg/log" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/AgricultureFacility" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" +) + +// agriFacilityProvider implements definition.ProviderStepProvider. +type agriFacilityProvider struct{} + +// newStepFunc creates a new step. Indirected for tests. +var newStepFunc = AgricultureFacility.New + +// parseConfig turns the plugin config map into a typed Config. Anything absent +// is left zero: AgricultureFacility.New applies the defaults and validates the auth +// scheme, so those rules live in one place. +func (p agriFacilityProvider) parseConfig(config map[string]string) (*AgricultureFacility.Config, error) { + cfg := &AgricultureFacility.Config{ + BindingKeys: splitList(config["bindingKeys"]), + // Absent means the Beckn v2 convention. See common.Config for why + // this is a default rather than something to set. + ProviderIDAt: config["providerIdAt"], + CapabilityCodeAt: config["capabilityCodeAt"], + } + + // One credential profile per provider, read from the flattened + // authScheme- settings. Shared with the other capability + // plugins so a scheme added in one place is not missed in three. + auth, err := common.ParseProviderAuth(config) + if err != nil { + return nil, err + } + cfg.AuthByProvider = auth + + if raw, exists := config["maxResponseBytes"]; exists && raw != "" { + value, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid maxResponseBytes value '%s': %w", raw, err) + } + if value <= 0 { + return nil, fmt.Errorf("maxResponseBytes must be positive, got %d", value) + } + cfg.MaxResponseBytes = value + } + + if raw, exists := config["searchConcurrency"]; exists && raw != "" { + value, err := strconv.Atoi(raw) + if err != nil { + return nil, fmt.Errorf("invalid searchConcurrency value '%s': %w", raw, err) + } + if value <= 0 { + return nil, fmt.Errorf("searchConcurrency must be positive, got %d", value) + } + cfg.SearchConcurrency = value + } + + return cfg, nil +} + +// New creates a new agriculture facility provider step instance. +func (p agriFacilityProvider) New(ctx context.Context, registry definition.ProviderRecordLookup, mapper definition.Mapper, config map[string]string) (definition.Step, func() error, error) { + if ctx == nil { + return nil, nil, errors.New("context cannot be nil") + } + + cfg, err := p.parseConfig(config) + if err != nil { + log.Errorf(ctx, err, "Failed to parse agriculture facility configuration") + return nil, nil, fmt.Errorf("failed to parse agriculture facility configuration: %w", err) + } + + step, closer, err := newStepFunc(ctx, registry, mapper, cfg) + if err != nil { + log.Errorf(ctx, err, "Failed to create agriculture facility step") + return nil, nil, err + } + + log.Infof(ctx, "Agriculture facility step created successfully") + return step, closer, nil +} + +// splitList reads a comma-separated config value, which is how a list reaches a +// plugin -- the config is map[string]string. Blanks are dropped and spaces +// trimmed, so a trailing comma or a wrapped line is not a config error. +// +// A comma is unambiguous here: a binding key separates its own halves with a +// pipe. +func splitList(raw string) []string { + if strings.TrimSpace(raw) == "" { + return nil + } + var out []string + for _, part := range strings.Split(raw, ",") { + if trimmed := strings.TrimSpace(part); trimmed != "" { + out = append(out, trimmed) + } + } + return out +} + +// Provider is the exported plugin instance. +var Provider = agriFacilityProvider{} + +// Compile-time proof the provider satisfies the interface the manager asserts +// against. A mismatch is otherwise a runtime cast failure at startup. +var _ definition.ProviderStepProvider = Provider diff --git a/pkg/plugin/implementation/AgricultureFacility/cmd/plugin_test.go b/pkg/plugin/implementation/AgricultureFacility/cmd/plugin_test.go new file mode 100644 index 00000000..7b1e733a --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/cmd/plugin_test.go @@ -0,0 +1,199 @@ +package main + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/AgricultureFacility" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" +) + +type stubRegistry struct{} + +func (stubRegistry) ProviderRecord(context.Context, string) (*model.ProviderRecord, error) { + return nil, nil +} + +type stubMapper struct{} + +func (stubMapper) Verify(context.Context, string, any) error { return nil } + +func (stubMapper) Transform(context.Context, string, definition.Direction, any) ([]byte, error) { + return nil, nil +} + +func TestParseConfig(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + config map[string]string + expected *AgricultureFacility.Config + expectedErr string + }{ + { + // Everything absent is left zero: AgricultureFacility.New defaults it, so + // the rules are defined in exactly one place. + name: "leaves everything unset for New to default", + config: map[string]string{}, + expected: &AgricultureFacility.Config{AuthByProvider: map[string]*common.AuthProfile{}}, + }, + { + // Every key this plugin reads. Auth is per provider, in settings + // named for the participant id, which pkg/plugin flattens from the + // nested block an operator writes. + name: "reads every supported setting", + config: map[string]string{ + "bindingKeys": "pocra|openagrinet:AgricultureFacility", + "providerIdAt": "who.provider", + "capabilityCodeAt": "what[].type", + "authScheme-pocra": "none", + "maxResponseBytes": "2048", + "searchConcurrency": "4", + }, + expected: &AgricultureFacility.Config{ + BindingKeys: []string{"pocra|openagrinet:AgricultureFacility"}, + ProviderIDAt: "who.provider", + CapabilityCodeAt: "what[].type", + AuthByProvider: map[string]*common.AuthProfile{ + "pocra": {Provider: "pocra", Scheme: "none"}, + }, + MaxResponseBytes: 2048, + SearchConcurrency: 4, + }, + }, + { + name: "reads several binding keys and trims them", + config: map[string]string{ + "bindingKeys": "pocra|openagrinet:AgricultureFacility, other|openagrinet:Thing ,", + }, + expected: &AgricultureFacility.Config{ + BindingKeys: []string{ + "pocra|openagrinet:AgricultureFacility", + "other|openagrinet:Thing", + }, + AuthByProvider: map[string]*common.AuthProfile{}, + }, + }, + { + name: "reads the binding key path override", + config: map[string]string{ + "providerIdAt": "who.provider", + "capabilityCodeAt": "what[].type", + }, + expected: &AgricultureFacility.Config{ + ProviderIDAt: "who.provider", + CapabilityCodeAt: "what[].type", + AuthByProvider: map[string]*common.AuthProfile{}, + }, + }, + { + name: "refuses a non-numeric maxResponseBytes", + config: map[string]string{"maxResponseBytes": "big"}, + expectedErr: "invalid maxResponseBytes", + }, + { + name: "refuses a non-positive maxResponseBytes", + config: map[string]string{"maxResponseBytes": "0"}, + expectedErr: "must be positive", + }, + { + name: "refuses a non-numeric searchConcurrency", + config: map[string]string{"searchConcurrency": "big"}, + expectedErr: "invalid searchConcurrency", + }, + { + name: "refuses a non-positive searchConcurrency", + config: map[string]string{"searchConcurrency": "0"}, + expectedErr: "must be positive", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got, err := agriFacilityProvider{}.parseConfig(tc.config) + if tc.expectedErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.expectedErr) { + t.Fatalf("error = %v, want it to mention %q", err, tc.expectedErr) + } + return + } + if err != nil { + t.Fatalf("parseConfig() returned an unexpected error: %v", err) + } + if !reflect.DeepEqual(got, tc.expected) { + t.Errorf("config = %+v, want %+v", got, tc.expected) + } + }) + } +} + +func TestNewRefusesANilContext(t *testing.T) { + t.Parallel() + + step, closer, err := agriFacilityProvider{}.New(nil, stubRegistry{}, stubMapper{}, map[string]string{}) //nolint:staticcheck // passing nil is what this asserts is refused + if err == nil { + t.Fatal("expected a nil context to be refused") + } + if step != nil || closer != nil { + t.Error("a refused construction must return neither a step nor a closer") + } +} + +func TestNewPropagatesAConstructionFailure(t *testing.T) { + original := newStepFunc + defer func() { newStepFunc = original }() + newStepFunc = func(context.Context, definition.ProviderRecordLookup, definition.Mapper, + *AgricultureFacility.Config) (definition.Step, func() error, error) { + return nil, nil, errors.New("boom") + } + + _, _, err := agriFacilityProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, + map[string]string{"bindingKeys": "pocra|openagrinet:AgricultureFacility"}) + if err == nil || !strings.Contains(err.Error(), "boom") { + t.Fatalf("error = %v, want it to carry the construction failure", err) + } +} + +// bindingKeys has no default, because a package serving a family cannot guess +// which of them a deployment has providers for. Absent must fail at startup +// rather than leave a step that answers nothing. +func TestNewRequiresBindingKeys(t *testing.T) { + t.Parallel() + + _, _, err := agriFacilityProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, + map[string]string{}) + if err == nil || !strings.Contains(err.Error(), "bindingKeys") { + t.Fatalf("error = %v, want it to name bindingKeys", err) + } +} + +func TestNewBuildsAStep(t *testing.T) { + t.Parallel() + + step, closer, err := agriFacilityProvider{}.New(context.Background(), stubRegistry{}, stubMapper{}, + map[string]string{ + "bindingKeys": "pocra|openagrinet:AgricultureFacility", + // A served provider declares its own auth, even when it is none. + "authScheme-pocra": "none", + }) + if err != nil { + t.Fatalf("New() returned an unexpected error: %v", err) + } + if step == nil { + t.Fatal("New() returned no step") + } + if closer == nil { + t.Fatal("New() returned no closer") + } + if err := closer(); err != nil { + t.Errorf("closer() returned %v", err) + } +} diff --git a/pkg/plugin/implementation/AgricultureFacility/conformance_test.go b/pkg/plugin/implementation/AgricultureFacility/conformance_test.go new file mode 100644 index 00000000..4eaecfdc --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/conformance_test.go @@ -0,0 +1,462 @@ +package AgricultureFacility_test + +// conformance_test.go validates what the mapping produces against the +// openagrinet:AgricultureFacility v0.1 schema pack, with a real JSON Schema +// validator. +// +// Every other test in this package asserts what the pack requires by hand, from +// a reading of attributes.yaml. That is the weakest link in the suite: a +// misreading produces a test that agrees with the mistake. This one hands the +// answer and the published schema to a validator and lets it decide. +// +// The schemas are fetched from the ref they are published under and cached +// under testdata/schema-cache, so this is offline and deterministic after a +// cold run -- see schemacache_test.go for the loader and how to refresh. + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/santhosh-tekuri/jsonschema/v6" +) + +// facilitySchemaURI is the AgricultureFacility schema inside its OpenAPI +// document, addressed by the URL that actually serves it. +// +// The URI matters more than the bytes: the packs $ref each other by URL, and a +// validator resolves a ref against the base URI of the document it is written +// in. AgricultureFacility refs "../../AgricultureResource/v0.1/attributes.yaml" +// relative to its own base, so compiling under the published URL is what makes +// that resolve -- and, unlike the schemas.openagrinet.global path this used to +// be registered under, it is an address something answers. +// +// The packs are OpenAPI 3.1.1, whose schema objects are JSON Schema 2020-12, so +// a validator can compile one directly out of components.schemas. +const facilitySchemaURI = facilityPackURL + "#/components/schemas/AgricultureFacility" + +// facilitySchema compiles the pack once per test. +func facilitySchema(t *testing.T) *jsonschema.Schema { + t.Helper() + + loader := newPackLoader() + compiler := jsonschema.NewCompiler() + compiler.DefaultDraft(jsonschema.Draft2020) + compiler.UseLoader(jsonschema.SchemeURLLoader{"https": loader}) + + schema, err := compiler.Compile(facilitySchemaURI) + if err != nil { + loader.skipIfOffline(t, err) + t.Fatalf("could not compile the AgricultureFacility schema: %v", err) + } + + // Both OAN documents have to have been loaded. A compile that satisfied + // every ref without reading AgricultureResource would mean the composed + // half of the schema went unenforced, and every result below would be + // measured against half a pack. + for _, required := range []string{facilityPackURL, resourcePackURL} { + if !loader.sawLoaded(required) { + t.Fatalf("the schema compiled without loading %s -- the pack is not the one "+ + "being validated against", required) + } + } + return schema +} + +// validateFacilities checks every resource in an answer against the pack, and +// reports the validator's own explanation when one fails. +func validateFacilities(t *testing.T, schema *jsonschema.Schema, answer map[string]any) int { + t.Helper() + + resources := answerResources(t, answer) + if len(resources) == 0 { + t.Fatal("the answer carries no resources to validate") + } + for _, entry := range resources { + id, _ := dig(entry, "id").(string) + attributes := dig(entry, "resourceAttributes") + if attributes == nil { + t.Fatalf("resource %s carries no resourceAttributes", id) + } + if err := schema.Validate(attributes); err != nil { + // The validator's message names the failing keyword and the + // instance location, which is the whole reason for this test. + t.Errorf("resource %s does not satisfy AgricultureFacility v0.1:\n%v\n%s", + id, err, mustIndent(t, attributes)) + } + } + return len(resources) +} + +func TestAnswerSatisfiesTheSchemaPack(t *testing.T) { + schema := facilitySchema(t) + + for _, tc := range []struct { + name string + facilityType string + provider string + }{ + // The three that carry their own category tag. + {"a KVK search", "KrishiVigyanKendra", providerResponse}, + // The one that does not, and whose type is derived from its provider's + // GSW fulfillment. This is the case that was silently omitting a + // REQUIRED field, so it is the reason this test exists. + {"a warehouse search", "Warehouse", warehouseResponse}, + // A mixed answer, so the filtered survivors are validated too. + {"a filtered mixed answer", "CustomHiringCentre", mixedResponse}, + } { + t.Run(tc.name, func(t *testing.T) { + answer := runAgainst(t, requestFor(t, tc.facilityType), tc.provider) + count := validateFacilities(t, schema, answer) + t.Logf("%d %s facilities validated against AgricultureFacility v0.1", + count, tc.facilityType) + }) + } +} + +// A resource missing facilityType must be REFUSED by the validator. Without +// this, a green suite would not distinguish "the pack is satisfied" from "the +// validator is not actually checking anything". +func TestTheValidatorRefusesAnIncompleteFacility(t *testing.T) { + schema := facilitySchema(t) + + answer := runAgainst(t, requestFor(t, "Warehouse"), warehouseResponse) + attributes, ok := dig(answerResources(t, answer)[0], "resourceAttributes").(map[string]any) + if !ok { + t.Fatal("the first resource carries no resourceAttributes") + } + if err := schema.Validate(attributes); err != nil { + t.Fatalf("the unmodified answer should validate: %v", err) + } + + for _, required := range []string{"facilityType", "informationMode", "source"} { + t.Run("without "+required, func(t *testing.T) { + broken := map[string]any{} + for key, value := range attributes { + if key != required { + broken[key] = value + } + } + if err := schema.Validate(broken); err == nil { + t.Errorf("the validator accepted a Direct resource with no %s, "+ + "so it is not enforcing the pack", required) + } + }) + } + + // And the whole point of the GSW fallback: the value has to be one the pack + // governs, not merely present. + t.Run("with an ungoverned facilityType", func(t *testing.T) { + broken := map[string]any{} + for key, value := range attributes { + broken[key] = value + } + broken["facilityType"] = "TractorShed" + if err := schema.Validate(broken); err == nil { + t.Error("the validator accepted a facilityType outside the governed enum") + } + }) +} + +func mustIndent(t *testing.T, value any) string { + t.Helper() + pretty, err := json.MarshalIndent(value, "", " ") + if err != nil { + return fmt.Sprintf("%v", value) + } + return string(pretty) +} + +// packExamples are the schema pack's own published examples, by file name. +// +// Listed rather than discovered, so a lost example is a failure rather than a +// shorter loop. Both information modes are published and they have disjoint +// required sets -- OnDemand forbids most of what Direct requires -- so the set +// is what gives the check below its reach. +var packExamples = []string{ + "custom-hiring-centre.json", + "krishi-vigyan-kendra.json", + "on-demand-facility-discovery.json", + "soil-testing-facility.json", + "warehouse.json", +} + +// The pack's OWN examples must satisfy the schema as this test compiles it. +// +// This checks the compilation rather than the mapping. Everything else here +// validates what this plugin produces, which cannot distinguish "the answer is +// right" from "the schema was compiled too loosely to reject anything". If the +// pack's published examples fail, the compilation is wrong and every other +// conformance result in this file is worth nothing. +// +// It is also a drift alarm: the examples are fetched from the same ref as the +// schemas, so one that stops validating means the pack moved under this plugin. +func TestThePacksOwnExamplesValidate(t *testing.T) { + schema := facilitySchema(t) + loader := newPackLoader() + + for _, name := range packExamples { + t.Run(name, func(t *testing.T) { + url := packExampleBase + "/" + name + instance, err := loader.Load(url) + if err != nil { + loader.skipIfOffline(t, err) + t.Fatalf("could not read %s: %v", url, err) + } + if err := schema.Validate(instance); err != nil { + t.Errorf("the pack's own example %s does not satisfy the schema as compiled here, "+ + "so the compilation is wrong:\n%v", name, err) + } + }) + } +} + +// governedFields are the property names AgricultureFacility v0.1 declares, +// across its own branch and the AgricultureResource it composes with. +// +// @context is in the list and is NOT a schema property -- it appears only under +// x-jsonld, which a validator ignores. Every one of the pack's own examples +// carries it, so it is conventional rather than governed, and it is listed here +// deliberately rather than by oversight. +var governedFields = map[string]bool{ + "@context": true, // conventional, see above + "@type": true, + "informationMode": true, + "subjectCategories": true, + "agricultureSubjects": true, + "languages": true, + "coverageAreas": true, + "supportedFacilityTypes": true, + "facilityType": true, + "location": true, + "address": true, + "services": true, + "capacity": true, + "publicContact": true, + "website": true, + "source": true, + "lastUpdatedAt": true, +} + +// Nothing this mapping emits may be a field the pack does not declare. +// +// The validator cannot check this. Neither AgricultureFacility nor +// AgricultureResource sets additionalProperties: false at the top level, so an +// invented field passes validation in silence -- and publishing an ungoverned +// field under a governed schema is how a private convention leaks onto the +// network and becomes load-bearing for somebody else. +// +// The nested types DO forbid extras -- SourceReference, Address, +// FacilityCapacity, Contact and Descriptor all set additionalProperties: false +// -- so the validator already covers everything inside source, address, +// capacity and publicContact. This test covers the one level it cannot. +func TestTheAnswerInventsNoUngovernedField(t *testing.T) { + for _, tc := range []struct { + name string + facilityType string + provider string + }{ + {"a KVK search", "KrishiVigyanKendra", providerResponse}, + {"a warehouse search", "Warehouse", warehouseResponse}, + {"a filtered mixed answer", "CustomHiringCentre", mixedResponse}, + } { + t.Run(tc.name, func(t *testing.T) { + answer := runAgainst(t, requestFor(t, tc.facilityType), tc.provider) + for _, entry := range answerResources(t, answer) { + id, _ := dig(entry, "id").(string) + attributes, ok := dig(entry, "resourceAttributes").(map[string]any) + if !ok { + t.Fatalf("resource %s carries no resourceAttributes", id) + } + for field := range attributes { + if !governedFields[field] { + t.Errorf("%s carries %q, which AgricultureFacility v0.1 does not declare. "+ + "The top level sets no additionalProperties, so the validator will not "+ + "catch it -- either the pack governs it and this list is stale, or the "+ + "mapping is inventing a field", id, field) + } + } + } + }) + } +} + +// The list above has to match the pack, or the test above proves nothing. +// +// Read straight out of the vendored schema rather than trusted: a field added +// upstream should widen what the mapping may emit, and a field removed should +// narrow it, without either going unnoticed. +func TestTheGovernedFieldListMatchesThePack(t *testing.T) { + loader := newPackLoader() + + declared := map[string]bool{"@context": true} // conventional, not a property + for _, section := range []struct{ url, schema, until string }{ + {facilityPackURL, " AgricultureFacility:", " FacilityType:"}, + {resourcePackURL, " AgricultureResource:", " AdministrativeAreaReference:"}, + } { + // Read as text, not as a compiled schema: the point is which property + // names the document declares, and a compiled schema has already + // flattened its allOf branches together. + raw, err := loader.text(section.url) + if err != nil { + loader.skipIfOffline(t, err) + t.Fatalf("could not read %s: %v", section.url, err) + } + + text := string(raw) + start := strings.Index(text, section.schema) + end := strings.Index(text, section.until) + if start < 0 || end < 0 || end < start { + t.Fatalf("could not locate %s in %s", section.schema, section.url) + } + for _, name := range propertyNames(text[start:end]) { + declared[name] = true + } + } + + for field := range governedFields { + if !declared[field] { + t.Errorf("governedFields lists %q, which the pack does not declare", field) + } + } + for field := range declared { + if !governedFields[field] { + t.Errorf("the pack declares %q, which governedFields omits -- the mapping is "+ + "permitted less than the pack allows", field) + } + } +} + +// propertyNames pulls the property names out of a schema section's properties +// blocks, keyed off indentation so nested object properties are not mistaken +// for top-level ones. +func propertyNames(section string) []string { + var names []string + lines := strings.Split(section, "\n") + for index, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed != "properties:" { + continue + } + indent := len(line) - len(strings.TrimLeft(line, " ")) + for _, next := range lines[index+1:] { + if strings.TrimSpace(next) == "" { + continue + } + nextIndent := len(next) - len(strings.TrimLeft(next, " ")) + if nextIndent <= indent { + break + } + if nextIndent != indent+2 { + continue + } + name := strings.TrimSuffix(strings.TrimSpace(next), ":") + name = strings.Trim(name, `"`) + if name != "" && !strings.Contains(name, " ") { + names = append(names, name) + } + } + } + return names +} + +// The INBOUND query resource must satisfy the pack too, in OnDemand mode. +// +// Every other conformance test here validates the answer. Nothing validated +// the request, and the request is where this plugin made its most debatable +// choice: the search origin is read from the Beckn fulfillment stop rather +// than from resourceAttributes, because an OnDemand resource used to be +// schema-forbidden from carrying location, address or facilityType there. +// +// That forbid clause is gone as of network-specs commit b76c9ad8a5 on +// schema-packs-v0.1 (see dev_docs/schema-onDemand-forbid-removed.md) -- +// OnDemand now permits, but does not require, all of those fields. This test +// used to assert the forbid; it no longer can, because the schema no longer +// enforces it. The convention -- origin in the fulfillment stop, not in +// resourceAttributes -- is kept anyway, as a choice rather than a schema +// requirement, so what is left to check is only that the fixture is still a +// valid OnDemand AgricultureFacility at all. +// +// If the fixture the whole suite is built on is not a valid OnDemand +// AgricultureFacility, then the convention is wrong and every test that uses it +// is testing the wrong shape. +func TestTheRequestResourceSatisfiesOnDemandMode(t *testing.T) { + schema := facilitySchema(t) + + var payload map[string]any + if err := json.Unmarshal([]byte(selectRequest), &payload); err != nil { + t.Fatalf("the fixture is not JSON: %v", err) + } + attributes := attributes(t, payload) + + if attributes["informationMode"] != "OnDemand" { + t.Fatalf("informationMode = %v, want OnDemand for a query resource", + attributes["informationMode"]) + } + if err := schema.Validate(attributes); err != nil { + t.Fatalf("the request resource does not satisfy AgricultureFacility v0.1 in OnDemand mode:\n%v\n%s", + err, mustIndent(t, attributes)) + } +} + +// The pack's README carries two mapping rules a validator cannot enforce, and +// both bind this plugin directly. They are asserted here because the schema is +// silent on them: an answer that breaks either still validates. +// +// "An adapter must not copy a request coordinate into location unless the +// Provider confirms that the returned coordinate belongs to the facility." +// +// "Query-relative distance, ranking, and price are not intrinsic facility +// attributes. Distance belongs in result metadata. Price and booking terms +// belong in the applicable Beckn offer or transaction contract." +// +// POCRA supplies material for every one of these -- a stub gps, a distance +// string, and a price and rating on warehouse items -- so each is a live +// temptation rather than a hypothetical. +func TestTheAnswerFollowsThePacksMappingRules(t *testing.T) { + for _, tc := range []struct { + name string + facilityType string + provider string + }{ + {"a KVK search", "KrishiVigyanKendra", providerResponse}, + // The warehouse fixture is the one carrying price and rating. + {"a warehouse search", "Warehouse", warehouseResponse}, + } { + t.Run(tc.name, func(t *testing.T) { + answer := runAgainst(t, requestFor(t, tc.facilityType), tc.provider) + + for _, entry := range answerResources(t, answer) { + id, _ := dig(entry, "id").(string) + attributes, ok := dig(entry, "resourceAttributes").(map[string]any) + if !ok { + t.Fatalf("resource %s carries no resourceAttributes", id) + } + + // No coordinate. POCRA publishes one gps per answer and it is a + // fixed stub unrelated to the point asked for, so there is no + // verified facility geometry to carry -- and the search origin + // must not be substituted for one. + if _, present := attributes["location"]; present { + t.Errorf("%s carries location; the pack forbids substituting the search origin "+ + "and POCRA confirms no facility coordinate", id) + } + + // No price, no rating, no distance, under any spelling. + encoded, err := json.Marshal(attributes) + if err != nil { + t.Fatalf("could not re-encode %s: %v", id, err) + } + for _, forbidden := range []string{"price", "rating", "distance", + "estimated_value", "minimum_value", "Km", " km"} { + if strings.Contains(string(encoded), forbidden) { + t.Errorf("%s carries %q; query-relative distance, ranking and price are not "+ + "facility attributes and belong in the Beckn offer or in result metadata", + id, forbidden) + } + } + } + }) + } +} diff --git a/pkg/plugin/implementation/AgricultureFacility/live_test.go b/pkg/plugin/implementation/AgricultureFacility/live_test.go new file mode 100644 index 00000000..90534d7e --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/live_test.go @@ -0,0 +1,399 @@ +package AgricultureFacility_test + +// live_test.go runs the shipped mapping against the REAL POCRA API. +// +// Skipped unless POCRA_LIVE=1, because it needs the public internet and a +// government API that takes about eight seconds to answer. Everything it covers +// that can be covered offline is covered offline in mappings_test.go; what only +// this can prove is that the request this mapping builds is one POCRA actually +// accepts, and that the answer it actually returns still maps. +// +// POCRA_LIVE=1 go test ./pkg/plugin/implementation/AgricultureFacility/ -run TestLive -v +// +// It was worth writing. The captured sample in the design notes covered kvk +// alone, and running it against all four types is what found that a warehouse +// comes from a different BPP with no category tag on its items. + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/AgricultureFacility" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/util" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" +) + +const pocraBaseURL = "https://middleware-bap-client.mahapocra.gov.in" + +// governedTypes is every facility type the pack defines. All four are exercised, +// because they do not all come from the same upstream BPP. +var governedTypes = []string{ + "KrishiVigyanKendra", + "CustomHiringCentre", + "SoilTestingFacility", + "Warehouse", +} + +// liveRequest builds a select for one facility type, with ids that have never +// been used before. +// +// The fresh transaction id matters: POCRA accumulates answers per transaction, +// so reusing one returns everything previously asked for under it as well. +func liveRequest(t *testing.T, facilityType string) string { + t.Helper() + var payload map[string]any + if err := json.Unmarshal([]byte(selectRequest), &payload); err != nil { + t.Fatalf("the fixture is not JSON: %v", err) + } + attributes(t, payload)["supportedFacilityTypes"] = []any{facilityType} + + // UUIDs, not just unique strings. POCRA's schema constrains transaction_id + // and message_id to the uuid format, and the mapping passes the Beckn ids + // through verbatim -- so a caller sending anything else earns a 400 from + // POCRA that names the field. + beckncontext := payload["context"].(map[string]any) + beckncontext["transactionId"] = uuid.NewString() + beckncontext["messageId"] = uuid.NewString() + + body, err := json.Marshal(payload) + if err != nil { + t.Fatalf("could not build the request: %v", err) + } + return string(body) +} + +// recordingProxy forwards to POCRA and keeps a copy of what came back. +// +// It exists so one upstream call answers two questions at once: what POCRA +// actually returned, and what the mapping made of it. Without it a zero-facility +// answer is ambiguous -- POCRA having nothing to say and the mapping dropping +// everything look identical from the outside, and only one of those is a bug. +// +// POCRA takes about eight seconds per call, so asking it twice to disambiguate +// would double an already slow test. +func recordingProxy(t *testing.T, captured *[]byte) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + sent, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("could not read the request bound for POCRA: %v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + request, err := http.NewRequestWithContext(r.Context(), r.Method, + pocraBaseURL+r.URL.Path, bytes.NewReader(sent)) + if err != nil { + t.Errorf("could not build the request to POCRA: %v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + request.Header.Set("Content-Type", "application/json") + + response, err := (&http.Client{Timeout: 60 * time.Second}).Do(request) + if err != nil { + // POCRA times out often enough that this is a normal outcome + // rather than a test failure. Passing the status through lets the + // step report it the way it would in production. + t.Logf("POCRA did not answer: %v", err) + w.WriteHeader(http.StatusBadGateway) + return + } + defer response.Body.Close() + + body, err := io.ReadAll(response.Body) + if err != nil { + t.Errorf("could not read POCRA's answer: %v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + *captured = body + w.WriteHeader(response.StatusCode) + _, _ = w.Write(body) + })) +} + +// pocraItemCount counts the facilities POCRA actually returned, across every +// responses[] entry, before the mapping touches them. +func pocraItemCount(t *testing.T, body []byte) int { + t.Helper() + if len(body) == 0 { + return 0 + } + var answer struct { + Responses []struct { + Message struct { + Catalog struct { + Providers []struct { + Items []json.RawMessage `json:"items"` + } `json:"providers"` + } `json:"catalog"` + } `json:"message"` + } `json:"responses"` + } + if err := json.Unmarshal(body, &answer); err != nil { + t.Fatalf("POCRA answered with something that is not JSON: %v", err) + } + count := 0 + for _, response := range answer.Responses { + for _, provider := range response.Message.Catalog.Providers { + count += len(provider.Items) + } + } + return count +} + +func TestLiveAgainstPocra(t *testing.T) { + if os.Getenv("POCRA_LIVE") != "1" { + t.Skip("set POCRA_LIVE=1 to run this against the real POCRA API") + } + + mappings := serveMappings(t) + defer mappings.Close() + + mapper, closeMapper, err := jsonmapper.New(context.Background(), &jsonmapper.Config{}) + if err != nil { + t.Fatalf("failed to build the mapper: %v", err) + } + defer closeMapper() + + var captured []byte + proxy := recordingProxy(t, &captured) + defer proxy.Close() + + registry := &stubRegistry{plan: &model.ProviderRecord{ + BindingKey: shippedBindingKey, + ParticipantID: "pocra", + CapabilityCode: shippedCapability, + BaseURL: proxy.URL, + Actions: map[string]model.ActionPlan{ + // The same budget the registry row publishes. POCRA answers in + // about eight seconds, so 30s is the right order of magnitude and + // 15s would be uncomfortably close. + "select": {Method: http.MethodPost, Path: "/search", + Mappings: mappings.URL + "/" + shippedMapping, TimeoutMs: 30000, RetryMax: 2}, + }, + }} + + step, closeStep, err := AgricultureFacility.New(context.Background(), registry, mapper, + &AgricultureFacility.Config{ + BindingKeys: []string{shippedBindingKey}, + // Auth is per provider now; POCRA needs no credential and says so. + AuthByProvider: map[string]*common.AuthProfile{ + strings.Split(shippedBindingKey, "|")[0]: {Scheme: util.AuthSchemeNone}, + }, + }) + if err != nil { + t.Fatalf("failed to build the step: %v", err) + } + defer closeStep() + + // The published pack, compiled by a real validator. The hand-written checks + // below say what this plugin believes the pack requires; this says what it + // actually does, against data nobody curated. + schema := facilitySchema(t) + + for _, facilityType := range governedTypes { + t.Run(facilityType, func(t *testing.T) { + captured = nil + stepCtx := &model.StepContext{Context: t.Context(), Body: []byte(liveRequest(t, facilityType))} + started := time.Now() + if err := step.Run(stepCtx); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + elapsed := time.Since(started) + + var answer map[string]any + if err := json.Unmarshal(stepCtx.ResponseBody, &answer); err != nil { + t.Fatalf("the answer is not JSON: %v", err) + } + resources := answerResources(t, answer) + offered := pocraItemCount(t, captured) + t.Logf("%s: POCRA returned %d items -> %d facilities in %s, %d bytes", + facilityType, offered, len(resources), elapsed.Round(time.Millisecond), + len(stepCtx.ResponseBody)) + + // An empty answer is only a bug when POCRA had something to say. + // Its BPPs answer with responses[] empty often enough that failing + // on it would make this test red for reasons nothing here controls + // -- chc did exactly that, minutes after returning five facilities. + if offered == 0 { + if len(resources) != 0 { + t.Errorf("POCRA returned nothing but the mapping produced %d facilities", + len(resources)) + } + t.Skipf("POCRA returned no %s facilities for this point", facilityType) + } + if len(resources) == 0 { + t.Fatalf("POCRA returned %d items and the mapping produced no facilities", offered) + } + // Deduplication only ever removes. + if len(resources) > offered { + t.Errorf("got %d facilities from %d items; the mapping invented some", + len(resources), offered) + } + + for _, entry := range resources { + id, _ := dig(entry, "id").(string) + attrs, ok := dig(entry, "resourceAttributes").(map[string]any) + if !ok { + t.Fatalf("resource %s carries no resourceAttributes", id) + } + // Direct mode's required set, which is what a consumer + // validating against the pack will refuse an answer for. + if attrs["facilityType"] != facilityType { + t.Errorf("%s: facilityType = %v, want %s", id, attrs["facilityType"], facilityType) + } + if attrs["informationMode"] != "Direct" { + t.Errorf("%s: informationMode = %v, want Direct", id, attrs["informationMode"]) + } + if attrs["source"] == nil { + t.Errorf("%s: Direct mode requires source", id) + } + if attrs["address"] == nil && attrs["location"] == nil { + t.Errorf("%s: Direct mode requires one of address or location", id) + } + if _, present := attrs["location"]; present { + t.Errorf("%s: carries location; POCRA publishes no verified facility geometry", id) + } + } + + // The real answer, against the real pack. Every offline conformance + // case is a captured body; this is the only check that runs the + // validator over data POCRA chose today. + validateFacilities(t, schema, answer) + + // Nothing POCRA uses as a placeholder may reach the network. + raw, err := json.Marshal(answer) + if err != nil { + t.Fatalf("could not re-encode the answer: %v", err) + } + for _, placeholder := range []string{"Unknown", "N/A", "000000", `"-"`} { + if strings.Contains(string(raw), placeholder) { + t.Errorf("the answer carries the placeholder %s", placeholder) + } + } + }) + } +} + +// TestLiveCategoryLeakIsFiltered provokes the real hazard and checks the mapping +// holds against it. +// +// POCRA caches per message_id for PT10M and accumulates across searches sharing +// one, so two searches with the same messageId and different categories make the +// second answer carry both. Beckn requires a fresh messageId per message, but an +// adapter cannot assume every caller obeys that -- a retry that reuses one is the +// obvious way in -- so the mapping filters to the requested type rather than +// trusting the upstream to have done it. +// +// This is the only test that proves the filter against POCRA's actual behaviour +// rather than a fixture built from it. +func TestLiveCategoryLeakIsFiltered(t *testing.T) { + if os.Getenv("POCRA_LIVE") != "1" { + t.Skip("set POCRA_LIVE=1 to run this against the real POCRA API") + } + + mappings := serveMappings(t) + defer mappings.Close() + + mapper, closeMapper, err := jsonmapper.New(context.Background(), &jsonmapper.Config{}) + if err != nil { + t.Fatalf("failed to build the mapper: %v", err) + } + defer closeMapper() + + var captured []byte + proxy := recordingProxy(t, &captured) + defer proxy.Close() + + registry := &stubRegistry{plan: &model.ProviderRecord{ + BindingKey: shippedBindingKey, BaseURL: proxy.URL, + Actions: map[string]model.ActionPlan{ + "select": {Method: http.MethodPost, Path: "/search", + Mappings: mappings.URL + "/" + shippedMapping, TimeoutMs: 30000}, + }, + }} + + step, closeStep, err := AgricultureFacility.New(context.Background(), registry, mapper, + &AgricultureFacility.Config{ + BindingKeys: []string{shippedBindingKey}, + // Auth is per provider now; POCRA needs no credential and says so. + AuthByProvider: map[string]*common.AuthProfile{ + strings.Split(shippedBindingKey, "|")[0]: {Scheme: util.AuthSchemeNone}, + }, + }) + if err != nil { + t.Fatalf("failed to build the step: %v", err) + } + defer closeStep() + + // One messageId, deliberately shared. Different transaction ids, to show it + // is the messageId the upstream keys its cache on. + shared := uuid.NewString() + + search := func(facilityType string) (raw []byte, answer map[string]any) { + t.Helper() + var payload map[string]any + if err := json.Unmarshal([]byte(selectRequest), &payload); err != nil { + t.Fatalf("the fixture is not JSON: %v", err) + } + attributes(t, payload)["supportedFacilityTypes"] = []any{facilityType} + beckncontext := payload["context"].(map[string]any) + beckncontext["messageId"] = shared + beckncontext["transactionId"] = uuid.NewString() + body, err := json.Marshal(payload) + if err != nil { + t.Fatalf("could not build the request: %v", err) + } + + captured = nil + stepCtx := &model.StepContext{Context: t.Context(), Body: body} + if err := step.Run(stepCtx); err != nil { + t.Fatalf("%s: Run() returned an unexpected error: %v", facilityType, err) + } + if err := json.Unmarshal(stepCtx.ResponseBody, &answer); err != nil { + t.Fatalf("%s: the answer is not JSON: %v", facilityType, err) + } + return captured, answer + } + + search("KrishiVigyanKendra") + raw, answer := search("CustomHiringCentre") + + // Did the hazard actually happen? If POCRA has stopped accumulating there is + // nothing here to filter, and passing would say more than it knows. + if !bytes.Contains(raw, []byte(`"kvk"`)) { + t.Skip("POCRA did not leak kvk into the chc search; nothing to filter this run") + } + t.Log("POCRA leaked kvk facilities into a chc search, as expected") + + resources := answerResources(t, answer) + if len(resources) == 0 { + t.Fatal("the mapping filtered everything away") + } + for _, entry := range resources { + id, _ := dig(entry, "id").(string) + attrs, ok := dig(entry, "resourceAttributes").(map[string]any) + if !ok { + t.Fatalf("resource %s carries no resourceAttributes", id) + } + if attrs["facilityType"] != "CustomHiringCentre" { + t.Errorf("%s: facilityType = %v, want CustomHiringCentre -- a leaked facility reached the answer", + id, attrs["facilityType"]) + } + } + t.Logf("kept %d facilities, all CustomHiringCentre", len(resources)) +} diff --git a/pkg/plugin/implementation/AgricultureFacility/mappings_test.go b/pkg/plugin/implementation/AgricultureFacility/mappings_test.go new file mode 100644 index 00000000..d0966ec5 --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/mappings_test.go @@ -0,0 +1,1762 @@ +package AgricultureFacility_test + +// mappings_test.go runs the shipped agriculture facility mapping through the +// real mapper and the real provider step. It is the only test that proves the +// three pieces fit: a mapping is JSONata inside YAML fetched over HTTP, and +// nothing but running it establishes that what is published actually produces +// valid Beckn. +// +// An external test package on purpose -- it uses the plugins exactly as the +// adapter does, through their exported surface and nothing else. + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "sync" + "testing" + + "github.com/google/uuid" + + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/AgricultureFacility" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/util" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" +) + +// mappingsDir is where the shipped mappings live, relative to this package. +// Reading the published file rather than a fixture is the whole point: this test +// breaks when what is deployed breaks. +const mappingsDir = "../../../../config/mappings/pocra" + +// shippedMapping is the file this binding-action publishes: one file, both +// directions. The action segment of the name must match the action the registry +// entry declares -- a mismatch would apply a correct mapping to the wrong call, +// silently. +const shippedMapping = "agriculture-facility.select.yaml" + +// shippedCapability is what the pack calls this capability, and the second half +// of the binding key the registry indexes the provider's record by. +const shippedCapability = "openagrinet:AgricultureFacility" + +const shippedBindingKey = "pocra|" + shippedCapability + +// selectRequest is a facility search in OnDemand mode. +// +// The query lives in the Beckn layer, not in resourceAttributes: the pack +// forbids an OnDemand resource from carrying location or facilityType, so the +// search origin is a fulfillment stop and the requested type is +// supportedFacilityTypes. +// +// PROVISIONAL: this convention was chosen on the design and has not yet been +// confirmed against a payload captured from the network. +// declaredContext is the @context the fixture declares, and the one the answer +// must echo back. +// +// It is the pack's published URL because that URL resolves. The identifier the +// pack names in its own x-jsonld, https://schemas.openagrinet.global/..., does +// not: the host has no DNS record, so a JSON-LD processor handed it can fetch +// nothing. Both are exercised in TestShippedMappingEchoesTheCallersContext, +// since the mapping must not care which it is given. +const declaredContext = "https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/AgricultureFacility/v0.1/context.jsonld" + +// callerMessageID is the messageId selectRequest declares. Named so the +// fan-out assertion that no upstream call reuses it reads from one place rather +// than repeating the literal. +const callerMessageID = "a1b2c3d4-e5f6-4789-abcd-ef1234567890" + +const selectRequest = `{ + "context": { "version": "2.0.0", "action": "select", + "networkId": "oan-dev", + "transactionId": "f9e8d7c6-b5a4-4321-9876-543210fedcba", + "messageId": "a1b2c3d4-e5f6-4789-abcd-ef1234567890", + "timestamp": "2026-09-02T06:12:01.330Z" }, + "message": { "contract": { "commitments": [{ + "status": { "descriptor": { "code": "DRAFT", "name": "Draft" } }, + "resources": [{ + "id": "res:pocra:facility-search", + "quantity": 1, + "resourceAttributes": { + "@context": "https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/AgricultureFacility/v0.1/context.jsonld", + "@type": "openagrinet:AgricultureFacility", + "informationMode": "OnDemand", + "subjectCategories": ["Facility"], + "supportedFacilityTypes": ["KrishiVigyanKendra"] + } + }], + "offer": { + "id": "offer:pocra:common-services", + "resourceIds": ["res:pocra:facility-search"], + "provider": { "id": "pocra", + "descriptor": { "code": "POCRA-01", "name": "PoCRA Provider Aggregator" } } + }, + "fulfillment": { + "stops": [{ + "location": { "geo": { "type": "Point", "coordinates": [74.5321, 19.5132] } }, + "time": { "range": { "start": "2026-09-19T08:00:00.108Z" } } + }] + } + }] } } +}` + +// providerResponse is POCRA's own shape, trimmed from a body captured against +// the live API (not checked into this repo). +// +// TWO responses[] entries, because POCRA returns one per answering BPP and the +// captured sample carried four. The second repeats COMMON-55043, which is what +// the dedupe has to collapse. +// +// Distances are deliberately out of order, and both items carry the placeholder +// values POCRA really sends: "Unknown", "N/A" and "000000". +const providerResponse = `{ + "context": { "action": "search", "version": "1.1.0" }, + "responses": [ + { + "context": { "action": "on_search", "version": "1.1.0" }, + "message": { "catalog": { + "descriptor": { "name": "Pocra Provider Aggregator Services" }, + "providers": [{ + "id": "COMMON_PROVIDER_KVK", + "fulfillments": [{ "id": "f1_kvk", "type": "Service", + "locations": { "id": "l1", "gps": "17.385,78.4867" }, + "categories": [{ "id": "c_kvk", "name": "KVK", + "descriptor": { "code": "kvk", "name": "KVK" } }] }], + "items": [ + { + "id": "COMMON-55043", + "descriptor": { "name": "Krishi Vigyan Kendra, Biloli" }, + "address": { "address": "Krishi Vigyan Kendra, Village- Sagroli, Taluka- Biloli, Distt.-Nanded", + "district": "Nanded", "region": "Unknown", "taluka": "Unknown", + "vilage": "Unknown", "pinCode": "000000" }, + "contact": { "person": "Chairman, Sanskriti Samvardhan Mandal, Sagroli", + "email": "N/A", "phone": "N/A", + "webUrl": "https://provider.mahapocra.gov.in" }, + "fulfillment_ids": ["f1_kvk"], + "category_ids": ["c_kvk"], + "tags": [{ "list": [ + { "descriptor": { "code": "distance" }, "value": "165 Km" }, + { "descriptor": { "code": "organization" }, "value": "Krishi Vigyan Kendra" }, + { "descriptor": { "code": "category" }, "value": "kvk" } ] }] + }, + { + "id": "COMMON-55007", + "descriptor": { "name": "Krishi Vigyan Kendra, Latur" }, + "address": { "address": "Krishi Vigyan Kendra, Chincholirao Wadi, MIDC, Latur-413512", + "district": "Latur", "region": "Unknown", "taluka": "Unknown", + "vilage": "Unknown", "pinCode": "000000" }, + "contact": { "person": "N/A", "email": "N/A", "phone": "N/A", + "webUrl": "https://provider.mahapocra.gov.in" }, + "fulfillment_ids": ["f1_kvk"], + "category_ids": ["c_kvk"], + "tags": [{ "list": [ + { "descriptor": { "code": "distance" }, "value": "98 Km" }, + { "descriptor": { "code": "category" }, "value": "kvk" } ] }] + } + ] + }] + } } + }, + { + "context": { "action": "on_search", "version": "1.1.0" }, + "message": { "catalog": { + "descriptor": { "name": "Pocra Provider Aggregator Services" }, + "providers": [{ + "id": "COMMON_PROVIDER_KVK", + "fulfillments": [{ "id": "f1_kvk", "type": "Service", + "locations": { "id": "l1", "gps": "17.385,78.4867" }, + "categories": [{ "id": "c_kvk", "name": "KVK", + "descriptor": { "code": "kvk", "name": "KVK" } }] }], + "items": [ + { + "id": "COMMON-55043", + "descriptor": { "name": "Krishi Vigyan Kendra, Biloli" }, + "address": { "address": "Krishi Vigyan Kendra, Village- Sagroli, Taluka- Biloli, Distt.-Nanded", + "district": "Nanded", "region": "Unknown", "taluka": "Unknown", + "vilage": "Unknown", "pinCode": "000000" }, + "contact": { "person": "Chairman, Sanskriti Samvardhan Mandal, Sagroli", + "email": "N/A", "phone": "N/A", + "webUrl": "https://provider.mahapocra.gov.in" }, + "fulfillment_ids": ["f1_kvk"], + "category_ids": ["c_kvk"], + "tags": [{ "list": [ + { "descriptor": { "code": "distance" }, "value": "165 Km" }, + { "descriptor": { "code": "category" }, "value": "kvk" } ] }] + } + ] + }] + } } + } + ] +}` + +// serveMappings publishes the shipped mapping files over HTTP, which is how the +// mapper fetches them in production. +func serveMappings(t *testing.T) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := os.ReadFile(filepath.Join(mappingsDir, filepath.Base(r.URL.Path))) + if err != nil { + t.Errorf("could not read the mapping %q: %v", r.URL.Path, err) + w.WriteHeader(http.StatusNotFound) + return + } + fmt.Fprint(w, string(body)) + })) +} + +// stubRegistry answers with the call plan the live registry holds for this +// capability. +type stubRegistry struct{ plan *model.ProviderRecord } + +func (s *stubRegistry) ProviderRecord(context.Context, string) (*model.ProviderRecord, error) { + return s.plan, nil +} + +// runSelect drives the real step over the real mapping against a fake POCRA, and +// returns the body POCRA was sent alongside the answer. +// +// It returns the error rather than failing on it, because refusing a payload is +// as much of this mapping's job as serving one. +func runSelect(t *testing.T, request string) (sent map[string]any, answer map[string]any, runErr error) { + t.Helper() + + mappings := serveMappings(t) + defer mappings.Close() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(&sent); err != nil { + t.Errorf("POCRA was sent something that is not JSON: %v", err) + } + fmt.Fprint(w, providerResponse) + })) + defer upstream.Close() + + mapper, closeMapper, err := jsonmapper.New(context.Background(), &jsonmapper.Config{}) + if err != nil { + t.Fatalf("failed to build the mapper: %v", err) + } + defer closeMapper() + + registry := &stubRegistry{plan: &model.ProviderRecord{ + BindingKey: shippedBindingKey, + ParticipantID: "pocra", + CapabilityCode: shippedCapability, + BaseURL: upstream.URL, + Actions: map[string]model.ActionPlan{ + "select": {Method: http.MethodPost, Path: "/search", + Mappings: mappings.URL + "/" + shippedMapping, TimeoutMs: 30000, RetryMax: 2}, + }, + }} + + step, closeStep, err := AgricultureFacility.New(context.Background(), registry, mapper, + &AgricultureFacility.Config{ + BindingKeys: []string{shippedBindingKey}, + // Auth is per provider now; POCRA needs no credential and says so. + AuthByProvider: map[string]*common.AuthProfile{ + strings.Split(shippedBindingKey, "|")[0]: {Scheme: util.AuthSchemeNone}, + }, + }) + if err != nil { + t.Fatalf("failed to build the step: %v", err) + } + defer closeStep() + + stepCtx := &model.StepContext{Context: t.Context(), Body: []byte(request)} + runErr = step.Run(stepCtx) + if runErr != nil { + return sent, nil, runErr + } + if len(stepCtx.ResponseBody) == 0 { + t.Fatal("the step produced no answer") + } + if err := json.Unmarshal(stepCtx.ResponseBody, &answer); err != nil { + t.Fatalf("the answer is not JSON: %v\n%s", err, stepCtx.ResponseBody) + } + return sent, answer, nil +} + +// dig walks a decoded JSON object by key, returning nil at the first miss so a +// failing assertion reports the path rather than panicking. +func dig(node any, path ...string) any { + for _, key := range path { + object, ok := node.(map[string]any) + if !ok { + return nil + } + node = object[key] + } + return node +} + +// answerResources returns the resources of the single commitment in an answer. +func answerResources(t *testing.T, answer map[string]any) []any { + t.Helper() + commitments, ok := dig(answer, "message", "contract", "commitments").([]any) + if !ok || len(commitments) != 1 { + t.Fatalf("got %d commitments, want 1", len(commitments)) + } + resources, ok := dig(commitments[0], "resources").([]any) + if !ok { + t.Fatal("the quoted commitment carries no resources") + } + return resources +} + +func TestShippedMappingSendsWhatPocraExpects(t *testing.T) { + sent, _, err := runSelect(t, selectRequest) + if err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + + stops, _ := dig(sent, "message", "intent", "fulfillment", "stops").([]any) + if len(stops) != 1 { + t.Fatalf("got %d stops, want 1", len(stops)) + } + // GPS is "lat,lon" and OAN GeoJSON is [lon, lat]. Reading them the same way + // round gives a valid request for a point in the wrong place, so it fails as + // wrong data rather than as an error. + if gps := dig(stops[0], "location", "gps"); gps != "19.5132,74.5321" { + t.Errorf("gps = %v, want 19.5132,74.5321 in lat,lon order", gps) + } + if start := dig(stops[0], "time", "range", "start"); start != "2026-09-19T08:00:00.108Z" { + t.Errorf("time.range.start = %v, want the stop's own start", start) + } + + // The governed type is translated into POCRA's private vocabulary here and + // nowhere else. + if code := dig(sent, "message", "intent", "category", "descriptor", "code"); code != "kvk" { + t.Errorf("category code = %v, want kvk for KrishiVigyanKendra", code) + } + if name := dig(sent, "message", "intent", "item", "descriptor", "name"); name != "service-locations" { + t.Errorf("item name = %v, want service-locations", name) + } + + // POCRA's request contract, pinned. These are not this adapter's identity, + // which travels in the signed Authorization header. + for _, f := range []struct{ key, want string }{ + {"domain", "advisory:mh-vistaar"}, + {"action", "search"}, + {"version", "1.1.0"}, + {"bap_id", "bap.mahapocra.gov.in"}, + {"bap_uri", "https://middleware.mahapocra.gov.in/bap/"}, + } { + if got := dig(sent, "context", f.key); got != f.want { + t.Errorf("context.%s = %v, want %v", f.key, got, f.want) + } + } + + // Correlation survives, so POCRA's logs can be matched to ours. + if got := dig(sent, "context", "transaction_id"); got != "f9e8d7c6-b5a4-4321-9876-543210fedcba" { + t.Errorf("transaction_id = %v, want the one from the request", got) + } + + // Epoch seconds as a string, unlike every other timestamp in either + // protocol. A wrong shape here is one of POCRA's schema NACKs. + timestamp, ok := dig(sent, "context", "timestamp").(string) + if !ok { + t.Fatalf("context.timestamp = %v, want a string", dig(sent, "context", "timestamp")) + } + if strings.ContainsAny(timestamp, "-:") { + t.Errorf("context.timestamp = %q, want epoch seconds rather than ISO 8601", timestamp) + } + if len(timestamp) < 10 { + t.Errorf("context.timestamp = %q, want at least 10 digits of epoch seconds", timestamp) + } +} + +// commitment reaches into a decoded payload, which is how the refusal cases are +// built. +// +// Editing the fixture's TEXT instead would be a trap: removing the last member +// of an object leaves a trailing comma, and the resulting parse error would look +// like a mapping failure rather than the case under test. +func commitment(t *testing.T, payload map[string]any) map[string]any { + t.Helper() + message, ok := payload["message"].(map[string]any) + if !ok { + t.Fatal("the fixture carries no message") + } + contract, ok := message["contract"].(map[string]any) + if !ok { + t.Fatal("the fixture carries no contract") + } + commitments, ok := contract["commitments"].([]any) + if !ok || len(commitments) == 0 { + t.Fatal("the fixture carries no commitments") + } + first, ok := commitments[0].(map[string]any) + if !ok { + t.Fatal("the first commitment is not an object") + } + return first +} + +// attributes reaches the resourceAttributes of a decoded payload. +func attributes(t *testing.T, payload map[string]any) map[string]any { + t.Helper() + resources, ok := commitment(t, payload)["resources"].([]any) + if !ok || len(resources) == 0 { + t.Fatal("the fixture carries no resources") + } + resource, ok := resources[0].(map[string]any) + if !ok { + t.Fatal("the first resource is not an object") + } + found, ok := resource["resourceAttributes"].(map[string]any) + if !ok { + t.Fatal("the first resource carries no resourceAttributes") + } + return found +} + +func TestShippedMappingRefusesWhatItCannotServe(t *testing.T) { + testCases := []struct { + name string + mutate func(t *testing.T, payload map[string]any) + wants string + }{ + { + name: "no fulfillment stop at all", + mutate: func(t *testing.T, payload map[string]any) { + commitment(t, payload)["fulfillment"] = map[string]any{"stops": []any{}} + }, + wants: "Point", + }, + { + name: "a polygon rather than a point", + mutate: func(t *testing.T, payload map[string]any) { + fulfillment := commitment(t, payload)["fulfillment"].(map[string]any) + stop := fulfillment["stops"].([]any)[0].(map[string]any) + stop["location"].(map[string]any)["geo"].(map[string]any)["type"] = "Polygon" + }, + wants: "Point", + }, + { + name: "a facility type nobody governs", + mutate: func(t *testing.T, payload map[string]any) { + attributes(t, payload)["supportedFacilityTypes"] = []any{"TractorShed"} + }, + wants: "facility type", + }, + { + name: "no facility type at all", + mutate: func(t *testing.T, payload map[string]any) { + delete(attributes(t, payload), "supportedFacilityTypes") + }, + wants: "facility type", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var payload map[string]any + if err := json.Unmarshal([]byte(selectRequest), &payload); err != nil { + t.Fatalf("the fixture is not JSON: %v", err) + } + tc.mutate(t, payload) + body, err := json.Marshal(payload) + if err != nil { + t.Fatalf("could not rebuild the payload: %v", err) + } + + sent, _, runErr := runSelect(t, string(body)) + if runErr == nil { + t.Fatal("expected the mapping's precondition to refuse this payload") + } + if !strings.Contains(runErr.Error(), tc.wants) { + t.Errorf("error %q should explain the problem by mentioning %q", runErr, tc.wants) + } + // The refusal has to happen before the call. Reaching POCRA with a + // payload we already know it cannot serve wastes its budget and + // turns our own bad request into its error message. + if sent != nil { + t.Errorf("POCRA was called with %v; a precondition must refuse before the call", sent) + } + }) + } +} + +func TestShippedMappingAnswersWithBeckn(t *testing.T) { + _, answer, err := runSelect(t, selectRequest) + if err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + + if got := dig(answer, "context", "action"); got != "on_select" { + t.Errorf("action = %v, want on_select", got) + } + if got := dig(answer, "context", "transactionId"); got != "f9e8d7c6-b5a4-4321-9876-543210fedcba" { + t.Errorf("transactionId = %v, want the one from the request", got) + } + if resources := answerResources(t, answer); len(resources) == 0 { + t.Fatal("the answer carries no resources") + } +} + +// POCRA returns one responses[] entry per answering BPP, and the same facility +// appears in more than one of them. A repeated id in the answer would be a +// broken reference the moment anything resolved it, so they collapse to one. +// +// Ordering is POCRA's own ranking, which arrives only as a distance string on +// each item. It is used to sort and then dropped: the schema pack states that +// query-relative distance is not an intrinsic facility attribute. +func TestShippedMappingDedupesAndOrdersByDistance(t *testing.T) { + _, answer, err := runSelect(t, selectRequest) + if err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + resources := answerResources(t, answer) + + // Three items arrive across two responses; COMMON-55043 appears twice. + if len(resources) != 2 { + t.Fatalf("got %d resources, want 2 -- COMMON-55043 arrives twice and must collapse", len(resources)) + } + + ids := make([]string, 0, len(resources)) + for _, entry := range resources { + id, _ := dig(entry, "id").(string) + ids = append(ids, id) + } + + // 98 Km before 165 Km. Sorting the strings rather than the numbers would put + // "165 Km" first, which nothing downstream could detect. + want := []string{"res:pocra:facility:COMMON-55007", "res:pocra:facility:COMMON-55043"} + for i, expected := range want { + if ids[i] != expected { + t.Errorf("resource %d = %q, want %q -- nearest first", i, ids[i], expected) + } + } + + // Distance must not survive into the answer in any form. + raw, err := json.Marshal(answer) + if err != nil { + t.Fatalf("could not re-encode the answer: %v", err) + } + if strings.Contains(string(raw), "165 Km") || strings.Contains(string(raw), `"distance"`) { + t.Error("the answer carries a distance; the pack says query-relative distance is not a facility attribute") + } +} + +// AgricultureFacility v0.1 Direct mode requires @type, informationMode, +// facilityType and source, plus at least one of location or address. +// +// It is address rather than location, and that is not a shortcut: POCRA returns +// no verified per-facility geometry -- the one gps in its answer is a fixed stub +// unrelated to what was asked for -- and the pack states that an adapter must +// not copy a request coordinate into location unless the provider confirms the +// coordinate belongs to the facility. +func TestShippedMappingFollowsTheAgricultureFacilityPack(t *testing.T) { + _, answer, err := runSelect(t, selectRequest) + if err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + + for _, entry := range answerResources(t, answer) { + id, _ := dig(entry, "id").(string) + attrs, ok := dig(entry, "resourceAttributes").(map[string]any) + if !ok { + t.Fatalf("resource %s carries no resourceAttributes", id) + } + + for _, f := range []struct{ key, want string }{ + {"@context", declaredContext}, + {"@type", "openagrinet:AgricultureFacility"}, + {"informationMode", "Direct"}, + {"facilityType", "KrishiVigyanKendra"}, + } { + if attrs[f.key] != f.want { + t.Errorf("%s: %s = %v, want %v", id, f.key, attrs[f.key], f.want) + } + } + + // Direct mode requires source, and anyOf location or address. + if attrs["source"] == nil { + t.Errorf("%s: Direct mode requires source", id) + } + if attrs["address"] == nil { + t.Errorf("%s: no verified geometry is available, so address is required", id) + } + if _, present := attrs["location"]; present { + t.Errorf("%s: carries location; POCRA supplies no verified facility geometry", id) + } + + // Not supplied truthfully by POCRA, so not asserted. Deriving services + // from the facility type, or lastUpdatedAt from fetch time, would be + // inventing data under a governed schema. + for _, absent := range []string{"services", "lastUpdatedAt", "capacity", "website"} { + if _, present := attrs[absent]; present { + t.Errorf("%s: carries %q, which POCRA does not supply", id, absent) + } + } + } +} + +// POCRA sends "Unknown", "N/A" and "000000" where it has no value. Publishing +// them would put junk on the network under a governed schema. +func TestShippedMappingScrubsPocraPlaceholders(t *testing.T) { + _, answer, err := runSelect(t, selectRequest) + if err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + + raw, err := json.Marshal(answer) + if err != nil { + t.Fatalf("could not re-encode the answer: %v", err) + } + for _, placeholder := range []string{"Unknown", "N/A", "000000"} { + if strings.Contains(string(raw), placeholder) { + t.Errorf("the answer carries the placeholder %q", placeholder) + } + } + + resources := answerResources(t, answer) + + // COMMON-55007 sorts first and has no usable contact person at all, so it + // must carry no publicContact rather than an empty one. + nearest, ok := dig(resources[0], "resourceAttributes").(map[string]any) + if !ok { + t.Fatal("the nearest resource carries no resourceAttributes") + } + if id := dig(resources[0], "id"); id != "res:pocra:facility:COMMON-55007" { + t.Fatalf("resource 0 = %v, want COMMON-55007", id) + } + if _, present := nearest["publicContact"]; present { + t.Error("COMMON-55007 has no usable contact, so publicContact must be absent") + } + + // COMMON-55043 does have one, carried as an organisational company name -- + // the pack restricts publicContact to contact data approved for catalog + // publication, and POCRA's person field is an office rather than a person. + farther, ok := dig(resources[1], "resourceAttributes").(map[string]any) + if !ok { + t.Fatal("the farther resource carries no resourceAttributes") + } + contact, ok := farther["publicContact"].(map[string]any) + if !ok { + t.Fatal("COMMON-55043 has a contact person and must carry publicContact") + } + if company, _ := contact["company"].(string); !strings.Contains(company, "Sanskriti") { + t.Errorf("publicContact.company = %v, want POCRA's organisational contact", contact["company"]) + } + + // The district survives scrubbing even though region, taluka and village do + // not, so the address is still useful. + address, ok := farther["address"].(map[string]any) + if !ok { + t.Fatal("COMMON-55043 carries no address") + } + if extended, _ := address["extendedAddress"].(string); !strings.Contains(extended, "Nanded") { + t.Errorf("extendedAddress = %v, want the district that survived scrubbing", address["extendedAddress"]) + } + if _, present := address["postalCode"]; present { + t.Error("pinCode was 000000, so postalCode must be absent rather than zeroed") + } + if address["addressCountry"] != "IN" || address["addressRegion"] != "Maharashtra" { + t.Errorf("address = %v, want IN / Maharashtra", address) + } + + // contact.webUrl is POCRA's own site repeated on every item, so it + // identifies the source and not the facility. + source, ok := farther["source"].(map[string]any) + if !ok { + t.Fatal("COMMON-55043 carries no source") + } + if source["sourceId"] != "pocra" { + t.Errorf("source.sourceId = %v, want pocra", source["sourceId"]) + } + if source["sourceUri"] != "https://provider.mahapocra.gov.in" { + t.Errorf("source.sourceUri = %v, want POCRA's own site", source["sourceUri"]) + } +} + +// The request names an abstract search resource and the answer returns the +// concrete facilities that satisfy it, so the offer arrives referencing an id +// that appears nowhere in the answer. Echoing the offer unchanged leaves that +// dangling reference behind. +func TestShippedMappingRewritesTheOffersReferences(t *testing.T) { + _, answer, err := runSelect(t, selectRequest) + if err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + + commitments, _ := dig(answer, "message", "contract", "commitments").([]any) + resources := answerResources(t, answer) + offer, ok := dig(commitments[0], "offer").(map[string]any) + if !ok { + t.Fatal("the quoted commitment carries no offer") + } + + returned := make([]string, 0, len(resources)) + for _, entry := range resources { + id, _ := dig(entry, "id").(string) + returned = append(returned, id) + } + + referenced, _ := offer["resourceIds"].([]any) + if len(referenced) != len(returned) { + t.Fatalf("offer.resourceIds has %d entries, want %d -- one per resource returned", + len(referenced), len(returned)) + } + for _, reference := range referenced { + if !slices.Contains(returned, reference.(string)) { + t.Errorf("offer references %v, which is not among the resources returned", reference) + } + if reference == "res:pocra:facility-search" { + t.Error("the offer still references the abstract resource the request asked for") + } + } + + // Only the references are rewritten. Everything else the request offered + // survives. + if offer["id"] != "offer:pocra:common-services" { + t.Errorf("offer id = %v, want the one the request offered", offer["id"]) + } + if dig(offer, "provider", "id") != "pocra" { + t.Errorf("offer provider = %v, want the one the request offered", offer["provider"]) + } + + // A mapping transforms a payload; it does not assert who anyone is. The two + // Uri fields in particular are only whatever the caller sent -- in a deployed + // stack a container-internal address -- so echoing them would republish + // another party's routing details as ours. + beckncontext, _ := dig(answer, "context").(map[string]any) + for _, field := range []string{"bapId", "bapUri", "bppId", "bppUri"} { + if _, present := beckncontext[field]; present { + t.Errorf("response context carries %q; a mapping must not assert identity", field) + } + } + + // The v2 status enum is DRAFT, ACTIVE and CLOSED. QUOTED reads better and is + // refused by base schema validation. + if code := dig(commitments[0], "status", "descriptor", "code"); code != "DRAFT" { + t.Errorf("status = %v, want DRAFT -- QUOTED is not in the spec's enum", code) + } + + for _, entry := range resources { + if _, present := dig(entry, "quantity").(float64); !present { + t.Errorf("resource %v carries no quantity; the spec requires one on every commitment resource", + dig(entry, "id")) + } + } +} + +// warehouseResponse is a verbatim POCRA answer to a warehouse search, captured +// on 4 Sep 2026. It is here because warehouses come from a DIFFERENT BPP than +// the other three facility types, and its items are not the same shape: +// +// - no "category" tag at all, so facilityType cannot be read off the item +// - a "capacity_estimate" tag, which the other three never carry +// - price and rating fields the others do not have +// - a fully populated address rather than "Unknown" and "000000" +// - a real email, and "-" as a phone placeholder rather than "N/A" +// +// The doc's sample covered kvk alone, which is why none of this was visible +// until the mapping was run against the live API. +const warehouseResponse = `{ + "context": { "action": "search", "version": "1.1.0" }, + "responses": [{ + "context": { "action": "on_search", "version": "1.1.0" }, + "message": { "catalog": { + "descriptor": { "name": "Warehouse Services" }, + "providers": [{ + "id": "WAREHOUSE001", + "fulfillments": [{ "id": "f1", "type": "Service", + "categories": [{ "id": "c1", "descriptor": { "code": "GSW", "name": "General Storage Warehouse" } }] }], + "items": [ + { + "id": "WARE-1433", + "descriptor": { "name": "Shrirampur Midc Warehouse", + "short_desc": "Warehouse in Shrirampur, Shrirampur" }, + "address": { "address": "Mswc, Shrirampurc Midc, Plot No. X-33 Khandala", + "district": "Ahmednagar", "region": "Pune", + "taluka": "Shrirampur", "vilage": "Shrirampur", "pinCode": "413720" }, + "contact": { "person": "Warehouse Manager", "email": "shrirampur.wh@mswc.in", + "phone": "-", "webUrl": "https://warehouse.com" }, + "price": { "currency": "INR", "value": "12" }, + "rating": "4.2", + "category_ids": ["c1"], + "fulfillment_ids": ["f1"], + "tags": [{ "list": [ + { "descriptor": { "code": "distance" }, "value": "15.2093 km" }, + { "descriptor": { "code": "capacity_estimate" }, "value": "500 tons" } ] }] + }, + { + "id": "WARE-1406", + "descriptor": { "name": "Shrirampur Warehouse" }, + "address": { "address": "Mswc, Krushi Utpanna Bazar Samiti, Market Yard, Shrirampur", + "district": "Ahmednagar", "region": "Pune", + "taluka": "Shrirampur", "vilage": "Shrirampur", "pinCode": "413709" }, + "contact": { "person": "Warehouse Manager", "email": "shrirampur.wh@mswc.in", + "phone": "02422-222735", "webUrl": "https://warehouse.com" }, + "category_ids": ["c1"], + "fulfillment_ids": ["f1"], + "tags": [{ "list": [ + { "descriptor": { "code": "distance" }, "value": "18.8434 km" }, + { "descriptor": { "code": "capacity_estimate" }, "value": "1200 tons" } ] }] + } + ] + }] + } } + }] +}` + +// warehouseRequest asks for the one facility type whose answer is shaped +// differently from the rest. +func warehouseRequest(t *testing.T) string { + t.Helper() + var payload map[string]any + if err := json.Unmarshal([]byte(selectRequest), &payload); err != nil { + t.Fatalf("the fixture is not JSON: %v", err) + } + attributes(t, payload)["supportedFacilityTypes"] = []any{"Warehouse"} + body, err := json.Marshal(payload) + if err != nil { + t.Fatalf("could not rebuild the payload: %v", err) + } + return string(body) +} + +// runAgainst drives the step over a provider answering with the given body. +func runAgainst(t *testing.T, request, providerBody string) map[string]any { + t.Helper() + + mappings := serveMappings(t) + defer mappings.Close() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, providerBody) + })) + defer upstream.Close() + + mapper, closeMapper, err := jsonmapper.New(context.Background(), &jsonmapper.Config{}) + if err != nil { + t.Fatalf("failed to build the mapper: %v", err) + } + defer closeMapper() + + registry := &stubRegistry{plan: &model.ProviderRecord{ + BindingKey: shippedBindingKey, BaseURL: upstream.URL, + Actions: map[string]model.ActionPlan{ + "select": {Method: http.MethodPost, Path: "/search", + Mappings: mappings.URL + "/" + shippedMapping, TimeoutMs: 30000}, + }, + }} + + step, closeStep, err := AgricultureFacility.New(context.Background(), registry, mapper, + &AgricultureFacility.Config{ + BindingKeys: []string{shippedBindingKey}, + // Auth is per provider now; POCRA needs no credential and says so. + AuthByProvider: map[string]*common.AuthProfile{ + strings.Split(shippedBindingKey, "|")[0]: {Scheme: util.AuthSchemeNone}, + }, + }) + if err != nil { + t.Fatalf("failed to build the step: %v", err) + } + defer closeStep() + + stepCtx := &model.StepContext{Context: t.Context(), Body: []byte(request)} + if err := step.Run(stepCtx); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + var answer map[string]any + if err := json.Unmarshal(stepCtx.ResponseBody, &answer); err != nil { + t.Fatalf("the answer is not JSON: %v\n%s", err, stepCtx.ResponseBody) + } + return answer +} + +// A warehouse item carries no category tag, so facilityType cannot be read off +// the item the way it is for the other three types. Direct mode REQUIRES +// facilityType, so without a fallback every warehouse answer would violate the +// pack -- signed and delivered, because nothing in the adapter validates the +// answer against the pack. +func TestShippedMappingLabelsAWarehouseWithNoCategoryTag(t *testing.T) { + answer := runAgainst(t, warehouseRequest(t), warehouseResponse) + + resources := answerResources(t, answer) + if len(resources) != 2 { + t.Fatalf("got %d resources, want 2", len(resources)) + } + for _, entry := range resources { + id, _ := dig(entry, "id").(string) + attrs, ok := dig(entry, "resourceAttributes").(map[string]any) + if !ok { + t.Fatalf("resource %s carries no resourceAttributes", id) + } + if attrs["facilityType"] != "Warehouse" { + t.Errorf("%s: facilityType = %v, want Warehouse from the requested type", + id, attrs["facilityType"]) + } + } +} + +// The pack has a home for capacity -- value, unit and an optional basis -- and a +// warehouse is the one facility type POCRA reports it for. Dropping it would +// discard the single most useful thing a warehouse search returns. +func TestShippedMappingCarriesWarehouseCapacity(t *testing.T) { + answer := runAgainst(t, warehouseRequest(t), warehouseResponse) + + first, ok := dig(answerResources(t, answer)[0], "resourceAttributes").(map[string]any) + if !ok { + t.Fatal("the nearest resource carries no resourceAttributes") + } + capacity, ok := first["capacity"].(map[string]any) + if !ok { + t.Fatalf("the nearest warehouse carries no capacity; POCRA reported one") + } + if value, _ := capacity["value"].(float64); value != 500 { + t.Errorf("capacity.value = %v, want 500 as a number", capacity["value"]) + } + if capacity["unit"] != "tons" { + t.Errorf("capacity.unit = %v, want tons", capacity["unit"]) + } +} + +// "-" is a placeholder POCRA uses for a phone it does not have, alongside the +// "N/A", "Unknown" and "000000" the other providers use. +func TestShippedMappingScrubsTheWarehousePlaceholders(t *testing.T) { + answer := runAgainst(t, warehouseRequest(t), warehouseResponse) + + raw, err := json.Marshal(answer) + if err != nil { + t.Fatalf("could not re-encode the answer: %v", err) + } + if strings.Contains(string(raw), `"-"`) { + t.Error(`the answer carries "-", which is POCRA's placeholder for an absent phone`) + } + + resources := answerResources(t, answer) + first, _ := dig(resources[0], "resourceAttributes").(map[string]any) + contact, ok := first["publicContact"].(map[string]any) + if !ok { + t.Fatal("the nearest warehouse has a real email and must carry publicContact") + } + if contact["email"] != "shrirampur.wh@mswc.in" { + t.Errorf("publicContact.email = %v, want the address POCRA published", contact["email"]) + } + if _, present := contact["phone"]; present { + t.Error(`publicContact carries a phone, but POCRA sent "-" for it`) + } + + // The second warehouse has a real phone, so it must survive. + second, _ := dig(resources[1], "resourceAttributes").(map[string]any) + secondContact, ok := second["publicContact"].(map[string]any) + if !ok { + t.Fatal("the second warehouse has a real phone and must carry publicContact") + } + if secondContact["phone"] != "02422-222735" { + t.Errorf("publicContact.phone = %v, want the number POCRA published", secondContact["phone"]) + } +} + +// emptyResponse is what POCRA returns when the BPP for a category answers with +// nothing: HTTP 200, and responses[] empty. Observed live against chc on +// 4 Sep 2026, minutes after the same request returned five facilities. +// +// It is not an error. "No facilities near you" is a legitimate answer to a +// search, and the caller has to be able to tell it apart from a failure. +const emptyResponse = `{ + "context": { "action": "search", "version": "1.1.0" }, + "responses": [] +}` + +func TestShippedMappingAnswersAnEmptySearch(t *testing.T) { + answer := runAgainst(t, selectRequest, emptyResponse) + + // The answer still has to be a well-formed on_select that correlates, or + // the caller cannot tell "none found" from "something broke". + if got := dig(answer, "context", "action"); got != "on_select" { + t.Errorf("action = %v, want on_select", got) + } + if got := dig(answer, "context", "transactionId"); got != "f9e8d7c6-b5a4-4321-9876-543210fedcba" { + t.Errorf("transactionId = %v, want the one from the request", got) + } + + commitments, ok := dig(answer, "message", "contract", "commitments").([]any) + if !ok || len(commitments) != 1 { + t.Fatalf("got %d commitments, want 1", len(commitments)) + } + + // Whatever shape it takes, it must not claim facilities it does not have, + // and the offer must not reference resources that are not there. + resources, _ := dig(commitments[0], "resources").([]any) + offer, _ := dig(commitments[0], "offer").(map[string]any) + referenced, _ := offer["resourceIds"].([]any) + if len(resources) != 0 { + t.Errorf("got %d resources for an empty search, want none", len(resources)) + } + if len(referenced) != len(resources) { + t.Errorf("offer references %d resources but the answer carries %d", + len(referenced), len(resources)) + } +} + +// mixedResponse is POCRA answering a chc search with kvk facilities in it too. +// +// This is not hypothetical. POCRA caches per message_id for PT10M and ACCUMULATES +// across searches that share one: on 4 Sep 2026, two searches with the same +// message_id and different transaction_ids returned {kvk:5} and then +// {kvk:5, chc:5}. Beckn requires a fresh messageId per message, but an adapter +// cannot assume every caller is well behaved, and a retry that reuses one is the +// obvious way to trip it. +// +// The warehouse provider is in here too, carrying no category tag on its items -- +// only its fulfillment says GSW. It is the case that makes filtering on the item +// tag alone insufficient: with a tag-or-requested-type fallback, a leaked +// warehouse would be labelled as whatever was asked for and returned as one. +// +// And two providers that are not facilities at all. POCRA fans a search out to +// every BPP on its network, so a live warehouse search came back with 20 +// responses[] holding 182 items: 5 warehouses, 132 mandi price records, 6 +// administrative hierarchy rows and 4 weather station readings. apmcMandi is +// the awkward one -- its items have no tags key at all, so every tag lookup on +// them is a miss rather than a value. +const mixedResponse = `{ + "context": { "action": "search", "version": "1.1.0" }, + "responses": [ + { + "context": { "action": "on_search" }, + "message": { "catalog": { "providers": [{ + "id": "COMMON_PROVIDER_CHC", + "fulfillments": [{ "id": "f1_chc", + "categories": [{ "id": "c_chc", "descriptor": { "code": "chc", "name": "CHC" } }] }], + "items": [{ + "id": "COMMON-185", + "descriptor": { "name": "Custom Hiring Center" }, + "address": { "address": "Gogalgaon, Rahta", "district": "Ahmednagar", + "region": "Unknown", "taluka": "Rahta", "vilage": "Gogalgaon", + "pinCode": "000000" }, + "contact": { "person": "Rajesh Thoke", "email": "N/A", "phone": "9765007429", + "webUrl": "https://provider.mahapocra.gov.in" }, + "tags": [{ "list": [ + { "descriptor": { "code": "distance" }, "value": "14 Km" }, + { "descriptor": { "code": "category" }, "value": "chc" } ] }] + }] + }] } } + }, + { + "context": { "action": "on_search" }, + "message": { "catalog": { "providers": [{ + "id": "COMMON_PROVIDER_KVK", + "fulfillments": [{ "id": "f1_kvk", + "categories": [{ "id": "c_kvk", "descriptor": { "code": "kvk", "name": "KVK" } }] }], + "items": [{ + "id": "COMMON-55031", + "descriptor": { "name": "Krishi Vigyan Kendra, Babhleshwar" }, + "address": { "address": "KVK Babhleshwar", "district": "Ahmednagar", + "region": "Unknown", "taluka": "Unknown", "vilage": "Unknown", + "pinCode": "000000" }, + "contact": { "person": "President", "email": "N/A", "phone": "N/A", + "webUrl": "https://provider.mahapocra.gov.in" }, + "tags": [{ "list": [ + { "descriptor": { "code": "distance" }, "value": "11 Km" }, + { "descriptor": { "code": "category" }, "value": "kvk" } ] }] + }] + }] } } + }, + { + "context": { "action": "on_search" }, + "message": { "catalog": { "providers": [{ + "id": "apmcMandi", + "descriptor": { "name": "Maharashtra Mandi" }, + "items": [{ + "id": "mandi-6001", + "descriptor": { "name": "Tomato" }, + "location_ids": ["loc1"], + "price": { "minimum_value": "6000", "maximum_value": "13000", "estimated_value": "9000" }, + "time": { "label": "2026-09-03" } + }] + }] } } + }, + { + "context": { "action": "on_search" }, + "message": { "catalog": { "providers": [{ + "id": "ADMIN001", + "descriptor": { "name": "Administrative Information" }, + "fulfillments": [{ "id": "f_adm", + "categories": [{ "id": "c_adm", "descriptor": { "code": "ADM", "name": "Administrative Hierarchy" } }] }], + "items": [{ + "id": "ADM-802776", + "descriptor": { "name": "Nashik (M Corp.)" }, + "address": { "address": "Nashik", "district": "Nashik", "region": "Nashik", + "taluka": "Nashik", "vilage": "Nashik (M Corp.)", "pinCode": "422001" }, + "tags": [{ "list": [ + { "descriptor": { "code": "district_code" }, "value": "516" }, + { "descriptor": { "code": "village_code" }, "value": "802776" } ] }] + }] + }] } } + }, + { + "context": { "action": "on_search" }, + "message": { "catalog": { "providers": [{ + "id": "WAREHOUSE001", + "fulfillments": [{ "id": "f1", + "categories": [{ "id": "c1", "descriptor": { "code": "GSW", "name": "General Storage Warehouse" } }] }], + "items": [{ + "id": "WARE-1433", + "descriptor": { "name": "Shrirampur Midc Warehouse" }, + "address": { "address": "Mswc Shrirampur", "district": "Ahmednagar", + "region": "Pune", "taluka": "Shrirampur", "vilage": "Shrirampur", + "pinCode": "413720" }, + "contact": { "person": "Warehouse Manager", "email": "shrirampur.wh@mswc.in", + "phone": "-", "webUrl": "https://warehouse.com" }, + "tags": [{ "list": [ + { "descriptor": { "code": "distance" }, "value": "15.2093 km" }, + { "descriptor": { "code": "capacity_estimate" }, "value": "500 tons" } ] }] + }] + }] } } + } + ] +}` + +// requestFor builds a select for one governed facility type. +func requestFor(t *testing.T, facilityType string) string { + t.Helper() + var payload map[string]any + if err := json.Unmarshal([]byte(selectRequest), &payload); err != nil { + t.Fatalf("the fixture is not JSON: %v", err) + } + attributes(t, payload)["supportedFacilityTypes"] = []any{facilityType} + body, err := json.Marshal(payload) + if err != nil { + t.Fatalf("could not rebuild the payload: %v", err) + } + return string(body) +} + +// A request for one facility type must be answered with that type and nothing +// else, whatever POCRA chose to include. +func TestShippedMappingReturnsOnlyTheRequestedFacilityType(t *testing.T) { + for _, tc := range []struct { + facilityType string + wantIDs []string + }{ + {"CustomHiringCentre", []string{"res:pocra:facility:COMMON-185"}}, + {"KrishiVigyanKendra", []string{"res:pocra:facility:COMMON-55031"}}, + {"Warehouse", []string{"res:pocra:facility:WARE-1433"}}, + } { + t.Run(tc.facilityType, func(t *testing.T) { + answer := runAgainst(t, requestFor(t, tc.facilityType), mixedResponse) + resources := answerResources(t, answer) + + got := make([]string, 0, len(resources)) + for _, entry := range resources { + id, _ := dig(entry, "id").(string) + got = append(got, id) + + attrs, ok := dig(entry, "resourceAttributes").(map[string]any) + if !ok { + t.Fatalf("resource %s carries no resourceAttributes", id) + } + if attrs["facilityType"] != tc.facilityType { + t.Errorf("%s: facilityType = %v, want %s -- a leaked facility must not be relabelled", + id, attrs["facilityType"], tc.facilityType) + } + } + if !slices.Equal(got, tc.wantIDs) { + t.Errorf("returned %v, want exactly %v", got, tc.wantIDs) + } + + // Nothing from a provider that is not a facility may reach the + // answer -- not a mandi price, not an administrative code. + raw, err := json.Marshal(answer) + if err != nil { + t.Fatalf("could not re-encode the answer: %v", err) + } + for _, leak := range []string{"mandi-6001", "estimated_value", "ADM-802776", + "district_code", "802776", "Maharashtra Mandi"} { + if strings.Contains(string(raw), leak) { + t.Errorf("the answer leaked %q from a provider that is not a facility", leak) + } + } + + // The offer must not reference what was filtered out either. + commitments, _ := dig(answer, "message", "contract", "commitments").([]any) + offer, _ := dig(commitments[0], "offer").(map[string]any) + referenced, _ := offer["resourceIds"].([]any) + if len(referenced) != len(got) { + t.Errorf("offer references %d resources but the answer carries %d", + len(referenced), len(got)) + } + }) + } +} + +// The answer echoes whatever @context the request declared, rather than +// restating a pack URL of its own. +// +// Hardcoding it meant the mapping had to know which identifier is current, and +// could contradict what the caller actually sent. Both forms below are in use: +// the published pack URL, which is what resolves today and what the fixture +// declares, and the schemas.openagrinet.global identifier the packs name in +// their own x-jsonld -- which has no DNS record, so a caller cannot be assumed +// to send either one. +// +// The pack does not constrain @context -- it appears only under x-jsonld, which +// is annotation metadata a JSON Schema validator ignores -- so conformance +// cannot catch a wrong one. This test is the only thing that does. +func TestShippedMappingEchoesTheCallersContext(t *testing.T) { + for _, declared := range []string{ + declaredContext, + "https://schemas.openagrinet.global/schema/AgricultureFacility/v0.1/context.jsonld", + } { + t.Run(declared, func(t *testing.T) { + var payload map[string]any + if err := json.Unmarshal([]byte(selectRequest), &payload); err != nil { + t.Fatalf("the fixture is not JSON: %v", err) + } + attributes(t, payload)["@context"] = declared + body, err := json.Marshal(payload) + if err != nil { + t.Fatalf("could not rebuild the payload: %v", err) + } + + _, answer, runErr := runSelect(t, string(body)) + if runErr != nil { + t.Fatalf("Run() returned an unexpected error: %v", runErr) + } + for _, entry := range answerResources(t, answer) { + id, _ := dig(entry, "id").(string) + got := dig(entry, "resourceAttributes", "@context") + if got != declared { + t.Errorf("%s: @context = %v, want the one the request declared (%s)", + id, got, declared) + } + } + }) + } +} + +// -------------------------------------------------------------------------- +// Fan-out: one payload, one upstream call per facility type it asks for. +// +// POCRA's search takes exactly ONE category code -- a comma-separated pair +// answers 200 with no providers, and a category array is refused with "Schema +// validation failed", both verified against the live API. So a payload naming +// two types cannot be served by one call. +// +// Before fan-out existed the request half sent supportedFacilityTypes[0] and +// the response half filtered to that same entry, so a two-type search returned +// the first type and silently dropped the rest. Nothing in the answer said a +// type had been asked for and lost, which is what made it worth finding. +// -------------------------------------------------------------------------- + +// pocraByCategory is a fake POCRA that answers each category with its own +// captured response, and records what it was asked. +// +// Routing on the category code is the whole point: a fake that returns one +// canned body whatever it is asked cannot tell a fan-out that sent the right +// codes from one that sent the same code twice. +type pocraByCategory struct { + mu sync.Mutex + byCode map[string]string + codes []string + messages []string +} + +func (p *pocraByCategory) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var sent map[string]any + if err := json.NewDecoder(r.Body).Decode(&sent); err != nil { + http.Error(w, "not JSON", http.StatusBadRequest) + return + } + code, _ := dig(sent, "message", "intent", "category", "descriptor", "code").(string) + message, _ := dig(sent, "context", "message_id").(string) + + p.mu.Lock() + p.codes = append(p.codes, code) + p.messages = append(p.messages, message) + body, known := p.byCode[code] + p.mu.Unlock() + + if !known { + // A code with no fixture is a fan-out that sent something unasked for, + // and answering it with anything would hide that. + http.Error(w, "unexpected category "+code, http.StatusBadRequest) + return + } + fmt.Fprint(w, body) +} + +// asked returns the category codes POCRA was sent, sorted so an assertion does +// not depend on which concurrent call arrived first. +func (p *pocraByCategory) asked() []string { + p.mu.Lock() + defer p.mu.Unlock() + sorted := append([]string(nil), p.codes...) + sort.Strings(sorted) + return sorted +} + +// runSplitSearch runs the shipped mapping against a fake POCRA that answers per +// category, and returns the answer with the fake for inspection. +func runSplitSearch(t *testing.T, types []string, byCode map[string]string) (map[string]any, *pocraByCategory) { + t.Helper() + + mappings := serveMappings(t) + defer mappings.Close() + + pocra := &pocraByCategory{byCode: byCode} + upstream := httptest.NewServer(pocra) + defer upstream.Close() + + mapper, closeMapper, err := jsonmapper.New(context.Background(), &jsonmapper.Config{}) + if err != nil { + t.Fatalf("failed to build the mapper: %v", err) + } + defer closeMapper() + + registry := &stubRegistry{plan: &model.ProviderRecord{ + BindingKey: shippedBindingKey, BaseURL: upstream.URL, + Actions: map[string]model.ActionPlan{ + "select": {Method: http.MethodPost, Path: "/search", + Mappings: mappings.URL + "/" + shippedMapping, TimeoutMs: 30000}, + }, + }} + + step, closeStep, err := AgricultureFacility.New(context.Background(), registry, mapper, + &AgricultureFacility.Config{ + BindingKeys: []string{shippedBindingKey}, + // Auth is per provider now; POCRA needs no credential and says so. + AuthByProvider: map[string]*common.AuthProfile{ + strings.Split(shippedBindingKey, "|")[0]: {Scheme: util.AuthSchemeNone}, + }, + }) + if err != nil { + t.Fatalf("failed to build the step: %v", err) + } + defer closeStep() + + var payload map[string]any + if err := json.Unmarshal([]byte(selectRequest), &payload); err != nil { + t.Fatalf("the fixture is not JSON: %v", err) + } + attributes(t, payload)["supportedFacilityTypes"] = toAny(types) + body, err := json.Marshal(payload) + if err != nil { + t.Fatalf("could not rebuild the payload: %v", err) + } + + stepCtx := &model.StepContext{Context: t.Context(), Body: body} + if err := step.Run(stepCtx); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + var answer map[string]any + if err := json.Unmarshal(stepCtx.ResponseBody, &answer); err != nil { + t.Fatalf("the answer is not JSON: %v\n%s", err, stepCtx.ResponseBody) + } + return answer, pocra +} + +func toAny(values []string) []any { + out := make([]any, len(values)) + for index, value := range values { + out[index] = value + } + return out +} + +// typesIn counts the answer's resources by facilityType. +func typesIn(t *testing.T, answer map[string]any) map[string]int { + t.Helper() + counts := map[string]int{} + for _, entry := range answerResources(t, answer) { + got, _ := dig(entry, "resourceAttributes", "facilityType").(string) + counts[got]++ + } + return counts +} + +// The whole point: two types asked for, two types answered. +// +// This is the case that was broken. It returned KrishiVigyanKendra alone, and +// the Warehouse the caller also asked for was dropped without a word. +func TestATwoTypeSearchIsAnsweredWithBothTypes(t *testing.T) { + answer, pocra := runSplitSearch(t, + []string{"KrishiVigyanKendra", "Warehouse"}, + map[string]string{"kvk": providerResponse, "warehouse": warehouseResponse}) + + if want := []string{"kvk", "warehouse"}; !slices.Equal(pocra.asked(), want) { + t.Errorf("POCRA was asked for %v, want one call per type %v", pocra.asked(), want) + } + got := typesIn(t, answer) + if got["KrishiVigyanKendra"] == 0 || got["Warehouse"] == 0 { + t.Errorf("answer carries %v, want both requested types present", got) + } + for facilityType := range got { + if facilityType != "KrishiVigyanKendra" && facilityType != "Warehouse" { + t.Errorf("answer carries %q, which was not asked for", facilityType) + } + } +} + +// All four, because they do not all state their type the same way -- three +// carry a category tag and a COMMON_PROVIDER_ id, a warehouse carries neither. +func TestEveryGovernedTypeCanBeAskedForAtOnce(t *testing.T) { + answer, pocra := runSplitSearch(t, + []string{"KrishiVigyanKendra", "CustomHiringCentre", "SoilTestingFacility", "Warehouse"}, + map[string]string{ + "kvk": providerResponse, + "chc": mixedResponse, + "soil_lab": mixedResponse, + "warehouse": warehouseResponse, + }) + + if want := []string{"chc", "kvk", "soil_lab", "warehouse"}; !slices.Equal(pocra.asked(), want) { + t.Errorf("POCRA was asked for %v, want %v", pocra.asked(), want) + } + got := typesIn(t, answer) + // soil_lab is routed to a fixture that carries none, so three of the four + // have facilities to return. The assertion is that nothing UNASKED appears + // and that the types with data all do. + for _, want := range []string{"KrishiVigyanKendra", "CustomHiringCentre", "Warehouse"} { + if got[want] == 0 { + t.Errorf("answer carries %v, want %s present", got, want) + } + } + if got["SoilTestingFacility"] != 0 { + t.Errorf("answer carries %d SoilTestingFacility, but the fixture has none", + got["SoilTestingFacility"]) + } +} + +// A leaked facility is still dropped when fanning out. +// +// mixedResponse is POCRA answering a chc search with kvk, mandi, administrative +// and warehouse providers in it too. Asking for chc and warehouse must return +// the chc facilities and the warehouse ones -- not the kvk that leaked into the +// chc answer, and not the mandi or administrative providers, which are not +// facilities at all. +func TestASplitSearchStillDropsWhatWasNotAskedFor(t *testing.T) { + answer, _ := runSplitSearch(t, + []string{"CustomHiringCentre", "Warehouse"}, + map[string]string{"chc": mixedResponse, "warehouse": warehouseResponse}) + + got := typesIn(t, answer) + if got["KrishiVigyanKendra"] != 0 { + t.Errorf("the kvk that leaked into the chc answer was returned: %v", got) + } + for facilityType, count := range got { + if facilityType != "CustomHiringCentre" && facilityType != "Warehouse" { + t.Errorf("answer carries %d of %q, which was not asked for", count, facilityType) + } + } + if got["CustomHiringCentre"] == 0 || got["Warehouse"] == 0 { + t.Errorf("answer carries %v, want both requested types", got) + } +} + +// Each fan-out call carries its OWN message_id, and none of them is the +// caller's. +// +// POCRA keeps a message_id's answers for ten minutes and returns the union of +// everything asked for under it -- verified against the live API, where a kvk +// search under a reused id came back carrying an earlier chc search's +// facilities. Sending one id across the fan-out would make every call return +// every other call's facilities, so this is what stops the adapter causing the +// leak it also filters. +func TestEachSearchCallCarriesItsOwnRequestId(t *testing.T) { + _, pocra := runSplitSearch(t, + []string{"KrishiVigyanKendra", "Warehouse"}, + map[string]string{"kvk": providerResponse, "warehouse": warehouseResponse}) + + if len(pocra.messages) != 2 { + t.Fatalf("POCRA saw %d calls, want 2", len(pocra.messages)) + } + if pocra.messages[0] == pocra.messages[1] { + t.Errorf("both calls used message_id %q, so POCRA would blend their answers", + pocra.messages[0]) + } + for _, message := range pocra.messages { + // POCRA's schema refuses a message_id that is not a UUID: a derived one + // like "-kvk" is answered with 400 Schema validation failed. + if _, err := uuid.Parse(message); err != nil { + t.Errorf("message_id %q is not a UUID, which POCRA refuses: %v", message, err) + } + if message == callerMessageID { + t.Errorf("a call reused the caller's messageId %q", message) + } + } +} + +// An ungoverned type anywhere in the list is refused, not just in first place. +// +// The precondition used to read supportedFacilityTypes[0], so a payload naming +// a governed type first and nonsense second passed the check and then fanned +// out to a call POCRA would refuse. +func TestAnUngovernedTypeIsRefusedWhereverItAppears(t *testing.T) { + for _, types := range [][]string{ + {"NotAFacility"}, + {"KrishiVigyanKendra", "NotAFacility"}, + {"NotAFacility", "KrishiVigyanKendra"}, + } { + t.Run(strings.Join(types, "+"), func(t *testing.T) { + var payload map[string]any + if err := json.Unmarshal([]byte(selectRequest), &payload); err != nil { + t.Fatalf("the fixture is not JSON: %v", err) + } + attributes(t, payload)["supportedFacilityTypes"] = toAny(types) + body, err := json.Marshal(payload) + if err != nil { + t.Fatalf("could not rebuild the payload: %v", err) + } + + _, _, runErr := runSelect(t, string(body)) + if runErr == nil { + t.Fatal("the payload was served, want it refused for naming an ungoverned type") + } + if !strings.Contains(runErr.Error(), "governed facility type") { + t.Errorf("error = %v, want the mapping's own explanation", runErr) + } + }) + } +} + +// -------------------------------------------------------------------------- +// A placeholder value must not crash the whole search. +// +// $number() throws on anything it cannot parse, and the throw is not local to +// the item carrying the bad value -- it propagates out of the whole response +// mapping. One placeholder distance or capacity_estimate on one item then +// failed a search that also carried good facilities, which is worse than +// dropping the one bad field: it drops everything. +// -------------------------------------------------------------------------- + +// pocraSingleWith builds a minimal, valid POCRA envelope around one item's +// extra tags, so a test can vary just the field it is checking. +func pocraSingleWith(providerID, categoryCode, itemID, extraTags string) string { + return `{ "responses": [ { + "context": { "action": "on_search", "version": "1.1.0" }, + "message": { "catalog": { "descriptor": { "name": "x" }, + "providers": [{ + "id": "` + providerID + `", + "fulfillments": [{ "id": "f1", "categories": [{ "descriptor": { "code": "` + categoryCode + `" } }] }], + "items": [ + { "id": "` + itemID + `", "descriptor": {"name":"a"}, "address": {}, "contact": {}, + "fulfillment_ids": ["f1"], "category_ids": ["c1"], + "tags": [{ "list": [` + extraTags + `] }] } + ] + }] + } } + } ] }` +} + +// A placeholder distance is dropped from ranking, not fatal to the search. +// +// POCRA reports "165 Km" when it knows the distance and "Unknown" -- among the +// same placeholders this file already strips elsewhere -- when it does not. +// TestReadingTreatsAnAbsentDistanceAsUnranked (below, existing) covers the +// field being missing outright; this covers it being PRESENT but unparseable, +// which used to throw D3030 rather than return the resource unranked. +func TestAPlaceholderDistanceDoesNotFailTheSearch(t *testing.T) { + body := `{ "responses": [ { + "context": { "action": "on_search", "version": "1.1.0" }, + "message": { "catalog": { "descriptor": { "name": "x" }, + "providers": [{ + "id": "COMMON_PROVIDER_KVK", + "fulfillments": [{ "id": "f1", "categories": [{ "descriptor": { "code": "kvk" } }] }], + "items": [ + { "id": "NEAR", "descriptor": {"name":"a"}, "address": {}, "contact": {}, + "fulfillment_ids": ["f1"], "category_ids": ["c1"], + "tags": [{ "list": [{ "descriptor": {"code":"distance"}, "value": "12 Km" }] }] }, + { "id": "PLACEHOLDER", "descriptor": {"name":"b"}, "address": {}, "contact": {}, + "fulfillment_ids": ["f1"], "category_ids": ["c1"], + "tags": [{ "list": [{ "descriptor": {"code":"distance"}, "value": "Unknown" }] }] } + ] + }] + } } + } ] }` + + answer := runAgainst(t, requestFor(t, "KrishiVigyanKendra"), body) + resources := answerResources(t, answer) + if len(resources) != 2 { + t.Fatalf("got %d resources, want 2 -- a placeholder distance must not drop the item, "+ + "only its ranking", len(resources)) + } + // The good reading sorts first; the placeholder, like an absent distance, + // sorts last. + if got := dig(resources[0], "id"); got != "res:pocra:facility:NEAR" { + t.Errorf("resources[0] = %v, want NEAR (the one with a real distance) first", got) + } + if got := dig(resources[1], "id"); got != "res:pocra:facility:PLACEHOLDER" { + t.Errorf("resources[1] = %v, want PLACEHOLDER last, same as an absent distance", got) + } +} + +// A non-numeric distance with no space -- "12Km" rather than "12 Km" -- is the +// same failure by a different route: $substringBefore returns the WHOLE string +// when there is no separator, so "12Km" reached $number unparsed. +func TestADistanceWithNoSpaceDoesNotFailTheSearch(t *testing.T) { + body := pocraSingleWith("COMMON_PROVIDER_KVK", "kvk", "NOSPACE", + `{ "descriptor": {"code":"distance"}, "value": "12Km" }`) + answer := runAgainst(t, requestFor(t, "KrishiVigyanKendra"), body) + if len(answerResources(t, answer)) != 1 { + t.Fatalf("got %d resources, want 1 -- a malformed distance must not fail the search", + len(answerResources(t, answer))) + } +} + +// A placeholder capacity_estimate is omitted, not fatal, on the one facility +// type that reports it. +func TestAPlaceholderCapacityDoesNotFailTheSearch(t *testing.T) { + body := pocraSingleWith("WAREHOUSE001", "GSW", "WARE-PLACEHOLDER", + `{ "descriptor": {"code":"capacity_estimate"}, "value": "Unknown" }`) + answer := runAgainst(t, requestFor(t, "Warehouse"), body) + resources := answerResources(t, answer) + if len(resources) != 1 { + t.Fatalf("got %d resources, want 1 -- a placeholder capacity must not fail the search", + len(resources)) + } + if capacity := dig(resources[0], "resourceAttributes", "capacity"); capacity != nil { + t.Errorf("capacity = %v, want omitted for a placeholder value", capacity) + } +} + +// A real capacity still parses -- the guard added for the placeholder must not +// have broken the value it is meant to let through. +func TestARealCapacityStillParses(t *testing.T) { + body := pocraSingleWith("WAREHOUSE001", "GSW", "WARE-REAL", + `{ "descriptor": {"code":"capacity_estimate"}, "value": "500 tons" }`) + answer := runAgainst(t, requestFor(t, "Warehouse"), body) + resources := answerResources(t, answer) + if len(resources) != 1 { + t.Fatalf("got %d resources, want 1", len(resources)) + } + capacity, ok := dig(resources[0], "resourceAttributes", "capacity").(map[string]any) + if !ok { + t.Fatalf("capacity = %v, want a parsed {value, unit}", + dig(resources[0], "resourceAttributes", "capacity")) + } + if capacity["value"] != float64(500) || capacity["unit"] != "tons" { + t.Errorf("capacity = %v, want {value: 500, unit: \"tons\"}", capacity) + } +} + +// A provider naming more than one fulfillment category must not crash the +// search either. +// +// $lookup throws T0410 on anything but a scalar key, and +// provider.fulfillments.categories.descriptor.code is a SEQUENCE the moment a +// provider has more than one fulfillment or a fulfillment has more than one +// category. Every shipped fixture has exactly one of each, so nothing else in +// this suite reaches the path. +// +// The provider id here deliberately does not match COMMON_PROVIDER_*, so type +// resolution falls all the way through to the fulfillment-category source this +// is testing -- the id and item-tag sources are tried first and would mask the +// bug otherwise. +func TestAProviderWithTwoFulfillmentCategoriesDoesNotFailTheSearch(t *testing.T) { + body := `{ "responses": [ { + "context": { "action": "on_search", "version": "1.1.0" }, + "message": { "catalog": { "descriptor": { "name": "x" }, + "providers": [{ + "id": "NOT_A_COMMON_PROVIDER", + "fulfillments": [ + { "id": "f1", "categories": [{ "descriptor": { "code": "kvk" } }] }, + { "id": "f2", "categories": [{ "descriptor": { "code": "chc" } }] } + ], + "items": [ + { "id": "MIXED-1", "descriptor": {"name":"a"}, "address": {}, "contact": {}, + "fulfillment_ids": ["f1","f2"], "category_ids": ["c1"], + "tags": [{ "list": [{ "descriptor": {"code":"distance"}, "value": "12 Km" }] }] } + ] + }] + } } + } ] }` + answer := runAgainst(t, requestFor(t, "KrishiVigyanKendra"), body) + if len(answerResources(t, answer)) != 1 { + t.Fatalf("got %d resources, want 1 -- a provider with two fulfillment categories "+ + "must not fail the search", len(answerResources(t, answer))) + } +} + +// An item carrying more than one category tag must not crash the search +// either -- same T0410 hazard as the fulfillment-categories case above, but +// for $item.tags.list[descriptor.code = "category"].value. +// +// providerID is deliberately not COMMON_PROVIDER_* and categoryCode is +// deliberately ungoverned, so type resolution falls through to the item's own +// tag -- the two sources tried first would otherwise mask the bug. +func TestADuplicateCategoryTagDoesNotFailTheSearch(t *testing.T) { + body := pocraSingleWith("NOT_A_COMMON_PROVIDER", "not_governed", "DUP-CAT-1", + `{ "descriptor": {"code":"category"}, "value": "kvk" }, + { "descriptor": {"code":"category"}, "value": "chc" }`) + answer := runAgainst(t, requestFor(t, "KrishiVigyanKendra"), body) + resources := answerResources(t, answer) + if len(resources) != 1 { + t.Fatalf("got %d resources, want 1 -- a duplicated category tag must not fail the search", + len(resources)) + } + if got := dig(resources[0], "resourceAttributes", "facilityType"); got != "KrishiVigyanKendra" { + t.Errorf("facilityType = %v, want the first tag that resolves", got) + } +} + +// A duplicated distance tag must not crash the search -- same T0410 hazard as +// the placeholder-distance case above, but from two VALID values rather than +// one placeholder. +func TestADuplicateDistanceTagDoesNotFailTheSearch(t *testing.T) { + body := pocraSingleWith("COMMON_PROVIDER_KVK", "kvk", "DUP-DIST-1", + `{ "descriptor": {"code":"distance"}, "value": "12 Km" }, + { "descriptor": {"code":"distance"}, "value": "20 Km" }`) + answer := runAgainst(t, requestFor(t, "KrishiVigyanKendra"), body) + resources := answerResources(t, answer) + if len(resources) != 1 { + t.Fatalf("got %d resources, want 1 -- a duplicated distance tag must not fail the search", + len(resources)) + } +} + +// A duplicated capacity_estimate tag must not crash the search, and the value +// taken is the first, same as distance above. +func TestADuplicateCapacityTagDoesNotFailTheSearch(t *testing.T) { + body := pocraSingleWith("WAREHOUSE001", "GSW", "DUP-CAP-1", + `{ "descriptor": {"code":"capacity_estimate"}, "value": "500 tons" }, + { "descriptor": {"code":"capacity_estimate"}, "value": "800 tons" }`) + answer := runAgainst(t, requestFor(t, "Warehouse"), body) + resources := answerResources(t, answer) + if len(resources) != 1 { + t.Fatalf("got %d resources, want 1 -- a duplicated capacity_estimate tag must not fail the search", + len(resources)) + } + capacity, ok := dig(resources[0], "resourceAttributes", "capacity").(map[string]any) + if !ok { + t.Fatalf("capacity = %v, want a parsed {value, unit}", + dig(resources[0], "resourceAttributes", "capacity")) + } + if capacity["value"] != float64(500) || capacity["unit"] != "tons" { + t.Errorf("capacity = %v, want the FIRST tag's {value: 500, unit: \"tons\"}", capacity) + } +} + +// supportedFacilityTypes repeating a type is refused rather than absorbed -- +// it would otherwise pass both governed-type checks and make one duplicate +// upstream call per repeat for nothing, up to the fan-out ceiling. +func TestDuplicateFacilityTypesAreRefused(t *testing.T) { + var payload map[string]any + if err := json.Unmarshal([]byte(selectRequest), &payload); err != nil { + t.Fatalf("the fixture is not JSON: %v", err) + } + attributes(t, payload)["supportedFacilityTypes"] = toAny([]string{"Warehouse", "Warehouse"}) + body, err := json.Marshal(payload) + if err != nil { + t.Fatalf("could not rebuild the payload: %v", err) + } + + _, _, runErr := runSelect(t, string(body)) + if runErr == nil { + t.Fatal("the payload was served, want it refused for repeating a facility type") + } + if !strings.Contains(runErr.Error(), "not repeat") { + t.Errorf("error = %v, want the mapping's own explanation", runErr) + } +} diff --git a/pkg/plugin/implementation/AgricultureFacility/payload.go b/pkg/plugin/implementation/AgricultureFacility/payload.go new file mode 100644 index 00000000..3f85ffba --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/payload.go @@ -0,0 +1,134 @@ +// payload.go is the one place in this repo that reads a field out of a Beckn +// payload in Go rather than in a mapping. +// +// It exists because the facility types have to be known BEFORE any mapping +// runs: they decide how many single-type payloads one inbound payload becomes, +// and each of those is then mapped and called on its own. Reading them here is +// what keeps splitting out of jsonmapper, out of pkg/plugin/definition and out +// of internal/upstream -- see search.go. +// +// The cost, stated plainly because it is the one real cost of that: this path +// is compiled in. The mapping's required: checks read the SAME path in +// JSONata, so a Beckn payload shape change touches both, and only one of them +// is a config edit. Keep the two in step; a mismatch means a payload the +// checks accept and this refuses, which reads as an adapter bug rather than as +// a bad request. +// +// Separated from the package clause by a blank line on purpose: the package's +// own doc comment is in AgricultureFacility.go, and this is a note about one +// file. + +package AgricultureFacility + +import ( + "fmt" + "slices" + + "github.com/beckn-one/beckn-onix/pkg/model" +) + +// facilityTypesFrom returns the facility types a payload asks for, in the +// order the payload wrote them -- one upstream call each. +// +// Every failure is a bad request rather than an adapter fault: the payload is +// the thing that is wrong, and the caller is the only one who can fix it. The +// mapping's own required: checks refuse most of these first, with better +// messages; these are what is left if a check is relaxed or a payload reaches +// here another way, and they exist so that path is a refusal rather than a +// panic on a nil map. +func facilityTypesFrom(beckn any) ([]string, error) { + document, ok := beckn.(map[string]any) + if !ok { + return nil, model.NewBadReqErr("", fmt.Errorf( + "agriculture facility: the payload is %T, not an object, so it names no commitment to search from", beckn)) + } + + commitments, _ := dig(document, "message", "contract", "commitments").([]any) + if len(commitments) == 0 { + return nil, model.NewBadReqErr("", fmt.Errorf( + "agriculture facility: the payload carries no commitment to read a facility search from")) + } + commitment, ok := commitments[0].(map[string]any) + if !ok { + return nil, model.NewBadReqErr("", fmt.Errorf( + "agriculture facility: the payload's first commitment is %T, not an object", commitments[0])) + } + + resources, _ := commitment["resources"].([]any) + if len(resources) == 0 { + return nil, model.NewBadReqErr("", fmt.Errorf( + "agriculture facility: the payload's commitment carries no resource to read a facility search from")) + } + resource, ok := resources[0].(map[string]any) + if !ok { + return nil, model.NewBadReqErr("", fmt.Errorf( + "agriculture facility: the payload's first resource is %T, not an object", resources[0])) + } + + declared := dig(resource, "resourceAttributes", "supportedFacilityTypes") + + // A bare string is one value: the same instruction as a one-element list, + // which is what a caller writing a single type may well send. + var declaredList []any + switch typed := declared.(type) { + case nil: + declaredList = nil + case []any: + declaredList = typed + default: + declaredList = []any{typed} + } + if len(declaredList) == 0 { + return nil, model.NewBadReqErr("", fmt.Errorf( + "agriculture facility: the payload names no facility type in supportedFacilityTypes, "+ + "so there is nothing to ask the provider for")) + } + + // Each value becomes a category code in the request half's $codes lookup, + // so a non-string is a request this capability cannot build -- refused + // here rather than sent as a null category POCRA answers with everything. + values := make([]string, 0, len(declaredList)) + for _, value := range declaredList { + text, ok := value.(string) + if !ok { + return nil, model.NewBadReqErr("", fmt.Errorf( + "agriculture facility: supportedFacilityTypes contains %v (%T), which is not a facility type", + value, value)) + } + values = append(values, text) + } + + // A repeated type is refused rather than absorbed. It would otherwise pass + // every governed-type check and make one duplicate upstream call per + // repeat for nothing, up to MaxFacilityTypes. + // + // Checked here rather than left to the mapping, which states the same rule + // in its own required: block: the split happens before either half of the + // mapping runs, and each part it produces names exactly one type, so the + // mapping's version can no longer see a repeat. The wording matches so a + // caller cannot tell which guard answered. + for index, value := range values { + if slices.Contains(values[:index], value) { + return nil, model.NewBadReqErr("", fmt.Errorf( + "agriculture facility: supportedFacilityTypes must not repeat a facility type, and %q appears twice", + value)) + } + } + return values, nil +} + +// dig walks a chain of object keys, returning nil the moment one is absent or +// is not an object. Written out rather than reached for from a library because +// a mistyped key must read as an absent field, which is a bad request, and not +// as a panic. +func dig(document map[string]any, keys ...string) any { + var current any = document + for _, key := range keys { + object, ok := current.(map[string]any) + if !ok { + return nil + } + current = object[key] + } + return current +} diff --git a/pkg/plugin/implementation/AgricultureFacility/payload_test.go b/pkg/plugin/implementation/AgricultureFacility/payload_test.go new file mode 100644 index 00000000..c58701d6 --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/payload_test.go @@ -0,0 +1,133 @@ +package AgricultureFacility + +// payload_test.go covers the one path read this package does in Go rather +// than in a mapping. Table-driven because the interesting cases are all +// malformed payloads, and each one has to say something a caller can act on. +// +// An INTERNAL test package, unlike every other test file here: what it tests +// is unexported, and exporting a payload path read so a test can reach it +// would publish it as something another package may depend on. + +import ( + "encoding/json" + "strings" + "testing" +) + +// decode is what Step.Run works from: the payload, already decoded into +// map[string]any / []any by encoding/json. +func decode(t *testing.T, body string) any { + t.Helper() + var beckn any + if err := json.Unmarshal([]byte(body), &beckn); err != nil { + t.Fatalf("the fixture is not JSON: %v", err) + } + return beckn +} + +// A payload naming three types yields three values, in the order the payload +// wrote them -- the order the answers come back in, which is the only thing +// making the result reproducible. +func TestFacilityTypesFromReadsThemInPayloadOrder(t *testing.T) { + t.Parallel() + + beckn := decode(t, `{"message":{"contract":{"commitments":[{"resources":[{"resourceAttributes":{ + "supportedFacilityTypes":["KrishiVigyanKendra","Warehouse","SoilTestingFacility"]}}]}]}}}`) + + values, err := facilityTypesFrom(beckn) + if err != nil { + t.Fatalf("facilityTypesFrom() returned an unexpected error: %v", err) + } + want := []string{"KrishiVigyanKendra", "Warehouse", "SoilTestingFacility"} + if len(values) != len(want) { + t.Fatalf("values = %v, want %v", values, want) + } + for index := range want { + if values[index] != want[index] { + t.Errorf("values[%d] = %v, want %q -- payload order, not any other", index, values[index], want[index]) + } + } +} + +// One type is one value and one call, not a special case. +func TestFacilityTypesFromAcceptsASingleType(t *testing.T) { + t.Parallel() + + beckn := decode(t, `{"message":{"contract":{"commitments":[{"resources":[{"resourceAttributes":{ + "supportedFacilityTypes":["Warehouse"]}}]}]}}}`) + + values, err := facilityTypesFrom(beckn) + if err != nil { + t.Fatalf("facilityTypesFrom() returned an unexpected error: %v", err) + } + if len(values) != 1 || values[0] != "Warehouse" { + t.Errorf("values = %v, want [Warehouse]", values) + } +} + +// A payload that carries the field as a bare string rather than a list is +// accepted as one value. The mapping's own required checks wrap it in [] to +// compare it against the governed list, so a caller that satisfies those +// checks may still have written a scalar here. +func TestFacilityTypesFromAcceptsABareString(t *testing.T) { + t.Parallel() + + beckn := decode(t, `{"message":{"contract":{"commitments":[{"resources":[{"resourceAttributes":{ + "supportedFacilityTypes":"KrishiVigyanKendra"}}]}]}}}`) + + values, err := facilityTypesFrom(beckn) + if err != nil { + t.Fatalf("facilityTypesFrom() returned an unexpected error: %v", err) + } + if len(values) != 1 || values[0] != "KrishiVigyanKendra" { + t.Errorf("values = %v, want [KrishiVigyanKendra]", values) + } +} + +// Every shape that is not a payload this capability can serve is refused with +// a message naming what was missing -- not a nil-map panic, and not an empty +// list that would silently become zero calls. +func TestFacilityTypesFromRefusesWhatItCannotRead(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name, body, wants string + }{ + {"an empty object", `{}`, "commitment"}, + {"no commitments", `{"message":{"contract":{"commitments":[]}}}`, "commitment"}, + {"no resources", `{"message":{"contract":{"commitments":[{"resources":[]}]}}}`, "resource"}, + {"no resourceAttributes", `{"message":{"contract":{"commitments":[{"resources":[{}]}]}}}`, "no facility type"}, + {"an empty list", `{"message":{"contract":{"commitments":[{"resources":[{"resourceAttributes":{ + "supportedFacilityTypes":[]}}]}]}}}`, "no facility type"}, + {"a repeated type", `{"message":{"contract":{"commitments":[{"resources":[{"resourceAttributes":{ + "supportedFacilityTypes":["Warehouse","Warehouse"]}}]}]}}}`, "not repeat"}, + {"a number instead of a type", `{"message":{"contract":{"commitments":[{"resources":[{"resourceAttributes":{ + "supportedFacilityTypes":[42]}}]}]}}}`, "not a facility type"}, + {"a nested list", `{"message":{"contract":{"commitments":[{"resources":[{"resourceAttributes":{ + "supportedFacilityTypes":[["Warehouse"]]}}]}]}}}`, "not a facility type"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + values, err := facilityTypesFrom(decode(t, tc.body)) + if err == nil { + t.Fatalf("facilityTypesFrom() returned %v, want %s refused", values, tc.name) + } + if !strings.Contains(err.Error(), tc.wants) { + t.Errorf("error = %v, want it to mention %q", err, tc.wants) + } + }) + } +} + +// A payload that is not an object at all -- the shape a decoder handed +// something odd would produce -- is refused rather than panicking. +func TestFacilityTypesFromRefusesANonObjectPayload(t *testing.T) { + t.Parallel() + + for _, beckn := range []any{nil, "a string", 42.0, []any{"a", "list"}} { + if _, err := facilityTypesFrom(beckn); err == nil { + t.Errorf("facilityTypesFrom(%v) was accepted, want it refused", beckn) + } + } +} diff --git a/pkg/plugin/implementation/AgricultureFacility/prerequisites.go b/pkg/plugin/implementation/AgricultureFacility/prerequisites.go new file mode 100644 index 00000000..ee307863 --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/prerequisites.go @@ -0,0 +1,17 @@ +package AgricultureFacility + +import "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" + +// prerequisites is what an agriculture facility capability needs that its +// payload does not carry, keyed by binding key. +// +// Empty. A facility search names the point to search from and the facility type +// it wants, and POCRA's search takes exactly those, so there is nothing to +// resolve before the call. The mapping reads both straight off the payload. +// +// An entry would be needed only for real I/O: a district name to resolve to a +// code, a token to exchange, an address to geocode. Each of those is a +// different upstream than the one this was written against, so prefer keeping +// the payload explicit over adding an entry here -- and see the note in +// WeatherObservation/prerequisites.go before doing so. +var prerequisites = common.Prerequisites{} diff --git a/pkg/plugin/implementation/AgricultureFacility/schemacache_test.go b/pkg/plugin/implementation/AgricultureFacility/schemacache_test.go new file mode 100644 index 00000000..d4421487 --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/schemacache_test.go @@ -0,0 +1,309 @@ +package AgricultureFacility_test + +// schemacache_test.go fetches the AgricultureFacility v0.1 schema pack from +// where it is actually published, and caches it under testdata/schema-cache. +// +// This package used to carry the nine schemas the pack reaches as one vendored +// 700-line multi-document YAML file, with a hand-written table naming the URI +// each document had to be registered under. Two problems with that. It was a +// transcription of the pack that could disagree with the pack and only a manual +// refresh would notice. And the base URI the table registered the OAN packs +// under -- https://schemas.openagrinet.global/... -- has no DNS record at all, +// so the one address the tests presented as the schema's identity was an +// address nothing serves. +// +// Now the validator resolves the pack's $refs itself and this loader answers +// them: once over the network, then from disk. So the base URIs are the real, +// dereferenceable ones, +// +// https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/AgricultureFacility/v0.1 +// +// the relative refs the pack writes ("../../AgricultureResource/v0.1/...") +// resolve against them with no table to keep in step, and the beckn.io +// schemas -- which redirect to schema.nfh.global, and which the two sides +// disagree about the naming of -- are simply loaded under whichever name the +// ref that reached them was written with. +// +// The cache is not committed; see testdata/.gitignore. A cold run needs the +// network, and every run after it is offline and deterministic. With an empty +// cache AND no network the schema tests SKIP rather than fail, because they +// cannot be performed at all on that machine -- what they cover that does not +// need the pack is covered offline in mappings_test.go. +// +// To refresh against upstream, empty the cache and run the tests: +// +// rm -rf pkg/plugin/implementation/AgricultureFacility/testdata/schema-cache +// go test ./pkg/plugin/implementation/AgricultureFacility/ + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + neturl "net/url" + "os" + "path" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/santhosh-tekuri/jsonschema/v6" + "gopkg.in/yaml.v3" +) + +// The published pack. Pinned to the schema-packs-v0.1 ref rather than to a +// moving default branch: a suite whose verdict depends on the day it ran is not +// a conformance suite. +// +// This is a BRANCH name, not a commit -- it moves as OpenAgriNet/network-specs +// moves, deliberately, so this suite always tests against what schema-packs-v0.1 +// currently publishes rather than a snapshot that quietly falls behind. +// +// The cost of that: the branch moves under this suite, so a change upstream +// lands as a red build here on the next COLD cache -- CI always is, a +// developer's machine usually is not. Two have happened so far, and they are +// worth reading as the two shapes this takes. +// +// A constraint DROPPED. Commit b76c9ad8a5, titled "docs #5: generate composed +// schema reference pages," silently removed the `not: anyOf: [...]` clause +// that made an OnDemand resource reject facilityType/location/address/ +// services/capacity/publicContact/website/source/lastUpdatedAt. That was the +// constraint this plugin's design leans on (see the package doc and README's +// "Where the query lives"), and nothing here could restore it: the test that +// asserted the forbid now asserts only that the fixture is a valid OnDemand +// resource, and the convention is kept as a choice. See +// dev_docs/schema-onDemand-forbid-removed.md. +// +// A constraint ADDED. AgricultureResource v0.1 later made subjectCategories +// REQUIRED (it was optional), added "Facility" to its enum, and +// AgricultureFacility now requires the list to contain "Facility". That one +// this repo could and did satisfy: the mapping states +// subjectCategories: ["Facility"] on every facility it answers with, and the +// request fixture carries it too. See the mapping's own comment beside the +// field. +// +// The rule for the next one: a red cold-cache run here is a question about +// which of those two it is. If the pack asks for something this adapter can +// state, state it. If the pack dropped something this adapter relies on, that +// is not ours to silence -- say so in the test and in dev_docs. +const ( + packBase = "https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema" + + facilityPackURL = packBase + "/AgricultureFacility/v0.1/attributes.yaml" + resourcePackURL = packBase + "/AgricultureResource/v0.1/attributes.yaml" + + // The pack's own examples, published alongside it. + packExampleBase = packBase + "/AgricultureFacility/v0.1/examples" +) + +// schemaCacheDir mirrors each fetched URL as host/path, so what is on disk says +// where it came from without an index to consult. +const schemaCacheDir = "testdata/schema-cache" + +// fetchTimeout is per document. Generous, because a cold cache pulls a dozen +// of them. +const fetchTimeout = 30 * time.Second + +// maxSchemaSize caps what is read from a response. The largest of these +// documents is a few kilobytes; anything near this is not a schema. +const maxSchemaSize = 4 << 20 + +// packLoader answers the validator's $ref lookups, from the cache when it can +// and from the network when it must. +// +// It records what it loaded so a test can assert the schema was compiled out of +// the pack rather than out of whatever else happened to be cached, and records +// the first fetch failure so an offline machine can be told apart from a broken +// schema. +type packLoader struct { + mu sync.Mutex + loaded map[string]bool + fetchErr error +} + +func newPackLoader() *packLoader { + return &packLoader{loaded: map[string]bool{}} +} + +// Load satisfies jsonschema.URLLoader. +func (l *packLoader) Load(url string) (any, error) { + raw, err := l.text(url) + if err != nil { + return nil, err + } + return decodeSchema(url, raw) +} + +// text returns a document's bytes verbatim, which is what the governed-field +// check needs -- it reads the pack as YAML text rather than as a compiled +// schema. +func (l *packLoader) text(url string) ([]byte, error) { + cached, err := cachePath(url) + if err != nil { + return nil, err + } + if raw, err := os.ReadFile(cached); err == nil { + l.markLoaded(url) + return raw, nil + } + + raw, err := fetchSchema(url) + if err != nil { + l.mu.Lock() + if l.fetchErr == nil { + l.fetchErr = err + } + l.mu.Unlock() + return nil, err + } + if err := writeCache(cached, raw); err != nil { + return nil, err + } + l.markLoaded(url) + return raw, nil +} + +func (l *packLoader) markLoaded(url string) { + l.mu.Lock() + defer l.mu.Unlock() + l.loaded[url] = true +} + +func (l *packLoader) sawLoaded(url string) bool { + l.mu.Lock() + defer l.mu.Unlock() + return l.loaded[url] +} + +// skipIfOffline turns "the pack could not be reached and was not cached" into a +// skip, and leaves every other failure to the caller -- including a reachable +// server that answered with something other than 200, which is not offline, +// it is a finding. +// +// The distinction is the point. A schema that no longer compiles is a finding; a +// laptop on a train is not. A 404 for a pack that moved or was deleted is a +// finding too, not a train. +func (l *packLoader) skipIfOffline(t *testing.T, cause error) { + t.Helper() + + l.mu.Lock() + fetchErr := l.fetchErr + l.mu.Unlock() + if fetchErr == nil { + return + } + var status httpStatusError + if errors.As(fetchErr, &status) { + return + } + t.Skipf("the schema pack is neither cached under %s nor reachable, so conformance "+ + "cannot be checked here -- run once with network access to prime the cache.\n"+ + "fetch: %v\nfailure: %v", schemaCacheDir, fetchErr, cause) +} + +// decodeSchema converts one fetched document into the shape the validator +// takes. +// +// The packs are published as YAML and the beckn examples as JSON. YAML parses +// both, since JSON is a subset of it, and the round trip through JSON is a +// format conversion and nothing more: no key is renamed and no value is +// touched. It is there so numbers and keys arrive as the validator's own types +// rather than as yaml's. +func decodeSchema(url string, raw []byte) (any, error) { + var parsed any + if err := yaml.Unmarshal(raw, &parsed); err != nil { + return nil, fmt.Errorf("%s is not YAML or JSON: %w", url, err) + } + asJSON, err := json.Marshal(parsed) + if err != nil { + return nil, fmt.Errorf("%s could not be converted to JSON: %w", url, err) + } + return jsonschema.UnmarshalJSON(strings.NewReader(string(asJSON))) +} + +// cachePath maps a URL to its place in the cache, and refuses anything that +// would write outside it. +func cachePath(rawURL string) (string, error) { + parsed, err := neturl.Parse(rawURL) + if err != nil { + return "", fmt.Errorf("could not parse %s: %w", rawURL, err) + } + if parsed.Scheme != "https" { + return "", fmt.Errorf("refusing to load %s: the pack is fetched over https only", rawURL) + } + if parsed.Host == "" { + return "", fmt.Errorf("refusing to load %s: no host", rawURL) + } + clean := path.Clean("/" + parsed.Path) + if strings.Contains(clean, "..") || strings.ContainsAny(parsed.Host, `/\`) { + return "", fmt.Errorf("refusing to cache %s: the path escapes %s", rawURL, schemaCacheDir) + } + return filepath.Join(schemaCacheDir, parsed.Host, filepath.FromSlash(clean)), nil +} + +// httpStatusError marks a fetch that reached the server and got back +// something other than 200 -- a reachable pack that 404s or 500s, not an +// unreachable one. skipIfOffline uses the type to tell the two apart. +type httpStatusError struct { + url string + status string +} + +func (e httpStatusError) Error() string { + return fmt.Sprintf("GET %s: %s", e.url, e.status) +} + +// fetchSchema retrieves one document. Redirects are followed -- schema.beckn.io +// 301s to schema.nfh.global -- but the URL the ref was written with is what the +// document is cached and registered under, which is what makes the refs +// resolve. +func fetchSchema(url string) ([]byte, error) { + client := &http.Client{Timeout: fetchTimeout} + response, err := client.Get(url) + if err != nil { + return nil, fmt.Errorf("GET %s: %w", url, err) + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + return nil, httpStatusError{url: url, status: response.Status} + } + raw, err := io.ReadAll(io.LimitReader(response.Body, maxSchemaSize+1)) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", url, err) + } + if len(raw) > maxSchemaSize { + return nil, fmt.Errorf("%s is larger than %d bytes, which no schema in this pack is", + url, maxSchemaSize) + } + return raw, nil +} + +// writeCache writes through a temporary file in the same directory, so an +// interrupted run leaves no half-written document behind for the next one to +// parse. +func writeCache(cached string, raw []byte) error { + if err := os.MkdirAll(filepath.Dir(cached), 0o755); err != nil { + return fmt.Errorf("could not create %s: %w", filepath.Dir(cached), err) + } + temp, err := os.CreateTemp(filepath.Dir(cached), "."+filepath.Base(cached)+".*") + if err != nil { + return fmt.Errorf("could not stage %s: %w", cached, err) + } + defer os.Remove(temp.Name()) + + if _, err := temp.Write(raw); err != nil { + temp.Close() + return fmt.Errorf("could not write %s: %w", temp.Name(), err) + } + if err := temp.Close(); err != nil { + return fmt.Errorf("could not close %s: %w", temp.Name(), err) + } + if err := os.Rename(temp.Name(), cached); err != nil { + return fmt.Errorf("could not move %s into place: %w", cached, err) + } + return nil +} diff --git a/pkg/plugin/implementation/AgricultureFacility/search.go b/pkg/plugin/implementation/AgricultureFacility/search.go new file mode 100644 index 00000000..ddcd9b41 --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/search.go @@ -0,0 +1,398 @@ +// search.go is the whole of what a multi-type facility search means, and it +// lives here because every part of it is a fact about POCRA. +// +// POCRA's search takes exactly ONE category code: a comma-separated pair +// answers 200 with no providers at all, and a category array is refused +// outright, both verified against the live API. So a payload asking for three +// facility types cannot be served by one call, however the mapping is written. +// +// What this file does about that: read the types out of the payload, split one +// inbound payload into one single-type payload per type, run the ordinary +// one-payload-one-call step over each of them concurrently, and merge the +// answers back into one. +// +// Nothing below this file knows any of that. internal/upstream serves one +// payload with one call and has no notion of splitting; jsonmapper compiles +// the two halves every mapping has and no third thing; internal/concurrent +// runs N of anything, bounded and ordered, and has never heard of Beckn. The +// mapping this runs is written for a single-type payload, which is what it is +// always handed. +// +// Separated from the package clause by a blank line on purpose: the package's +// own doc comment is in AgricultureFacility.go, and this is a note about one +// file. + +package AgricultureFacility + +import ( + "context" + "encoding/json" + "fmt" + "slices" + + "github.com/google/uuid" + + "github.com/beckn-one/beckn-onix/pkg/log" + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/definition" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/concurrent" +) + +const ( + // MaxFacilityTypes bounds how many upstream calls one inbound payload may + // become. + // + // Splitting is amplification: one request in, N out, each with its own + // retry budget. The count comes from the PAYLOAD, so without a ceiling a + // caller decides how much work this adapter and the provider do -- and the + // provider is a government API that takes eight seconds to answer. + // + // Refused rather than clamped, unlike the registry's budgets. A clamped + // timeout still answers the question asked; a clamped search silently + // drops facility types from the answer, which is the exact defect this + // facility exists to fix. + MaxFacilityTypes = 8 + + // DefaultSearchConcurrency is how many of a split search's calls are in + // flight at once when a deployment does not say. + // + // One -- sequential. POCRA's aggregator waits a fixed window for its BPPs + // to reply and returns whatever arrived, so a slow BPP that misses the + // window is reported as no results rather than as an error. Concurrency + // makes that more likely, and its failure mode is silent data loss -- the + // exact defect this facility exists to fix. + // + // A provider that tolerates concurrency can say so per deployment, which + // is a config change rather than a code one. The cost of the safe default + // is latency: N calls take N times as long, and the operator can see that + // in the log line each call writes. + DefaultSearchConcurrency = 1 +) + +// Step serves the agriculture facility capability by splitting a multi-type +// search across the ordinary one-call step and merging what comes back. +// +// A step rather than a hook into internal/upstream: what a payload splits +// across, how many calls that is, what is too many and how the answers +// recombine are all facts about POCRA, so they belong to a thing that knows +// POCRA. internal/upstream stays a step that serves one payload with one +// call, which is what every other capability needs of it. +type Step struct { + // inner serves one payload with one call: the ordinary upstream step, + // built by this package's New and never handed a payload naming more than + // one facility type. + inner definition.Step + + // paths and bindingKeys answer "is this payload mine?", the same question + // and the same way inner answers it. Asked here first because a payload + // that is NOT this capability's must reach inner untouched -- it passes + // through, as it does for every capability it does not serve -- while one + // that IS must be split before inner sees it. + paths common.Paths + bindingKeys []string + + // concurrency is how many of a split search's calls run at once. + concurrency int +} + +// Run serves one inbound payload. +// +// A payload naming one facility type is one call, which is the common case. A +// payload naming several becomes one call per type, run concurrently up to +// this step's limit, and their answers are merged into one. A payload that is +// not this capability's is handed to inner unchanged, which passes it through. +func (s *Step) Run(ctx *model.StepContext) error { + if !s.mine(ctx.Body) { + return s.inner.Run(ctx) + } + + var beckn any + if err := json.Unmarshal(ctx.Body, &beckn); err != nil { + return model.NewBadReqErr("", fmt.Errorf( + "agriculture facility: the payload is not JSON: %w", err)) + } + + types, err := facilityTypesFrom(beckn) + if err != nil { + return err + } + if len(types) > MaxFacilityTypes { + return model.NewBadReqErr("", fmt.Errorf( + "agriculture facility: this payload asks for %d facility types and the ceiling is %d; "+ + "split it across more than one request", len(types), MaxFacilityTypes)) + } + + // Every type gets its own payload, even when there is only one of them: + // the fresh message id each part carries is what stops POCRA blending the + // answers, and a search of one still wants its own id rather than the + // caller's. See splitByType. + parts, err := splitByType(ctx.Body, types) + if err != nil { + return err + } + + log.Debugf(ctx, "agriculture facility: serving %d facility type(s) as %d call(s), %d at a time", + len(types), len(parts), s.concurrency) + + // Bounded, ordered and fail-fast, which is internal/concurrent's whole + // job. One failure fails the request: a partial answer is the defect this + // was built to fix wearing a different hat -- the caller asked for four + // facility types, would receive three, and nothing in the payload would + // say that the fourth was asked for and lost. + answers, err := concurrent.Map(ctx, parts, s.concurrency, + func(callCtx context.Context, part []byte) ([]byte, error) { + // Its own StepContext: inner reads Body and writes ResponseBody, + // so the parts must not share either. callCtx rather than ctx so a + // sibling's failure cancels this call too. + partCtx := *ctx + partCtx.Context = callCtx + partCtx.Body = part + partCtx.ResponseBody = nil + + if err := s.runPart(&partCtx); err != nil { + return nil, err + } + return partCtx.ResponseBody, nil + }) + if err != nil { + return err + } + + merged, err := mergeAnswers(answers, messageIDOf(beckn)) + if err != nil { + return err + } + ctx.ResponseBody = merged + return nil +} + +// runPart runs the one-call step over one part and insists it answered. +// +// An empty ResponseBody means inner passed the payload through rather than +// serving it, which for a part built from a payload this step already +// recognised can only be a mismatch between this step's binding keys and +// inner's. Reported rather than merged, because merging nothing would answer +// a four-type search with three types and no error. +func (s *Step) runPart(partCtx *model.StepContext) error { + if err := s.inner.Run(partCtx); err != nil { + return err + } + if len(partCtx.ResponseBody) == 0 { + return fmt.Errorf( + "agriculture facility: the upstream step served no answer for one part of this search, " + + "which means it does not serve this capability -- check that both are configured " + + "with the same binding keys") + } + return nil +} + +// mine reports whether this payload names one of the capabilities this step +// serves. A payload it cannot read is not refused here: inner refuses it, with +// the message it has always used. +func (s *Step) mine(body []byte) bool { + binding, err := common.BindingFrom(s.paths, body) + if err != nil { + return false + } + return slices.Contains(s.bindingKeys, binding.Key()) +} + +// splitByType turns one payload into one payload per facility type. +// +// Each part names exactly one type, so the mapping's request half reads a +// single value rather than choosing among several -- choosing the first is +// what made a multi-type search answer with one type and silently drop the +// rest. +// +// Each part also carries a FRESH context.messageId. POCRA keeps a message_id's +// answers for ten minutes and returns the union of everything asked for under +// it, so two parts sharing an id would each come back carrying the other's +// facilities. The mapping copies this straight through to POCRA's message_id, +// whose schema refuses anything that is not a UUID. +// +// Re-decoded per part rather than deep-copied: the parts are mutated +// independently and a shared nested map would have them overwrite each other's +// type. A payload is a few kilobytes and this happens once per request. +func splitByType(body []byte, types []string) ([][]byte, error) { + parts := make([][]byte, 0, len(types)) + for _, facilityType := range types { + var part map[string]any + if err := json.Unmarshal(body, &part); err != nil { + return nil, model.NewBadReqErr("", fmt.Errorf( + "agriculture facility: the payload is not a JSON object: %w", err)) + } + + attributes, ok := dig(part, "message", "contract", "commitments").([]any) + if !ok || len(attributes) == 0 { + return nil, model.NewBadReqErr("", fmt.Errorf( + "agriculture facility: the payload carries no commitment to split")) + } + commitment, ok := attributes[0].(map[string]any) + if !ok { + return nil, model.NewBadReqErr("", fmt.Errorf( + "agriculture facility: the payload's first commitment is %T, not an object", attributes[0])) + } + resources, ok := commitment["resources"].([]any) + if !ok || len(resources) == 0 { + return nil, model.NewBadReqErr("", fmt.Errorf( + "agriculture facility: the payload's commitment carries no resource to split")) + } + resource, ok := resources[0].(map[string]any) + if !ok { + return nil, model.NewBadReqErr("", fmt.Errorf( + "agriculture facility: the payload's first resource is %T, not an object", resources[0])) + } + resourceAttributes, ok := resource["resourceAttributes"].(map[string]any) + if !ok { + return nil, model.NewBadReqErr("", fmt.Errorf( + "agriculture facility: the payload's first resource carries no resourceAttributes")) + } + + // A list of one, not a bare string: the required: checks and the + // request half both read this through [] and a scalar would work, but + // a part that does not look like the payload it came from is a trap + // for whoever reads one in a log. + resourceAttributes["supportedFacilityTypes"] = []any{facilityType} + + becknContext, ok := part["context"].(map[string]any) + if !ok { + return nil, model.NewBadReqErr("", fmt.Errorf( + "agriculture facility: the payload carries no context to stamp a message id on")) + } + becknContext["messageId"] = uuid.NewString() + + encoded, err := json.Marshal(part) + if err != nil { + return nil, fmt.Errorf("agriculture facility: could not rebuild the payload for %s: %w", + facilityType, err) + } + parts = append(parts, encoded) + } + return parts, nil +} + +// mergeAnswers combines one answer per facility type into the single answer the +// caller gets. +// +// The first answer is the shape of the result -- context, status and offer are +// the same in all of them, because all the parts came from one payload. What +// differs is the resources, which are concatenated in the order the payload +// named the types, deduplicated by id, and pointed at by the offer. +// +// The caller's own messageId is restored: each part carried a generated one so +// POCRA would not blend them, and the answer has to correlate with the request +// the caller actually sent. +// +// ORDERING, worth knowing: within one facility type the mapping ranks by +// POCRA's distance, and that ranking survives here. ACROSS types the result is +// type-blocked -- every KrishiVigyanKendra, then every Warehouse -- rather +// than globally nearest-first, because distance is not an intrinsic facility +// attribute and the mapping drops it before this sees the answers. A globally +// ranked multi-type search would need the mapping to publish a distance this +// could sort on, which the schema pack says it must not. +func mergeAnswers(answers [][]byte, callerMessageID string) ([]byte, error) { + if len(answers) == 0 { + return nil, fmt.Errorf("agriculture facility: nothing to merge; this search made no calls") + } + + var merged map[string]any + if err := json.Unmarshal(answers[0], &merged); err != nil { + return nil, fmt.Errorf("agriculture facility: the answer for the first facility type is not JSON: %w", err) + } + commitment, err := commitmentOf(merged) + if err != nil { + return nil, err + } + + resources := make([]any, 0) + seen := map[string]bool{} + for index, answer := range answers { + var document map[string]any + if err := json.Unmarshal(answer, &document); err != nil { + return nil, fmt.Errorf( + "agriculture facility: the answer for facility type %d is not JSON: %w", index, err) + } + part, err := commitmentOf(document) + if err != nil { + return nil, err + } + found, _ := part["resources"].([]any) + for _, entry := range found { + resource, ok := entry.(map[string]any) + if !ok { + continue + } + id, _ := resource["id"].(string) + // Deduplicated by id because a repeated id in the answer is a + // broken reference the moment anything resolves it. The mapping + // already collapses POCRA's own repeats within one call; this + // catches a facility that answered for two types. + if id != "" && seen[id] { + continue + } + seen[id] = true + resources = append(resources, resource) + } + } + commitment["resources"] = resources + + // The offer points at what the answer actually carries. Leaving the first + // part's resourceIds would name only its own type's facilities. + if offer, ok := commitment["offer"].(map[string]any); ok { + ids := make([]any, 0, len(resources)) + for _, entry := range resources { + if resource, ok := entry.(map[string]any); ok { + ids = append(ids, resource["id"]) + } + } + offer["resourceIds"] = ids + } + + if becknContext, ok := merged["context"].(map[string]any); ok && callerMessageID != "" { + becknContext["messageId"] = callerMessageID + } + + encoded, err := json.Marshal(merged) + if err != nil { + return nil, fmt.Errorf("agriculture facility: could not encode the merged answer: %w", err) + } + return encoded, nil +} + +// commitmentOf reaches the one commitment an answer carries. +func commitmentOf(document map[string]any) (map[string]any, error) { + commitments, _ := dig(document, "message", "contract", "commitments").([]any) + if len(commitments) == 0 { + return nil, fmt.Errorf("agriculture facility: an answer carries no commitment to merge") + } + commitment, ok := commitments[0].(map[string]any) + if !ok { + return nil, fmt.Errorf("agriculture facility: an answer's commitment is %T, not an object", commitments[0]) + } + return commitment, nil +} + +// messageIDOf reads the caller's own message id, for the merged answer to +// correlate with. Absent is not an error: the mapping copies whatever the +// payload carried, so an absent one stays absent, as it did before splitting +// existed. +func messageIDOf(beckn any) string { + document, ok := beckn.(map[string]any) + if !ok { + return "" + } + id, _ := dig(document, "context", "messageId").(string) + return id +} + +// searchConcurrency resolves the deployment's setting, clamped to +// MaxFacilityTypes and defaulting to DefaultSearchConcurrency when left unset +// or non-positive. +// +// The arithmetic is internal/concurrent's, because every caller of it needs +// the same clamp and a zero limit there means UNBOUNDED. The three numbers are +// this package's, because each one is a fact about POCRA. +func searchConcurrency(configured int) int { + return concurrent.Bound(configured, DefaultSearchConcurrency, MaxFacilityTypes) +} diff --git a/pkg/plugin/implementation/AgricultureFacility/search_test.go b/pkg/plugin/implementation/AgricultureFacility/search_test.go new file mode 100644 index 00000000..33bcb9be --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/search_test.go @@ -0,0 +1,249 @@ +package AgricultureFacility_test + +// search_test.go covers the POLICY this package owns: how many of a +// multi-type search's calls run at once, in what order, and what happens when +// one fails. mappings_test.go already covers that splitting happens at all +// and produces the right facilities (TestATwoTypeSearchIsAnsweredWithBothTypes +// and neighbours) and that each call carries its own identity +// (TestEachSearchCallCarriesItsOwnRequestId). + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/AgricultureFacility" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/util" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/jsonmapper" +) + +// runSearchTimed is runSplitSearch's sibling: it hands the caller a raw HTTP +// handler (so a test can measure timing or count in-flight calls) and an +// explicit Config (so a test can set SearchConcurrency), rather than a canned +// per-category response map. +func runSearchTimed(t *testing.T, types []string, cfg *AgricultureFacility.Config, handler http.HandlerFunc) error { + t.Helper() + + mappings := serveMappings(t) + defer mappings.Close() + + upstream := httptest.NewServer(handler) + defer upstream.Close() + + mapper, closeMapper, err := jsonmapper.New(context.Background(), &jsonmapper.Config{}) + if err != nil { + t.Fatalf("failed to build the mapper: %v", err) + } + defer closeMapper() + + registry := &stubRegistry{plan: &model.ProviderRecord{ + BindingKey: shippedBindingKey, BaseURL: upstream.URL, + Actions: map[string]model.ActionPlan{ + "select": {Method: http.MethodPost, Path: "/search", + Mappings: mappings.URL + "/" + shippedMapping, TimeoutMs: 30000}, + }, + }} + + cfg.BindingKeys = []string{shippedBindingKey} + // Auth is per provider, and a served provider without a block is refused at + // startup. POCRA needs no credential and says so. + if cfg.AuthByProvider == nil { + cfg.AuthByProvider = map[string]*common.AuthProfile{ + strings.Split(shippedBindingKey, "|")[0]: {Scheme: util.AuthSchemeNone}, + } + } + step, closeStep, err := AgricultureFacility.New(context.Background(), registry, mapper, cfg) + if err != nil { + t.Fatalf("failed to build the step: %v", err) + } + defer closeStep() + + var payload map[string]any + if err := json.Unmarshal([]byte(selectRequest), &payload); err != nil { + t.Fatalf("the fixture is not JSON: %v", err) + } + attributes(t, payload)["supportedFacilityTypes"] = toAny(types) + body, err := json.Marshal(payload) + if err != nil { + t.Fatalf("could not rebuild the payload: %v", err) + } + + return step.Run(&model.StepContext{Context: t.Context(), Body: body}) +} + +// A search's calls are sequential unless the deployment raised the limit. +// +// The default is not a performance choice. A provider that answers one +// question at a time is often not built to be asked several at once, and +// POCRA's failure mode when pushed is a 200 with an empty catalog -- +// indistinguishable from having no results, so the loss is silent. Parallel +// is opt-in per deployment for that reason, and this pins the default so it +// cannot drift. +func TestSearchCallsAreSequentialByDefault(t *testing.T) { + t.Parallel() + + var mu sync.Mutex + var inFlight, peak int + handler := func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + inFlight++ + if inFlight > peak { + peak = inFlight + } + mu.Unlock() + + // Long enough that overlapping calls would be caught. Without a wait + // each call could finish before the next begins and a parallel step + // would look sequential. + time.Sleep(40 * time.Millisecond) + + mu.Lock() + inFlight-- + mu.Unlock() + fmt.Fprint(w, providerResponse) + } + + types := []string{"KrishiVigyanKendra", "CustomHiringCentre", "Warehouse"} + if err := runSearchTimed(t, types, &AgricultureFacility.Config{}, handler); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + + mu.Lock() + defer mu.Unlock() + if peak != 1 { + t.Errorf("%d calls were in flight at once, want 1 by default", peak) + } +} + +// A deployment that raises the limit gets calls in parallel, bounded by it. +func TestSearchHonoursTheConfiguredConcurrency(t *testing.T) { + t.Parallel() + + var mu sync.Mutex + var inFlight, peak int + handler := func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + inFlight++ + if inFlight > peak { + peak = inFlight + } + mu.Unlock() + time.Sleep(60 * time.Millisecond) + mu.Lock() + inFlight-- + mu.Unlock() + fmt.Fprint(w, providerResponse) + } + + types := []string{"KrishiVigyanKendra", "CustomHiringCentre", "Warehouse", "SoilTestingFacility"} + cfg := &AgricultureFacility.Config{SearchConcurrency: 2} + if err := runSearchTimed(t, types, cfg, handler); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + + mu.Lock() + defer mu.Unlock() + if peak != 2 { + t.Errorf("peak concurrency was %d, want the configured 2", peak) + } +} + +// A call not yet issued when an earlier one fails is skipped, not made and +// discarded. Sequential (the default) makes this deterministic: the first +// type's failure cancels the shared context before the second type's call is +// ever issued. +func TestSearchSkipsCallsNotYetIssuedAfterAFailure(t *testing.T) { + t.Parallel() + + var mu sync.Mutex + var calls int + handler := func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + calls++ + mu.Unlock() + http.Error(w, "no", http.StatusInternalServerError) + } + + types := []string{"KrishiVigyanKendra", "CustomHiringCentre", "Warehouse"} + err := runSearchTimed(t, types, &AgricultureFacility.Config{}, handler) + if err == nil { + t.Fatal("Run() served a partial answer, want it refused") + } + + mu.Lock() + defer mu.Unlock() + // Sequential (limit 1): the first type fails and cancels the shared + // context before the loop ever reaches the second or third type's call. + if calls != 1 { + t.Errorf("the provider was called %d times, want 1 -- later facility types "+ + "must be skipped once an earlier one fails", calls) + } +} + +// A payload asking for more calls than the ceiling is refused, not clamped. +// +// Uses a repeated governed type to exceed MaxFacilityTypes, which the +// mapping's own duplicate-type required check ALSO refuses -- both guards +// agree the payload is bad, so the assertion that matters is that the provider +// was never called, not which guard fired first. +func TestSearchRefusesOverTheCeiling(t *testing.T) { + t.Parallel() + + handler := func(w http.ResponseWriter, r *http.Request) { + t.Error("the provider was called for a search that should have been refused") + } + + types := make([]string, AgricultureFacility.MaxFacilityTypes+1) + for i := range types { + types[i] = "KrishiVigyanKendra" + } + err := runSearchTimed(t, types, &AgricultureFacility.Config{}, handler) + if err == nil { + t.Fatal("a search over the ceiling was served, want it refused") + } +} + +// A payload naming one type is one call. +func TestSearchOfOneTypeIsOneCall(t *testing.T) { + t.Parallel() + + var calls int32 + handler := func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&calls, 1) + fmt.Fprint(w, providerResponse) + } + + if err := runSearchTimed(t, []string{"KrishiVigyanKendra"}, &AgricultureFacility.Config{}, handler); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + if got := atomic.LoadInt32(&calls); got != 1 { + t.Errorf("the provider saw %d calls, want 1 for a one-type search", got) + } +} + +// A payload naming no types at all is refused before the provider is called. +// +// The mapping's own required checks refuse this first, with a better message. +// This asserts the Go read refuses it too, because a payload with nothing to +// split across would otherwise become zero calls and an empty answer -- the +// silent-loss failure this whole facility exists to prevent. +func TestSearchRefusesAPayloadNamingNoTypes(t *testing.T) { + t.Parallel() + + handler := func(w http.ResponseWriter, r *http.Request) { + t.Error("the provider was called for a payload naming no facility types") + } + + if err := runSearchTimed(t, []string{}, &AgricultureFacility.Config{}, handler); err == nil { + t.Fatal("a payload naming no facility types was served, want it refused") + } +} diff --git a/pkg/plugin/implementation/AgricultureFacility/testdata/.gitignore b/pkg/plugin/implementation/AgricultureFacility/testdata/.gitignore new file mode 100644 index 00000000..e216875f --- /dev/null +++ b/pkg/plugin/implementation/AgricultureFacility/testdata/.gitignore @@ -0,0 +1,4 @@ +# The schema pack is fetched from where it is published and cached here, rather +# than vendored into the repo -- see schemacache_test.go. The cache is a local +# build artifact: delete it to refresh against upstream. +schema-cache/ diff --git a/pkg/plugin/implementation/internal/common/binding.go b/pkg/plugin/implementation/internal/common/binding.go index 7f0759ce..8d657d32 100644 --- a/pkg/plugin/implementation/internal/common/binding.go +++ b/pkg/plugin/implementation/internal/common/binding.go @@ -31,16 +31,19 @@ func (b Binding) Key() string { return b.ParticipantID + separator + b.CapabilityCode } -// bindingFrom derives the capability binding a payload is asking for. +// BindingFrom derives the capability binding a payload is asking for. // // Returns errNoBinding when the payload names no provider or type -- the // ordinary case for a request this step is not meant to serve. // +// EXPORTED for a domain package that must answer "is this payload mine?" the +// same way serve does, rather than reading the payload a second way. +// // More than one distinct provider or type is refused, not resolved to the // first: the two halves index ONE registry row for ONE call, so a payload // spanning several is asking for something this design cannot express, and // guessing would silently serve part of it. -func bindingFrom(paths Paths, body []byte) (Binding, error) { +func BindingFrom(paths Paths, body []byte) (Binding, error) { var payload any if err := json.Unmarshal(body, &payload); err != nil { return Binding{}, fmt.Errorf("payload could not be read: %w", err) diff --git a/pkg/plugin/implementation/internal/common/binding_test.go b/pkg/plugin/implementation/internal/common/binding_test.go index a7507f2a..d0195221 100644 --- a/pkg/plugin/implementation/internal/common/binding_test.go +++ b/pkg/plugin/implementation/internal/common/binding_test.go @@ -43,9 +43,9 @@ const realSelectPayload = `{ func TestFromReadsARealSelectPayload(t *testing.T) { t.Parallel() - got, err := bindingFrom(BecknV2, []byte(realSelectPayload)) + got, err := BindingFrom(BecknV2, []byte(realSelectPayload)) if err != nil { - t.Fatalf("bindingFrom() returned an unexpected error: %v", err) + t.Fatalf("BindingFrom() returned an unexpected error: %v", err) } if got.ParticipantID != "mausamgram" { t.Errorf("participant = %q, want mausamgram", got.ParticipantID) @@ -80,7 +80,7 @@ func TestFromReportsAPayloadWithNoBinding(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - if _, err := bindingFrom(BecknV2, []byte(tc.body)); !errors.Is(err, errNoBinding) { + if _, err := BindingFrom(BecknV2, []byte(tc.body)); !errors.Is(err, errNoBinding) { t.Errorf("expected errNoBinding, got %v", err) } }) @@ -119,7 +119,7 @@ func TestFromRefusesAnAmbiguousPayload(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - _, err := bindingFrom(BecknV2, []byte(tc.body)) + _, err := BindingFrom(BecknV2, []byte(tc.body)) if err == nil { t.Fatal("expected an ambiguous payload to be refused") } @@ -136,7 +136,7 @@ func TestFromRefusesAnAmbiguousPayload(t *testing.T) { func TestFromReportsAnUnreadablePayload(t *testing.T) { t.Parallel() - _, err := bindingFrom(BecknV2, []byte(`{"message":`)) + _, err := BindingFrom(BecknV2, []byte(`{"message":`)) if err == nil { t.Fatal("expected unreadable JSON to be reported") } @@ -161,9 +161,9 @@ func TestBecknV2IsTheDefault(t *testing.T) { if BecknV2.ProviderID == "" || BecknV2.CapabilityCode == "" { t.Fatal("the default paths must be set") } - got, err := bindingFrom(BecknV2, []byte(realSelectPayload)) + got, err := BindingFrom(BecknV2, []byte(realSelectPayload)) if err != nil { - t.Fatalf("bindingFrom() returned an unexpected error: %v", err) + t.Fatalf("BindingFrom() returned an unexpected error: %v", err) } if got.ParticipantID != "mausamgram" || got.CapabilityCode != "openagrinet:WeatherObservation" { t.Errorf("binding = %+v, want the Beckn v2 convention's answer", got) @@ -176,12 +176,12 @@ func TestFromReadsAnOverriddenPath(t *testing.T) { t.Parallel() body := `{"who":{"provider":"agmarknet"},"what":[{"type":"openagrinet:MandiPrice"}]}` - got, err := bindingFrom(Paths{ + got, err := BindingFrom(Paths{ ProviderID: "who.provider", CapabilityCode: "what[].type", }, []byte(body)) if err != nil { - t.Fatalf("bindingFrom() returned an unexpected error: %v", err) + t.Fatalf("BindingFrom() returned an unexpected error: %v", err) } if got.Key() != "agmarknet|openagrinet:MandiPrice" { t.Errorf("binding key = %q, want it read from the overridden paths", got.Key()) @@ -193,7 +193,7 @@ func TestFromReadsAnOverriddenPath(t *testing.T) { func TestFromReportsNoBindingWhenAPathMatchesNothing(t *testing.T) { t.Parallel() - _, err := bindingFrom(Paths{ProviderID: "nowhere.at.all", CapabilityCode: "what[].type"}, + _, err := BindingFrom(Paths{ProviderID: "nowhere.at.all", CapabilityCode: "what[].type"}, []byte(`{"what":[{"type":"x"}]}`)) if !errors.Is(err, errNoBinding) { t.Errorf("expected errNoBinding, got %v", err) @@ -205,7 +205,7 @@ func TestFromReportsNoBindingWhenAPathMatchesNothing(t *testing.T) { func TestFromStillRefusesSeveralValuesUnderAnOverride(t *testing.T) { t.Parallel() - _, err := bindingFrom(Paths{ProviderID: "who[].provider", CapabilityCode: "what[].type"}, + _, err := BindingFrom(Paths{ProviderID: "who[].provider", CapabilityCode: "what[].type"}, []byte(`{"who":[{"provider":"a"},{"provider":"b"}],"what":[{"type":"x"}]}`)) if err == nil || errors.Is(err, errNoBinding) { t.Errorf("expected a refusal naming both providers, got %v", err) @@ -303,7 +303,7 @@ func TestFromRefusesSeveralCommitments(t *testing.T) { "resources":[{"resourceAttributes":{"@type":"openagrinet:WeatherObservation"}}]} ]}}}` - _, err := bindingFrom(BecknV2, []byte(body)) + _, err := BindingFrom(BecknV2, []byte(body)) if err == nil { t.Fatal("expected two commitments to be refused rather than halved") } @@ -319,7 +319,7 @@ func TestFromRefusesSeveralCommitments(t *testing.T) { {"offer":{"provider":{"id":"mausamgram"}}, "resources":[{"resourceAttributes":{"@type":"openagrinet:WeatherObservation"}}]} ]}}}` - binding, err := bindingFrom(BecknV2, []byte(single)) + binding, err := BindingFrom(BecknV2, []byte(single)) if err != nil { t.Fatalf("one commitment must still resolve: %v", err) } @@ -370,7 +370,7 @@ func TestFromRefusesSeveralCommitmentsEvenWhenOneDoesNotResolve(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - _, err := bindingFrom(BecknV2, []byte(tt.payload)) + _, err := BindingFrom(BecknV2, []byte(tt.payload)) if err == nil { t.Fatal("two commitments must be refused, not half answered") } @@ -394,7 +394,7 @@ func TestFromStillPassesThroughASingleUnresolvableCommitment(t *testing.T) { {"offer":{"provider":{}}, "resources":[{"resourceAttributes":{"@type":"openagrinet:WeatherObservation"}}]} ]}}}` - _, err := bindingFrom(BecknV2, []byte(payload)) + _, err := BindingFrom(BecknV2, []byte(payload)) if !errors.Is(err, errNoBinding) { t.Errorf("err = %v, want errNoBinding so the payload passes through", err) } diff --git a/pkg/plugin/implementation/internal/common/common.go b/pkg/plugin/implementation/internal/common/common.go index bbf7ff98..ccfcd565 100644 --- a/pkg/plugin/implementation/internal/common/common.go +++ b/pkg/plugin/implementation/internal/common/common.go @@ -106,7 +106,7 @@ func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper d return nil, nil, err } - paths, err := bindingPaths(cfg) + paths, err := BindingPaths(cfg) if err != nil { return nil, nil, err } @@ -135,11 +135,16 @@ func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper d return step, closer, nil } -// bindingPaths resolves where this step reads a binding key from. +// BindingPaths resolves where this step reads a binding key from. // // Both halves or neither: one overridden and one defaulted would match nothing, // silently, on every request. -func bindingPaths(cfg *Config) (Paths, error) { +// +// EXPORTED for a domain package that has to answer "is this payload mine?" the +// same way this step does -- AgricultureFacility, whose search.go fans one +// request out over several capability types. The alternative is reading the +// same two config fields a second time, which is how the two drift apart. +func BindingPaths(cfg *Config) (Paths, error) { if cfg.ProviderIDAt == "" && cfg.CapabilityCodeAt == "" { return BecknV2, nil } diff --git a/pkg/plugin/implementation/internal/common/serve.go b/pkg/plugin/implementation/internal/common/serve.go index c902fe1f..6d34eca7 100644 --- a/pkg/plugin/implementation/internal/common/serve.go +++ b/pkg/plugin/implementation/internal/common/serve.go @@ -22,12 +22,12 @@ import ( // recognises its own work, so adding a provider is one config entry rather than // a routing-table change. func (s *Step) Run(ctx *model.StepContext) error { - binding, err := bindingFrom(s.paths, ctx.Body) + binding, err := BindingFrom(s.paths, ctx.Body) if errors.Is(err, errNoBinding) { return nil } if err != nil { - // Everything bindingFrom refuses is about the payload -- unreadable + // Everything BindingFrom refuses is about the payload -- unreadable // JSON, or more than one call named. Unclassified it becomes a 500, // which blames this adapter and hides the reason from the caller. return model.NewBadReqErr("", err) diff --git a/pkg/plugin/implementation/internal/concurrent/concurrent.go b/pkg/plugin/implementation/internal/concurrent/concurrent.go new file mode 100644 index 00000000..7bd9b231 --- /dev/null +++ b/pkg/plugin/implementation/internal/concurrent/concurrent.go @@ -0,0 +1,116 @@ +// Package concurrent is one generic engine: run a known number of +// independent calls, bounded, with the first failure cancelling the rest. +// +// It knows nothing about Beckn, upstream providers, or any specific +// capability. Every policy question -- how many is too many, what identity +// each call carries, what a failure should say -- belongs to whoever calls +// this, not to this package: see +// pkg/plugin/implementation/AgricultureFacility/search.go for the one caller +// that exists today. +// +// pkg/plugin/implementation/catalogpublisher's own runConcurrent looks +// similar but is NOT this: it collects every call's error rather than +// cancelling on the first one, because its caller needs to inspect all of +// them to decide whether any is worth retrying. That is a different +// concurrency shape, kept separate rather than forced into one function +// with a mode flag. +package concurrent + +import ( + "context" + "fmt" + + "golang.org/x/sync/errgroup" +) + +// Run calls work(ctx, i) for every i in [0, n), at most limit at a time, and +// returns each call's result at its own index -- in the order Run was asked +// for them, not the order they finished, so a caller can rely on results[i] +// meaning "the i-th thing I asked for" regardless of which one answered +// first. +// +// The first error any call returns cancels the ctx handed to every other +// call, in flight or not yet started: a call not yet issued when an earlier +// one already failed is skipped rather than made and discarded, and an +// in-flight call that itself respects ctx cancellation (as an +// http.NewRequestWithContext-built request does) is aborted rather than run +// to completion for an answer nobody will use. Run itself returns that first +// error, wrapped by nothing -- whatever work returned is what the caller +// gets, so the caller's own wrapping (naming which of the n things failed, +// what identity it carried) survives unchanged. What each call's own input or +// identity looks like is entirely work's business -- Run passes only an +// index, so it never has to know or care what a caller maps that index onto. +// +// n == 0 calls work zero times and returns (nil, nil). limit is passed +// straight to errgroup.Group.SetLimit -- see its own doc for what a +// non-positive value means. +func Run[T any](ctx context.Context, n, limit int, work func(ctx context.Context, index int) (T, error)) ([]T, error) { + if n == 0 { + return nil, nil + } + + group, groupCtx := errgroup.WithContext(ctx) + group.SetLimit(limit) + + results := make([]T, n) + for index := range n { + group.Go(func() error { + if groupCtx.Err() != nil { + return nil + } + result, err := work(groupCtx, index) + if err != nil { + return fmt.Errorf("%w", err) + } + results[index] = result + return nil + }) + } + + if err := group.Wait(); err != nil { + return nil, err + } + return results, nil +} + +// Map is Run over a list of VALUES rather than a count: one call per value, +// results in the order the values were given. +// +// The same bounded, ordered, fail-fast guarantees as Run -- it is Run, with +// the index-to-value step done here instead of in every caller. Prefer it +// whenever the work is "one call per thing in this list". +// +// values is []V and results are []T, so a caller keeps whatever types it +// already has; nothing here inspects either. +// +// What Map does NOT decide, on purpose: how many values are too many, and +// what to tell a caller who asked for too many. A ceiling is a fact about +// whatever is being called and its refusal is that domain's error to write, +// so it belongs to the package that knows the domain. Use Bound below for the +// other half of the arithmetic, defaulting and clamping a configured +// concurrency. +func Map[V, T any](ctx context.Context, values []V, limit int, + call func(ctx context.Context, value V) (T, error)) ([]T, error) { + + return Run(ctx, len(values), limit, func(ctx context.Context, index int) (T, error) { + return call(ctx, values[index]) + }) +} + +// Bound resolves a configured concurrency: fallback when it is unset or +// non-positive, ceiling when it is over. +// +// Here rather than in each caller because every caller of Run or Map needs +// the same three lines, and getting them wrong is quiet: a zero limit means +// UNBOUNDED in errgroup, so a deployment that leaves the setting out would get +// every call at once from a mistake that reads like a default. +func Bound(configured, fallback, ceiling int) int { + limit := configured + if limit <= 0 { + limit = fallback + } + if limit > ceiling { + limit = ceiling + } + return limit +} diff --git a/pkg/plugin/implementation/internal/concurrent/concurrent_test.go b/pkg/plugin/implementation/internal/concurrent/concurrent_test.go new file mode 100644 index 00000000..a35d4c4f --- /dev/null +++ b/pkg/plugin/implementation/internal/concurrent/concurrent_test.go @@ -0,0 +1,288 @@ +package concurrent + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestRunReturnsResultsInOrderNotArrivalOrder(t *testing.T) { + t.Parallel() + + // index 0 is the slowest, so an arrival-ordered result would put it + // last -- this proves the result is indexed, not appended. + delays := []time.Duration{30 * time.Millisecond, 10 * time.Millisecond, 0} + results, err := Run(context.Background(), len(delays), len(delays), + func(ctx context.Context, i int) (int, error) { + time.Sleep(delays[i]) + return i, nil + }) + if err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + for i, got := range results { + if got != i { + t.Errorf("results[%d] = %d, want %d", i, got, i) + } + } +} + +func TestRunIsSequentialWhenLimitIsOne(t *testing.T) { + t.Parallel() + + var mu sync.Mutex + var inFlight, peak int + _, err := Run(context.Background(), 4, 1, func(ctx context.Context, i int) (struct{}, error) { + mu.Lock() + inFlight++ + if inFlight > peak { + peak = inFlight + } + mu.Unlock() + + time.Sleep(10 * time.Millisecond) + + mu.Lock() + inFlight-- + mu.Unlock() + return struct{}{}, nil + }) + if err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + if peak != 1 { + t.Errorf("peak concurrency = %d, want 1", peak) + } +} + +func TestRunHonoursALimitAboveOne(t *testing.T) { + t.Parallel() + + var mu sync.Mutex + var inFlight, peak int + _, err := Run(context.Background(), 6, 2, func(ctx context.Context, i int) (struct{}, error) { + mu.Lock() + inFlight++ + if inFlight > peak { + peak = inFlight + } + mu.Unlock() + + time.Sleep(15 * time.Millisecond) + + mu.Lock() + inFlight-- + mu.Unlock() + return struct{}{}, nil + }) + if err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + if peak != 2 { + t.Errorf("peak concurrency = %d, want the configured 2", peak) + } +} + +func TestRunReturnsTheCallsOwnError(t *testing.T) { + t.Parallel() + + sentinel := errors.New("boom") + _, err := Run(context.Background(), 3, 3, func(ctx context.Context, i int) (int, error) { + if i == 1 { + return 0, fmt.Errorf("call %d failed: %w", i, sentinel) + } + return i, nil + }) + if err == nil || !errors.Is(err, sentinel) { + t.Fatalf("error = %v, want it to wrap the call's own error", err) + } +} + +func TestRunSkipsCallsNotYetIssuedAfterAFailureWhenSequential(t *testing.T) { + t.Parallel() + + var called int32 + _, err := Run(context.Background(), 3, 1, func(ctx context.Context, i int) (struct{}, error) { + atomic.AddInt32(&called, 1) + if i == 0 { + return struct{}{}, errors.New("first call fails") + } + return struct{}{}, nil + }) + if err == nil { + t.Fatal("Run() returned no error, want the first call's failure") + } + // Sequential (limit 1): index 0 fails and cancels the shared context + // before index 1's call is ever issued, so only index 0 was called. + if got := atomic.LoadInt32(&called); got != 1 { + t.Errorf("work was called %d times, want 1 -- calls not yet issued "+ + "when an earlier one fails must be skipped, not made and discarded", got) + } +} + +func TestRunOfZeroReturnsNoResultsAndNoError(t *testing.T) { + t.Parallel() + + results, err := Run(context.Background(), 0, 4, func(ctx context.Context, i int) (int, error) { + t.Error("work was called for n == 0") + return 0, nil + }) + if err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + if len(results) != 0 { + t.Errorf("results = %v, want empty", results) + } +} + +func TestRunOfOneRunsExactlyOnce(t *testing.T) { + t.Parallel() + + var called int32 + results, err := Run(context.Background(), 1, 1, func(ctx context.Context, i int) (string, error) { + atomic.AddInt32(&called, 1) + return "ok", nil + }) + if err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + if got := atomic.LoadInt32(&called); got != 1 { + t.Errorf("work was called %d times, want 1", got) + } + if len(results) != 1 || results[0] != "ok" { + t.Errorf("results = %v, want [\"ok\"]", results) + } +} + +func TestRunCancelsTheContextItHandsToInFlightWork(t *testing.T) { + t.Parallel() + + var slowCtxDone int32 + // inFlight is closed by index 1 once it is genuinely running, and index 0 + // waits for that before failing. Without the handshake this test is a + // race: index 1's call may not have started when index 0 fails, and Run + // then SKIPS it rather than cancelling it -- correct behaviour, and what + // TestRunSkipsCallsNotYetIssuedAfterAFailureWhenSequential is for, but + // not what this test is about. + inFlight := make(chan struct{}) + _, err := Run(context.Background(), 2, 2, func(ctx context.Context, i int) (struct{}, error) { + if i == 0 { + <-inFlight + return struct{}{}, errors.New("fails once the other call is in flight") + } + close(inFlight) + // In flight when index 0 fails: wait for our own ctx to be cancelled, + // or give up after a bound that would mean it never was. A bound + // rather than a bare receive so a broken Run fails this test instead + // of hanging it. + select { + case <-ctx.Done(): + atomic.AddInt32(&slowCtxDone, 1) + case <-time.After(2 * time.Second): + } + return struct{}{}, nil + }) + if err == nil { + t.Fatal("Run() returned no error, want index 0's failure") + } + if atomic.LoadInt32(&slowCtxDone) != 1 { + t.Error("the in-flight call's context was not cancelled after the other call failed") + } +} + +// --- Map -------------------------------------------------------------------- + +// Map hands each value to its own call and returns the answers in the order +// the values were given, whatever order the calls finished in. +func TestMapCallsOncePerValueInOrder(t *testing.T) { + t.Parallel() + + values := []string{"kvk", "warehouse", "soil_lab"} + results, err := Map(context.Background(), values, 3, func(ctx context.Context, value string) (string, error) { + // The first value sleeps longest, so arrival order is the reverse of + // the order asked for. A caller relying on results[i] meaning "the + // answer for values[i]" has to hold regardless. + if value == "kvk" { + time.Sleep(40 * time.Millisecond) + } + return "answered:" + value, nil + }) + if err != nil { + t.Fatalf("Map() returned an unexpected error: %v", err) + } + want := []string{"answered:kvk", "answered:warehouse", "answered:soil_lab"} + if len(results) != len(want) { + t.Fatalf("results = %v, want %v", results, want) + } + for index := range want { + if results[index] != want[index] { + t.Errorf("results[%d] = %q, want %q -- value order, not arrival order", + index, results[index], want[index]) + } + } +} + +// An empty list makes no calls and is not an error: a caller deciding that an +// empty list is a bad request says so itself, in its own words. +func TestMapOfNoValuesMakesNoCalls(t *testing.T) { + t.Parallel() + + results, err := Map(context.Background(), nil, 2, func(ctx context.Context, value string) (int, error) { + t.Error("a call was made for an empty value list") + return 0, nil + }) + if err != nil { + t.Fatalf("Map() returned an unexpected error: %v", err) + } + if len(results) != 0 { + t.Errorf("results = %v, want empty", results) + } +} + +// One value's failure fails the whole call, and the value's own error is what +// comes back -- the caller named which value it was and why. +func TestMapFailsWhenOneValueFails(t *testing.T) { + t.Parallel() + + sentinel := errors.New("warehouse is unreachable") + _, err := Map(context.Background(), []string{"kvk", "warehouse"}, 1, + func(ctx context.Context, value string) (string, error) { + if value == "warehouse" { + return "", sentinel + } + return "ok", nil + }) + if !errors.Is(err, sentinel) { + t.Errorf("error = %v, want the failing value's own error", err) + } +} + +// --- Bound ------------------------------------------------------------------ + +// Bound is the arithmetic every caller of Run and Map needs: a configured +// limit, a fallback when it is unset, a ceiling when it is over. Zero must +// become the fallback and not reach errgroup, where it means UNBOUNDED. +func TestBoundDefaultsAndClamps(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + configured, fallback, ceiling, want int + }{ + {"unset takes the fallback", 0, 1, 8, 1}, + {"negative takes the fallback", -4, 2, 8, 2}, + {"within range is honoured", 4, 1, 8, 4}, + {"exactly the ceiling is honoured", 8, 1, 8, 8}, + {"over the ceiling is clamped", 64, 1, 8, 8}, + {"a fallback over the ceiling is clamped too", 0, 16, 8, 8}, + } { + if got := Bound(tc.configured, tc.fallback, tc.ceiling); got != tc.want { + t.Errorf("%s: Bound(%d, %d, %d) = %d, want %d", + tc.name, tc.configured, tc.fallback, tc.ceiling, got, tc.want) + } + } +} From 45f16e54aff4f10ac9fba1057187aa21ff0029a1 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Thu, 10 Sep 2026 00:50:49 +0530 Subject: [PATCH 124/130] refactor(AgricultureFacility): read the facility types by path, not by hand [#91] facilityTypesFrom walked the document in Go -- dig(message, contract, commitments), then [0], then resources, then [0], then resourceAttributes -- with a type assertion and an error message at each level. That compiles the payload's shape into the binary: a spec change needs a rebuild, and eleven lines re-check what a walker already checks. The shape is the network's convention, not this adapter's, so it is now a PATH, defaulting to the Beckn v2 location and overridable per deployment -- exactly as common.Config's providerIdAt and capabilityCodeAt already are: facilityTypesAt: message.contract.commitments[].resources[]. resourceAttributes.supportedFacilityTypes[] The reading is common's walker, the same one that finds a binding key, so the traversal and its absent-field handling live in one place. A test proves the point: a payload carrying the types at search.facilities[].kinds[] is read correctly with nothing but a config path. WHAT THIS NEARLY BROKE, caught by their own tests. ValuesAt drops a non-string leaf, which is right for a binding key and wrong here: supportedFacilityTypes ["KrishiVigyanKendra", 42] would have searched for one type and reported success, a partial answer with nothing recording the loss -- the same class of bug the fan-out work existed to fix. common now exposes LeavesAt alongside ValuesAt: one walker, two views, type policy left to the caller. The plugin takes the leaves and refuses a non-string with the message it always used. An empty supportedFacilityTypes needed care too: it reads as zero leaves like an absent one, and the bare-string fallback would then hand back the empty list itself, reported as "[] is not a facility type" -- true and useless. A leaf that is itself a list is skipped, so an empty list keeps the "names no facility type" message. common's own tests pass untouched, so binding-key extraction is unchanged. 69 packages green, vet clean, no races. dig() remains for now: search.go still uses it to WRITE into a payload when splitting by type, which is a different primitive from reading and not something common offers. --- .../AgricultureFacility.go | 16 ++- .../AgricultureFacility/cmd/plugin.go | 3 + .../AgricultureFacility/payload.go | 103 ++++++++++-------- .../AgricultureFacility/payload_test.go | 41 ++++++- .../AgricultureFacility/search.go | 6 +- .../implementation/internal/common/binding.go | 4 +- .../internal/common/binding_test.go | 6 +- .../implementation/internal/common/paths.go | 47 ++++++-- 8 files changed, 153 insertions(+), 73 deletions(-) diff --git a/pkg/plugin/implementation/AgricultureFacility/AgricultureFacility.go b/pkg/plugin/implementation/AgricultureFacility/AgricultureFacility.go index 17f1e8c8..59a35d61 100644 --- a/pkg/plugin/implementation/AgricultureFacility/AgricultureFacility.go +++ b/pkg/plugin/implementation/AgricultureFacility/AgricultureFacility.go @@ -49,6 +49,13 @@ type Config struct { // schemes are defined and validated -- see common.AuthProfile. AuthByProvider map[string]*common.AuthProfile `yaml:"-" json:"-"` + // FacilityTypesAt is where the payload carries the facility types a search + // asks for. Absent means DefaultFacilityTypesAt, which is the Beckn v2 + // convention and what every deployment should be using -- the override is + // for tracking a spec change without waiting for a release, exactly as + // common.Config's providerIdAt and capabilityCodeAt are. + FacilityTypesAt string `yaml:"facilityTypesAt" json:"facilityTypesAt"` + // SearchConcurrency is how many of a multi-type search's calls may be in // flight at once. See search.go's DefaultSearchConcurrency and // MaxFacilityTypes for what absent and too-large mean. @@ -96,9 +103,10 @@ func New(ctx context.Context, registry definition.ProviderRecordLookup, mapper d } return &Step{ - inner: one, - paths: paths, - bindingKeys: cfg.BindingKeys, - concurrency: searchConcurrency(cfg.SearchConcurrency), + inner: one, + paths: paths, + bindingKeys: cfg.BindingKeys, + facilityTypesAt: cfg.FacilityTypesAt, + concurrency: searchConcurrency(cfg.SearchConcurrency), }, closer, nil } diff --git a/pkg/plugin/implementation/AgricultureFacility/cmd/plugin.go b/pkg/plugin/implementation/AgricultureFacility/cmd/plugin.go index 25a13ef3..d2ecb2be 100644 --- a/pkg/plugin/implementation/AgricultureFacility/cmd/plugin.go +++ b/pkg/plugin/implementation/AgricultureFacility/cmd/plugin.go @@ -35,6 +35,9 @@ func (p agriFacilityProvider) parseConfig(config map[string]string) (*Agricultur // this is a default rather than something to set. ProviderIDAt: config["providerIdAt"], CapabilityCodeAt: config["capabilityCodeAt"], + // Absent means the Beckn v2 convention. See + // AgricultureFacility.DefaultFacilityTypesAt. + FacilityTypesAt: config["facilityTypesAt"], } // One credential profile per provider, read from the flattened diff --git a/pkg/plugin/implementation/AgricultureFacility/payload.go b/pkg/plugin/implementation/AgricultureFacility/payload.go index 3f85ffba..89eb36f0 100644 --- a/pkg/plugin/implementation/AgricultureFacility/payload.go +++ b/pkg/plugin/implementation/AgricultureFacility/payload.go @@ -23,77 +23,83 @@ package AgricultureFacility import ( "fmt" "slices" + "strings" "github.com/beckn-one/beckn-onix/pkg/model" + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" ) +// DefaultFacilityTypesAt is where a Beckn v2 select carries the facility types +// a search asks for. +// +// A PATH rather than a walk written out in Go, for the same reason +// common.Config's providerIdAt and capabilityCodeAt are paths: the payload's +// shape is the network's convention, not this adapter's. Compiled in, a spec +// change needs a rebuild; as a path, it needs a config edit. The default is +// what every deployment should be using, and facilityTypesAt exists so a spec +// change can be tracked without waiting for a release. +const DefaultFacilityTypesAt = "message.contract.commitments[].resources[]." + + "resourceAttributes.supportedFacilityTypes[]" + // facilityTypesFrom returns the facility types a payload asks for, in the // order the payload wrote them -- one upstream call each. // +// The reading is common.ValuesAt's, the same walker that finds a binding key, +// so the type assertions and the absent-field handling are in one place rather +// than repeated here. It returns only the strings the path reaches: a missing +// field, an object where a list belongs, or a number among the types all +// arrive as "no value at that path", which is a bad request either way. +// // Every failure is a bad request rather than an adapter fault: the payload is // the thing that is wrong, and the caller is the only one who can fix it. The // mapping's own required: checks refuse most of these first, with better -// messages; these are what is left if a check is relaxed or a payload reaches -// here another way, and they exist so that path is a refusal rather than a -// panic on a nil map. -func facilityTypesFrom(beckn any) ([]string, error) { - document, ok := beckn.(map[string]any) - if !ok { - return nil, model.NewBadReqErr("", fmt.Errorf( - "agriculture facility: the payload is %T, not an object, so it names no commitment to search from", beckn)) - } - - commitments, _ := dig(document, "message", "contract", "commitments").([]any) - if len(commitments) == 0 { - return nil, model.NewBadReqErr("", fmt.Errorf( - "agriculture facility: the payload carries no commitment to read a facility search from")) - } - commitment, ok := commitments[0].(map[string]any) - if !ok { - return nil, model.NewBadReqErr("", fmt.Errorf( - "agriculture facility: the payload's first commitment is %T, not an object", commitments[0])) +// messages; these are what is left if a check is relaxed. +func facilityTypesFrom(beckn any, path string) ([]string, error) { + if path == "" { + path = DefaultFacilityTypesAt } - resources, _ := commitment["resources"].([]any) - if len(resources) == 0 { - return nil, model.NewBadReqErr("", fmt.Errorf( - "agriculture facility: the payload's commitment carries no resource to read a facility search from")) - } - resource, ok := resources[0].(map[string]any) - if !ok { - return nil, model.NewBadReqErr("", fmt.Errorf( - "agriculture facility: the payload's first resource is %T, not an object", resources[0])) - } - - declared := dig(resource, "resourceAttributes", "supportedFacilityTypes") + // LeavesAt, not ValuesAt: a non-string here is the caller's mistake and has + // to be reported. ValuesAt would drop it, and dropping one entry of a + // hand-written list means searching for the rest and reporting success -- + // a partial answer with nothing recording what was lost. + leaves := common.LeavesAt(beckn, path) // A bare string is one value: the same instruction as a one-element list, - // which is what a caller writing a single type may well send. - var declaredList []any - switch typed := declared.(type) { - case nil: - declaredList = nil - case []any: - declaredList = typed - default: - declaredList = []any{typed} + // which is what a caller writing a single type may well send. The path's + // "[]" suffix wants a list, so the scalar form is a second read rather than + // a looser walker -- keeping "[]" meaning exactly "a list" everywhere. + // + // A list that is PRESENT and empty reads as zero leaves too, and the second + // read would then hand back the empty list itself as one value -- reported + // as "[] is not a facility type", which is true and useless. Skipping a + // leaf that is itself a list keeps that case on the "names no facility + // type" message, which is what an empty list means. + if len(leaves) == 0 { + for _, leaf := range common.LeavesAt(beckn, strings.TrimSuffix(path, arrayMarker)) { + if _, isList := leaf.([]any); isList { + continue + } + leaves = append(leaves, leaf) + } } - if len(declaredList) == 0 { + + if len(leaves) == 0 { return nil, model.NewBadReqErr("", fmt.Errorf( - "agriculture facility: the payload names no facility type in supportedFacilityTypes, "+ - "so there is nothing to ask the provider for")) + "agriculture facility: the payload names no facility type at %s, "+ + "so there is nothing to ask the provider for", path)) } // Each value becomes a category code in the request half's $codes lookup, // so a non-string is a request this capability cannot build -- refused // here rather than sent as a null category POCRA answers with everything. - values := make([]string, 0, len(declaredList)) - for _, value := range declaredList { - text, ok := value.(string) + values := make([]string, 0, len(leaves)) + for _, leaf := range leaves { + text, ok := leaf.(string) if !ok { return nil, model.NewBadReqErr("", fmt.Errorf( "agriculture facility: supportedFacilityTypes contains %v (%T), which is not a facility type", - value, value)) + leaf, leaf)) } values = append(values, text) } @@ -117,6 +123,9 @@ func facilityTypesFrom(beckn any) ([]string, error) { return values, nil } +// arrayMarker is common.ValuesAt's "look in each element" suffix. +const arrayMarker = "[]" + // dig walks a chain of object keys, returning nil the moment one is absent or // is not an object. Written out rather than reached for from a library because // a mistyped key must read as an absent field, which is a bad request, and not diff --git a/pkg/plugin/implementation/AgricultureFacility/payload_test.go b/pkg/plugin/implementation/AgricultureFacility/payload_test.go index c58701d6..7465d78d 100644 --- a/pkg/plugin/implementation/AgricultureFacility/payload_test.go +++ b/pkg/plugin/implementation/AgricultureFacility/payload_test.go @@ -10,8 +10,11 @@ package AgricultureFacility import ( "encoding/json" + "reflect" "strings" "testing" + + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" ) // decode is what Step.Run works from: the payload, already decoded into @@ -34,7 +37,7 @@ func TestFacilityTypesFromReadsThemInPayloadOrder(t *testing.T) { beckn := decode(t, `{"message":{"contract":{"commitments":[{"resources":[{"resourceAttributes":{ "supportedFacilityTypes":["KrishiVigyanKendra","Warehouse","SoilTestingFacility"]}}]}]}}}`) - values, err := facilityTypesFrom(beckn) + values, err := facilityTypesFrom(beckn, "") if err != nil { t.Fatalf("facilityTypesFrom() returned an unexpected error: %v", err) } @@ -56,7 +59,7 @@ func TestFacilityTypesFromAcceptsASingleType(t *testing.T) { beckn := decode(t, `{"message":{"contract":{"commitments":[{"resources":[{"resourceAttributes":{ "supportedFacilityTypes":["Warehouse"]}}]}]}}}`) - values, err := facilityTypesFrom(beckn) + values, err := facilityTypesFrom(beckn, "") if err != nil { t.Fatalf("facilityTypesFrom() returned an unexpected error: %v", err) } @@ -75,7 +78,7 @@ func TestFacilityTypesFromAcceptsABareString(t *testing.T) { beckn := decode(t, `{"message":{"contract":{"commitments":[{"resources":[{"resourceAttributes":{ "supportedFacilityTypes":"KrishiVigyanKendra"}}]}]}}}`) - values, err := facilityTypesFrom(beckn) + values, err := facilityTypesFrom(beckn, "") if err != nil { t.Fatalf("facilityTypesFrom() returned an unexpected error: %v", err) } @@ -109,7 +112,7 @@ func TestFacilityTypesFromRefusesWhatItCannotRead(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - values, err := facilityTypesFrom(decode(t, tc.body)) + values, err := facilityTypesFrom(decode(t, tc.body), "") if err == nil { t.Fatalf("facilityTypesFrom() returned %v, want %s refused", values, tc.name) } @@ -126,8 +129,36 @@ func TestFacilityTypesFromRefusesANonObjectPayload(t *testing.T) { t.Parallel() for _, beckn := range []any{nil, "a string", 42.0, []any{"a", "list"}} { - if _, err := facilityTypesFrom(beckn); err == nil { + if _, err := facilityTypesFrom(beckn, ""); err == nil { t.Errorf("facilityTypesFrom(%v) was accepted, want it refused", beckn) } } } + +// THE POINT OF THE PATH. A payload that carries the types somewhere else is a +// config edit, not a rebuild: the same reading works against a different shape +// when facilityTypesAt says where to look. +// +// This is what the hand-written walk could not do. It knew +// message.contract.commitments[0].resources[0].resourceAttributes as Go code, +// so a spec change meant editing and shipping a binary. +func TestFacilityTypesFollowAConfiguredPath(t *testing.T) { + t.Parallel() + + // A shape the default path would find nothing in. + body := `{"search":{"facilities":[ + {"kinds":["Warehouse","KrishiVigyanKendra"]}]}}` + + if got := common.LeavesAt(decode(t, body), DefaultFacilityTypesAt); len(got) != 0 { + t.Fatalf("the default path found %v in a payload it should not read", got) + } + + values, err := facilityTypesFrom(decode(t, body), "search.facilities[].kinds[]") + if err != nil { + t.Fatalf("facilityTypesFrom() with a configured path: %v", err) + } + want := []string{"Warehouse", "KrishiVigyanKendra"} + if !reflect.DeepEqual(values, want) { + t.Errorf("values = %v, want %v", values, want) + } +} diff --git a/pkg/plugin/implementation/AgricultureFacility/search.go b/pkg/plugin/implementation/AgricultureFacility/search.go index ddcd9b41..452ea2c3 100644 --- a/pkg/plugin/implementation/AgricultureFacility/search.go +++ b/pkg/plugin/implementation/AgricultureFacility/search.go @@ -92,6 +92,10 @@ type Step struct { paths common.Paths bindingKeys []string + // facilityTypesAt is where the payload carries the types to search for. + // A path rather than a walk in Go, so the payload's shape stays config. + facilityTypesAt string + // concurrency is how many of a split search's calls run at once. concurrency int } @@ -113,7 +117,7 @@ func (s *Step) Run(ctx *model.StepContext) error { "agriculture facility: the payload is not JSON: %w", err)) } - types, err := facilityTypesFrom(beckn) + types, err := facilityTypesFrom(beckn, s.facilityTypesAt) if err != nil { return err } diff --git a/pkg/plugin/implementation/internal/common/binding.go b/pkg/plugin/implementation/internal/common/binding.go index 8d657d32..4a325306 100644 --- a/pkg/plugin/implementation/internal/common/binding.go +++ b/pkg/plugin/implementation/internal/common/binding.go @@ -64,8 +64,8 @@ func BindingFrom(paths Paths, body []byte) (Binding, error) { commitments) } - providers := distinct(valuesAt(payload, paths.ProviderID)) - types := distinct(valuesAt(payload, paths.CapabilityCode)) + providers := distinct(ValuesAt(payload, paths.ProviderID)) + types := distinct(ValuesAt(payload, paths.CapabilityCode)) if len(providers) == 0 || len(types) == 0 { return Binding{}, errNoBinding diff --git a/pkg/plugin/implementation/internal/common/binding_test.go b/pkg/plugin/implementation/internal/common/binding_test.go index d0195221..11552cce 100644 --- a/pkg/plugin/implementation/internal/common/binding_test.go +++ b/pkg/plugin/implementation/internal/common/binding_test.go @@ -248,13 +248,13 @@ func TestPathWalk(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - got := valuesAt(doc, tc.path) + got := ValuesAt(doc, tc.path) if len(got) != len(tc.want) { - t.Fatalf("valuesAt(%q) = %v, want %v", tc.path, got, tc.want) + t.Fatalf("ValuesAt(%q) = %v, want %v", tc.path, got, tc.want) } for i, want := range tc.want { if got[i] != want { - t.Errorf("valuesAt(%q)[%d] = %q, want %q", tc.path, i, got[i], want) + t.Errorf("ValuesAt(%q)[%d] = %q, want %q", tc.path, i, got[i], want) } } }) diff --git a/pkg/plugin/implementation/internal/common/paths.go b/pkg/plugin/implementation/internal/common/paths.go index af1ded59..96491903 100644 --- a/pkg/plugin/implementation/internal/common/paths.go +++ b/pkg/plugin/implementation/internal/common/paths.go @@ -45,13 +45,28 @@ func (p Paths) Validate() error { return nil } -// valuesAt collects every string the path reaches. +// ValuesAt collects every string the path reaches. // // The grammar is two things: segments separated by ".", and a "[]" suffix // meaning "look in each element". No wildcards, filters or indices -- each is // another way to write something subtly wrong in config nobody reviews. -func valuesAt(node any, path string) []string { - return walk(node, strings.Split(path, ".")) +// +// EXPORTED for a domain package that has to read its own field out of a +// payload -- AgricultureFacility reads supportedFacilityTypes this way. The +// alternative is hand-walking the document in Go, which compiles the payload's +// shape into the binary: a spec change then needs a rebuild rather than a +// config edit, and the traversal has to re-check every type assertion the +// walker already checks. +func ValuesAt(node any, path string) []string { + var found []string + for _, leaf := range walk(node, strings.Split(path, ".")) { + // Only strings are binding-key material; anything else means the path + // landed somewhere unintended. + if value, ok := leaf.(string); ok { + found = append(found, value) + } + } + return found } // countAt reports how many elements the first array segment holds, whether or @@ -83,14 +98,24 @@ func countAt(node any, path string) int { return 0 } -func walk(node any, segments []string) []string { +// LeavesAt collects every value the path reaches, whatever its type. +// +// ValuesAt's sibling, and the one a caller wants when a non-string at the leaf +// is a MISTAKE to report rather than a leaf to skip. ValuesAt drops it, which +// is right for a binding key -- a number where a capability code belongs means +// the path landed somewhere unintended, and there is nothing to say about it +// beyond "no binding here". It is wrong for a list the caller wrote by hand: +// dropping one entry of supportedFacilityTypes would search for the rest and +// report success, which is a partial answer with nothing recording the loss. +func LeavesAt(node any, path string) []any { + return walk(node, strings.Split(path, ".")) +} + +// walk collects the leaves a path reaches. Type policy is the caller's: this +// returns what is there, ValuesAt keeps the strings, LeavesAt keeps everything. +func walk(node any, segments []string) []any { if len(segments) == 0 { - // Only strings are binding-key material; anything else means the path - // landed somewhere unintended. - if value, ok := node.(string); ok { - return []string{value} - } - return nil + return []any{node} } segment := segments[0] @@ -114,7 +139,7 @@ func walk(node any, segments []string) []string { if !ok { return nil } - var found []string + var found []any for _, element := range elements { found = append(found, walk(element, rest)...) } From dcfeb03037923b2778dc2d7dd10c6afcc3cd8fb4 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Thu, 10 Sep 2026 01:35:23 +0530 Subject: [PATCH 125/130] refactor(AgricultureFacility): the mapping derives its own request id [#91] The plugin stamped a fresh UUID on every part it split out, so POCRA would not blend the answers. That put the reason in Go and the consequence in the mapping, which then had a comment explaining what the Go code had done for it. The mapping derives the id itself now, from the caller's messageId with its last hex digit replaced by a slot the facility type owns. The TYPE picks the slot rather than a call index, because every payload reaching the request half names exactly one type -- so no index has to be passed, and a retry reuses its own slot instead of taking a fresh one. WHY THE CALLER'S OWN ID CANNOT BE SENT, verified live against POCRA: its cache is keyed on message_id, holds for PT10M, and IGNORES LOCATION. The same id asked twice 350km apart returned the first location's facility alongside the second's, which no type filter can catch because the type matches. Reusing it would answer a farmer in Nagpur with a facility near Ahmednagar. Replacing the last character leaves a valid v4 UUID -- it sits in the node field, not the version or variant nibbles -- which POCRA's schema requires. Derived rather than random because JSONata here has no $uuid. That deletes, in order: the UUID stamping, the merged answer's messageId restore (parts never lose the caller's id now), messageIDOf, the uuid import, and dig -- the last hand-written traversal in this package's production code. splitByType writes through the SAME configured path the types were read from, so the read and the write cannot disagree about where they live. setAt and containersAt are local to this package, not added to common: common's step reads payloads and never rewrites one, and a write raises questions a read does not. They create nothing, so a path that does not resolve is a refusal rather than a payload invented to fit. Their own TestEachSearchCallCarriesItsOwnRequestId already asserted the exact property this change has to keep -- two calls, distinct ids, both valid UUIDs, neither the caller's. Negative control: giving two types the same slot fails it with "both calls used message_id ... so POCRA would blend their answers". 69 packages green, vet clean, no races. --- .../pocra/agriculture-facility.select.yaml | 40 ++++++--- .../AgricultureFacility/payload.go | 80 +++++++++++++++--- .../AgricultureFacility/search.go | 81 +++++-------------- 3 files changed, 116 insertions(+), 85 deletions(-) diff --git a/config/mappings/pocra/agriculture-facility.select.yaml b/config/mappings/pocra/agriculture-facility.select.yaml index 44aee0fe..9d98f3b3 100644 --- a/config/mappings/pocra/agriculture-facility.select.yaml +++ b/config/mappings/pocra/agriculture-facility.select.yaml @@ -116,6 +116,33 @@ request: | "Warehouse": "warehouse", "SoilTestingFacility": "soil_lab" }; + /* THIS CALL'S OWN message_id, derived from the caller's by replacing its + last hex digit with a slot this facility type owns. + + POCRA caches per message_id for PT10M and returns the UNION of + everything asked under it, so two calls sharing an id each come back + carrying the other's facilities. Worse, that cache IGNORES LOCATION: + verified live, the same id asked twice 350km apart returned the first + location's facility alongside the second's, which no type filter can + catch because the type matches. So the caller's own id cannot be sent + as it stands. + + The last character of a v4 UUID is in the node field, not the version + or variant nibbles, so replacing it leaves a valid UUID -- which POCRA's + schema requires. Derived rather than random because this expression has + no random source: JSONata here has no $uuid. + + The TYPE picks the slot, not a call index, because each payload reaching + here names exactly one type. That also makes a retry reuse its own slot + rather than take a fresh one. */ + $slots := { + "CustomHiringCentre": "0", + "KrishiVigyanKendra": "1", + "Warehouse": "2", + "SoilTestingFacility": "3" + }; + $callerId := beckn.context.messageId; + $messageId := $substring($callerId, 0, 35) & $lookup($slots, $requested); { "context": { "domain": "advisory:mh-vistaar", @@ -124,16 +151,9 @@ request: | "version": "1.1.0", "bap_id": "bap.mahapocra.gov.in", "bap_uri": "https://middleware.mahapocra.gov.in/bap/", - /* This call's own id. POCRA keeps a message_id's answers for ten - minutes and returns the UNION of everything asked for under it, so - two calls sharing an id would each return the other's facilities - too. - - Read straight off the payload: the plugin stamps a fresh UUID on - context.messageId of every single-type payload it splits out, for - exactly this reason, so each call already carries its own. POCRA's - schema refuses a message_id that is not a UUID. */ - "message_id": beckn.context.messageId, + /* Derived above, one slot per facility type. NOT the caller's own id: + POCRA's cache is keyed on it and ignores location. */ + "message_id": $messageId, "transaction_id": beckn.context.transactionId, "timestamp": $string($floor($millis() / 1000)) }, diff --git a/pkg/plugin/implementation/AgricultureFacility/payload.go b/pkg/plugin/implementation/AgricultureFacility/payload.go index 89eb36f0..d913a5c9 100644 --- a/pkg/plugin/implementation/AgricultureFacility/payload.go +++ b/pkg/plugin/implementation/AgricultureFacility/payload.go @@ -41,6 +41,14 @@ import ( const DefaultFacilityTypesAt = "message.contract.commitments[].resources[]." + "resourceAttributes.supportedFacilityTypes[]" +// CommitmentsAt is where a Beckn contract carries its commitments -- the level +// mergeAnswers appends one answer's resources onto another's. +// +// Not configurable, unlike DefaultFacilityTypesAt: this reads the answers this +// package's own mapping produced, not a payload a caller wrote, so it moves +// only when that mapping does and the two are edited together. +const CommitmentsAt = "message.contract.commitments[]" + // facilityTypesFrom returns the facility types a payload asks for, in the // order the payload wrote them -- one upstream call each. // @@ -126,18 +134,64 @@ func facilityTypesFrom(beckn any, path string) ([]string, error) { // arrayMarker is common.ValuesAt's "look in each element" suffix. const arrayMarker = "[]" -// dig walks a chain of object keys, returning nil the moment one is absent or -// is not an object. Written out rather than reached for from a library because -// a mistyped key must read as an absent field, which is a bad request, and not -// as a panic. -func dig(document map[string]any, keys ...string) any { - var current any = document - for _, key := range keys { - object, ok := current.(map[string]any) - if !ok { - return nil - } - current = object[key] +// setAt writes value at the leaf the path names, creating nothing. +// +// The mirror of common.ValuesAt, and the same grammar, so ONE configured path +// -- facilityTypesAt -- both reads the types out of a payload and narrows them +// in a part. Two paths that had to agree would be a way for them to disagree. +// +// Local to this package rather than in common: common's step reads a payload +// and never rewrites one, and a write raises questions a read does not -- +// whether to create the intermediates, what an array segment means when +// assigning. This answers both narrowly. It creates nothing, so a path that +// does not already resolve is a refusal rather than a payload invented to fit, +// and an array segment writes to EVERY element, which is what narrowing a +// search to one type means when a payload carries several commitments. +// +// Reported as a bad request: the payload is what is wrong, and the caller is +// the only one who can fix it. +func setAt(document any, path string, value any) error { + segments := strings.Split(path, ".") + targets := containersAt(document, segments[:len(segments)-1]) + if len(targets) == 0 { + return model.NewBadReqErr("", fmt.Errorf( + "agriculture facility: the payload has nothing at %s to narrow to one facility type", + strings.Join(segments[:len(segments)-1], "."))) + } + leaf := strings.TrimSuffix(segments[len(segments)-1], arrayMarker) + for _, target := range targets { + target[leaf] = value + } + return nil +} + +// containersAt collects every object the path reaches, so setAt has somewhere +// to write. Absent or wrongly-typed reaches nothing, which setAt reports. +func containersAt(node any, segments []string) []map[string]any { + object, ok := node.(map[string]any) + if !ok { + return nil + } + if len(segments) == 0 { + return []map[string]any{object} + } + + segment := segments[0] + child, present := object[strings.TrimSuffix(segment, arrayMarker)] + if !present { + return nil + } + if !strings.HasSuffix(segment, arrayMarker) { + return containersAt(child, segments[1:]) + } + + elements, ok := child.([]any) + if !ok { + return nil + } + var found []map[string]any + for _, element := range elements { + found = append(found, containersAt(element, segments[1:])...) } - return current + return found } diff --git a/pkg/plugin/implementation/AgricultureFacility/search.go b/pkg/plugin/implementation/AgricultureFacility/search.go index 452ea2c3..39a7d11d 100644 --- a/pkg/plugin/implementation/AgricultureFacility/search.go +++ b/pkg/plugin/implementation/AgricultureFacility/search.go @@ -29,8 +29,7 @@ import ( "encoding/json" "fmt" "slices" - - "github.com/google/uuid" + "strings" "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/model" @@ -131,7 +130,7 @@ func (s *Step) Run(ctx *model.StepContext) error { // the fresh message id each part carries is what stops POCRA blending the // answers, and a search of one still wants its own id rather than the // caller's. See splitByType. - parts, err := splitByType(ctx.Body, types) + parts, err := splitByType(ctx.Body, types, s.facilityTypesAt) if err != nil { return err } @@ -163,7 +162,7 @@ func (s *Step) Run(ctx *model.StepContext) error { return err } - merged, err := mergeAnswers(answers, messageIDOf(beckn)) + merged, err := mergeAnswers(answers) if err != nil { return err } @@ -218,7 +217,11 @@ func (s *Step) mine(body []byte) bool { // Re-decoded per part rather than deep-copied: the parts are mutated // independently and a shared nested map would have them overwrite each other's // type. A payload is a few kilobytes and this happens once per request. -func splitByType(body []byte, types []string) ([][]byte, error) { +func splitByType(body []byte, types []string, path string) ([][]byte, error) { + if path == "" { + path = DefaultFacilityTypesAt + } + parts := make([][]byte, 0, len(types)) for _, facilityType := range types { var part map[string]any @@ -227,44 +230,19 @@ func splitByType(body []byte, types []string) ([][]byte, error) { "agriculture facility: the payload is not a JSON object: %w", err)) } - attributes, ok := dig(part, "message", "contract", "commitments").([]any) - if !ok || len(attributes) == 0 { - return nil, model.NewBadReqErr("", fmt.Errorf( - "agriculture facility: the payload carries no commitment to split")) - } - commitment, ok := attributes[0].(map[string]any) - if !ok { - return nil, model.NewBadReqErr("", fmt.Errorf( - "agriculture facility: the payload's first commitment is %T, not an object", attributes[0])) - } - resources, ok := commitment["resources"].([]any) - if !ok || len(resources) == 0 { - return nil, model.NewBadReqErr("", fmt.Errorf( - "agriculture facility: the payload's commitment carries no resource to split")) - } - resource, ok := resources[0].(map[string]any) - if !ok { - return nil, model.NewBadReqErr("", fmt.Errorf( - "agriculture facility: the payload's first resource is %T, not an object", resources[0])) - } - resourceAttributes, ok := resource["resourceAttributes"].(map[string]any) - if !ok { - return nil, model.NewBadReqErr("", fmt.Errorf( - "agriculture facility: the payload's first resource carries no resourceAttributes")) - } - // A list of one, not a bare string: the required: checks and the // request half both read this through [] and a scalar would work, but // a part that does not look like the payload it came from is a trap // for whoever reads one in a log. - resourceAttributes["supportedFacilityTypes"] = []any{facilityType} - - becknContext, ok := part["context"].(map[string]any) - if !ok { - return nil, model.NewBadReqErr("", fmt.Errorf( - "agriculture facility: the payload carries no context to stamp a message id on")) + // + // Written at the SAME path the types were read from, so the two cannot + // disagree about where they live. No message id is stamped here: the + // request half derives POCRA's own from the caller's id and this + // part's type, which leaves the caller's id on the part where a Beckn + // answer wants it. + if err := setAt(part, path, []any{facilityType}); err != nil { + return nil, err } - becknContext["messageId"] = uuid.NewString() encoded, err := json.Marshal(part) if err != nil { @@ -295,7 +273,7 @@ func splitByType(body []byte, types []string) ([][]byte, error) { // attribute and the mapping drops it before this sees the answers. A globally // ranked multi-type search would need the mapping to publish a distance this // could sort on, which the schema pack says it must not. -func mergeAnswers(answers [][]byte, callerMessageID string) ([]byte, error) { +func mergeAnswers(answers [][]byte) ([]byte, error) { if len(answers) == 0 { return nil, fmt.Errorf("agriculture facility: nothing to merge; this search made no calls") } @@ -353,10 +331,6 @@ func mergeAnswers(answers [][]byte, callerMessageID string) ([]byte, error) { offer["resourceIds"] = ids } - if becknContext, ok := merged["context"].(map[string]any); ok && callerMessageID != "" { - becknContext["messageId"] = callerMessageID - } - encoded, err := json.Marshal(merged) if err != nil { return nil, fmt.Errorf("agriculture facility: could not encode the merged answer: %w", err) @@ -366,28 +340,11 @@ func mergeAnswers(answers [][]byte, callerMessageID string) ([]byte, error) { // commitmentOf reaches the one commitment an answer carries. func commitmentOf(document map[string]any) (map[string]any, error) { - commitments, _ := dig(document, "message", "contract", "commitments").([]any) + commitments := containersAt(document, strings.Split(CommitmentsAt, ".")) if len(commitments) == 0 { return nil, fmt.Errorf("agriculture facility: an answer carries no commitment to merge") } - commitment, ok := commitments[0].(map[string]any) - if !ok { - return nil, fmt.Errorf("agriculture facility: an answer's commitment is %T, not an object", commitments[0]) - } - return commitment, nil -} - -// messageIDOf reads the caller's own message id, for the merged answer to -// correlate with. Absent is not an error: the mapping copies whatever the -// payload carried, so an absent one stays absent, as it did before splitting -// existed. -func messageIDOf(beckn any) string { - document, ok := beckn.(map[string]any) - if !ok { - return "" - } - id, _ := dig(document, "context", "messageId").(string) - return id + return commitments[0], nil } // searchConcurrency resolves the deployment's setting, clamped to From 1cb385e8c47c9e259ff70f4945a63b0923163d19 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Thu, 10 Sep 2026 02:07:18 +0530 Subject: [PATCH 126/130] refactor(AgricultureFacility): bound the fan-out with errgroup, not a package [#91] internal/concurrent was 116 lines of generic bounded fail-fast machinery plus 288 lines of tests to run one loop, with a single caller, at a configured concurrency of 1. golang.org/x/sync is already a dependency and errgroup does the same job here in a method that reads alongside the code it serves. The clamp moves into searchConcurrency and gains a reason: SetLimit reads a non-positive limit as UNBOUNDED, so a misread setting would fan out over every facility type at once against a provider that rate-limits into silent empty answers. concurrent.Bound treated zero the same way; the comment now says why it matters at the point where it is enforced. Order is kept by indexing the answers rather than appending, so each slot is written by exactly one goroutine and the merge does not depend on which call finished first. No behaviour change: the five search tests that pin sequential-by-default, the configured limit, fail-fast skipping calls not yet issued, one-type-one-call and the ceiling all pass unmodified, under -race. --- .../AgricultureFacility/search.go | 90 ++++-- .../internal/concurrent/concurrent.go | 116 ------- .../internal/concurrent/concurrent_test.go | 288 ------------------ 3 files changed, 58 insertions(+), 436 deletions(-) delete mode 100644 pkg/plugin/implementation/internal/concurrent/concurrent.go delete mode 100644 pkg/plugin/implementation/internal/concurrent/concurrent_test.go diff --git a/pkg/plugin/implementation/AgricultureFacility/search.go b/pkg/plugin/implementation/AgricultureFacility/search.go index 39a7d11d..3cfbea88 100644 --- a/pkg/plugin/implementation/AgricultureFacility/search.go +++ b/pkg/plugin/implementation/AgricultureFacility/search.go @@ -11,12 +11,11 @@ // one-payload-one-call step over each of them concurrently, and merge the // answers back into one. // -// Nothing below this file knows any of that. internal/upstream serves one -// payload with one call and has no notion of splitting; jsonmapper compiles -// the two halves every mapping has and no third thing; internal/concurrent -// runs N of anything, bounded and ordered, and has never heard of Beckn. The -// mapping this runs is written for a single-type payload, which is what it is -// always handed. +// Nothing below this file knows any of that. internal/common serves one +// payload with one call and has no notion of splitting, and jsonmapper +// compiles the two halves every mapping has and no third thing. The mapping +// this runs is written for a single-type payload, which is what it is always +// handed. // // Separated from the package clause by a blank line on purpose: the package's // own doc comment is in AgricultureFacility.go, and this is a note about one @@ -25,17 +24,17 @@ package AgricultureFacility import ( - "context" "encoding/json" "fmt" "slices" "strings" + "golang.org/x/sync/errgroup" + "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/model" "github.com/beckn-one/beckn-onix/pkg/plugin/definition" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common" - "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/concurrent" ) const ( @@ -138,26 +137,7 @@ func (s *Step) Run(ctx *model.StepContext) error { log.Debugf(ctx, "agriculture facility: serving %d facility type(s) as %d call(s), %d at a time", len(types), len(parts), s.concurrency) - // Bounded, ordered and fail-fast, which is internal/concurrent's whole - // job. One failure fails the request: a partial answer is the defect this - // was built to fix wearing a different hat -- the caller asked for four - // facility types, would receive three, and nothing in the payload would - // say that the fourth was asked for and lost. - answers, err := concurrent.Map(ctx, parts, s.concurrency, - func(callCtx context.Context, part []byte) ([]byte, error) { - // Its own StepContext: inner reads Body and writes ResponseBody, - // so the parts must not share either. callCtx rather than ctx so a - // sibling's failure cancels this call too. - partCtx := *ctx - partCtx.Context = callCtx - partCtx.Body = part - partCtx.ResponseBody = nil - - if err := s.runPart(&partCtx); err != nil { - return nil, err - } - return partCtx.ResponseBody, nil - }) + answers, err := s.callEach(ctx, parts) if err != nil { return err } @@ -177,6 +157,46 @@ func (s *Step) Run(ctx *model.StepContext) error { // recognised can only be a mismatch between this step's binding keys and // inner's. Reported rather than merged, because merging nothing would answer // a four-type search with three types and no error. +// callEach runs one upstream call per part, at most s.concurrency at a time, +// and returns the answers in the parts' order. +// +// Fail-fast, and that is the point: one failure fails the whole request. A +// partial answer is the defect this split was built to fix wearing a different +// hat -- the caller asked for four facility types, would receive three, and +// nothing in the payload would say the fourth was asked for and lost. +func (s *Step) callEach(ctx *model.StepContext, parts [][]byte) ([][]byte, error) { + // Indexed rather than appended, so the answers keep the parts' order + // whichever call finishes first. Each slot is written by exactly one + // goroutine, which is what makes this safe without a lock. + answers := make([][]byte, len(parts)) + + group, callCtx := errgroup.WithContext(ctx.Context) + group.SetLimit(s.concurrency) + + for index, part := range parts { + group.Go(func() error { + // Its own StepContext: inner reads Body and writes ResponseBody, + // so the parts must not share either. callCtx rather than + // ctx.Context so a sibling's failure cancels this call too. + partCtx := *ctx + partCtx.Context = callCtx + partCtx.Body = part + partCtx.ResponseBody = nil + + if err := s.runPart(&partCtx); err != nil { + return err + } + answers[index] = partCtx.ResponseBody + return nil + }) + } + + if err := group.Wait(); err != nil { + return nil, err + } + return answers, nil +} + func (s *Step) runPart(partCtx *model.StepContext) error { if err := s.inner.Run(partCtx); err != nil { return err @@ -351,9 +371,15 @@ func commitmentOf(document map[string]any) (map[string]any, error) { // MaxFacilityTypes and defaulting to DefaultSearchConcurrency when left unset // or non-positive. // -// The arithmetic is internal/concurrent's, because every caller of it needs -// the same clamp and a zero limit there means UNBOUNDED. The three numbers are -// this package's, because each one is a fact about POCRA. +// The clamp is not decoration: errgroup.SetLimit takes a non-positive limit to +// mean UNBOUNDED, so a misread setting would fan out over every facility type +// at once against a provider that rate-limits into silent empty answers. func searchConcurrency(configured int) int { - return concurrent.Bound(configured, DefaultSearchConcurrency, MaxFacilityTypes) + if configured <= 0 { + return DefaultSearchConcurrency + } + if configured > MaxFacilityTypes { + return MaxFacilityTypes + } + return configured } diff --git a/pkg/plugin/implementation/internal/concurrent/concurrent.go b/pkg/plugin/implementation/internal/concurrent/concurrent.go deleted file mode 100644 index 7bd9b231..00000000 --- a/pkg/plugin/implementation/internal/concurrent/concurrent.go +++ /dev/null @@ -1,116 +0,0 @@ -// Package concurrent is one generic engine: run a known number of -// independent calls, bounded, with the first failure cancelling the rest. -// -// It knows nothing about Beckn, upstream providers, or any specific -// capability. Every policy question -- how many is too many, what identity -// each call carries, what a failure should say -- belongs to whoever calls -// this, not to this package: see -// pkg/plugin/implementation/AgricultureFacility/search.go for the one caller -// that exists today. -// -// pkg/plugin/implementation/catalogpublisher's own runConcurrent looks -// similar but is NOT this: it collects every call's error rather than -// cancelling on the first one, because its caller needs to inspect all of -// them to decide whether any is worth retrying. That is a different -// concurrency shape, kept separate rather than forced into one function -// with a mode flag. -package concurrent - -import ( - "context" - "fmt" - - "golang.org/x/sync/errgroup" -) - -// Run calls work(ctx, i) for every i in [0, n), at most limit at a time, and -// returns each call's result at its own index -- in the order Run was asked -// for them, not the order they finished, so a caller can rely on results[i] -// meaning "the i-th thing I asked for" regardless of which one answered -// first. -// -// The first error any call returns cancels the ctx handed to every other -// call, in flight or not yet started: a call not yet issued when an earlier -// one already failed is skipped rather than made and discarded, and an -// in-flight call that itself respects ctx cancellation (as an -// http.NewRequestWithContext-built request does) is aborted rather than run -// to completion for an answer nobody will use. Run itself returns that first -// error, wrapped by nothing -- whatever work returned is what the caller -// gets, so the caller's own wrapping (naming which of the n things failed, -// what identity it carried) survives unchanged. What each call's own input or -// identity looks like is entirely work's business -- Run passes only an -// index, so it never has to know or care what a caller maps that index onto. -// -// n == 0 calls work zero times and returns (nil, nil). limit is passed -// straight to errgroup.Group.SetLimit -- see its own doc for what a -// non-positive value means. -func Run[T any](ctx context.Context, n, limit int, work func(ctx context.Context, index int) (T, error)) ([]T, error) { - if n == 0 { - return nil, nil - } - - group, groupCtx := errgroup.WithContext(ctx) - group.SetLimit(limit) - - results := make([]T, n) - for index := range n { - group.Go(func() error { - if groupCtx.Err() != nil { - return nil - } - result, err := work(groupCtx, index) - if err != nil { - return fmt.Errorf("%w", err) - } - results[index] = result - return nil - }) - } - - if err := group.Wait(); err != nil { - return nil, err - } - return results, nil -} - -// Map is Run over a list of VALUES rather than a count: one call per value, -// results in the order the values were given. -// -// The same bounded, ordered, fail-fast guarantees as Run -- it is Run, with -// the index-to-value step done here instead of in every caller. Prefer it -// whenever the work is "one call per thing in this list". -// -// values is []V and results are []T, so a caller keeps whatever types it -// already has; nothing here inspects either. -// -// What Map does NOT decide, on purpose: how many values are too many, and -// what to tell a caller who asked for too many. A ceiling is a fact about -// whatever is being called and its refusal is that domain's error to write, -// so it belongs to the package that knows the domain. Use Bound below for the -// other half of the arithmetic, defaulting and clamping a configured -// concurrency. -func Map[V, T any](ctx context.Context, values []V, limit int, - call func(ctx context.Context, value V) (T, error)) ([]T, error) { - - return Run(ctx, len(values), limit, func(ctx context.Context, index int) (T, error) { - return call(ctx, values[index]) - }) -} - -// Bound resolves a configured concurrency: fallback when it is unset or -// non-positive, ceiling when it is over. -// -// Here rather than in each caller because every caller of Run or Map needs -// the same three lines, and getting them wrong is quiet: a zero limit means -// UNBOUNDED in errgroup, so a deployment that leaves the setting out would get -// every call at once from a mistake that reads like a default. -func Bound(configured, fallback, ceiling int) int { - limit := configured - if limit <= 0 { - limit = fallback - } - if limit > ceiling { - limit = ceiling - } - return limit -} diff --git a/pkg/plugin/implementation/internal/concurrent/concurrent_test.go b/pkg/plugin/implementation/internal/concurrent/concurrent_test.go deleted file mode 100644 index a35d4c4f..00000000 --- a/pkg/plugin/implementation/internal/concurrent/concurrent_test.go +++ /dev/null @@ -1,288 +0,0 @@ -package concurrent - -import ( - "context" - "errors" - "fmt" - "sync" - "sync/atomic" - "testing" - "time" -) - -func TestRunReturnsResultsInOrderNotArrivalOrder(t *testing.T) { - t.Parallel() - - // index 0 is the slowest, so an arrival-ordered result would put it - // last -- this proves the result is indexed, not appended. - delays := []time.Duration{30 * time.Millisecond, 10 * time.Millisecond, 0} - results, err := Run(context.Background(), len(delays), len(delays), - func(ctx context.Context, i int) (int, error) { - time.Sleep(delays[i]) - return i, nil - }) - if err != nil { - t.Fatalf("Run() returned an unexpected error: %v", err) - } - for i, got := range results { - if got != i { - t.Errorf("results[%d] = %d, want %d", i, got, i) - } - } -} - -func TestRunIsSequentialWhenLimitIsOne(t *testing.T) { - t.Parallel() - - var mu sync.Mutex - var inFlight, peak int - _, err := Run(context.Background(), 4, 1, func(ctx context.Context, i int) (struct{}, error) { - mu.Lock() - inFlight++ - if inFlight > peak { - peak = inFlight - } - mu.Unlock() - - time.Sleep(10 * time.Millisecond) - - mu.Lock() - inFlight-- - mu.Unlock() - return struct{}{}, nil - }) - if err != nil { - t.Fatalf("Run() returned an unexpected error: %v", err) - } - if peak != 1 { - t.Errorf("peak concurrency = %d, want 1", peak) - } -} - -func TestRunHonoursALimitAboveOne(t *testing.T) { - t.Parallel() - - var mu sync.Mutex - var inFlight, peak int - _, err := Run(context.Background(), 6, 2, func(ctx context.Context, i int) (struct{}, error) { - mu.Lock() - inFlight++ - if inFlight > peak { - peak = inFlight - } - mu.Unlock() - - time.Sleep(15 * time.Millisecond) - - mu.Lock() - inFlight-- - mu.Unlock() - return struct{}{}, nil - }) - if err != nil { - t.Fatalf("Run() returned an unexpected error: %v", err) - } - if peak != 2 { - t.Errorf("peak concurrency = %d, want the configured 2", peak) - } -} - -func TestRunReturnsTheCallsOwnError(t *testing.T) { - t.Parallel() - - sentinel := errors.New("boom") - _, err := Run(context.Background(), 3, 3, func(ctx context.Context, i int) (int, error) { - if i == 1 { - return 0, fmt.Errorf("call %d failed: %w", i, sentinel) - } - return i, nil - }) - if err == nil || !errors.Is(err, sentinel) { - t.Fatalf("error = %v, want it to wrap the call's own error", err) - } -} - -func TestRunSkipsCallsNotYetIssuedAfterAFailureWhenSequential(t *testing.T) { - t.Parallel() - - var called int32 - _, err := Run(context.Background(), 3, 1, func(ctx context.Context, i int) (struct{}, error) { - atomic.AddInt32(&called, 1) - if i == 0 { - return struct{}{}, errors.New("first call fails") - } - return struct{}{}, nil - }) - if err == nil { - t.Fatal("Run() returned no error, want the first call's failure") - } - // Sequential (limit 1): index 0 fails and cancels the shared context - // before index 1's call is ever issued, so only index 0 was called. - if got := atomic.LoadInt32(&called); got != 1 { - t.Errorf("work was called %d times, want 1 -- calls not yet issued "+ - "when an earlier one fails must be skipped, not made and discarded", got) - } -} - -func TestRunOfZeroReturnsNoResultsAndNoError(t *testing.T) { - t.Parallel() - - results, err := Run(context.Background(), 0, 4, func(ctx context.Context, i int) (int, error) { - t.Error("work was called for n == 0") - return 0, nil - }) - if err != nil { - t.Fatalf("Run() returned an unexpected error: %v", err) - } - if len(results) != 0 { - t.Errorf("results = %v, want empty", results) - } -} - -func TestRunOfOneRunsExactlyOnce(t *testing.T) { - t.Parallel() - - var called int32 - results, err := Run(context.Background(), 1, 1, func(ctx context.Context, i int) (string, error) { - atomic.AddInt32(&called, 1) - return "ok", nil - }) - if err != nil { - t.Fatalf("Run() returned an unexpected error: %v", err) - } - if got := atomic.LoadInt32(&called); got != 1 { - t.Errorf("work was called %d times, want 1", got) - } - if len(results) != 1 || results[0] != "ok" { - t.Errorf("results = %v, want [\"ok\"]", results) - } -} - -func TestRunCancelsTheContextItHandsToInFlightWork(t *testing.T) { - t.Parallel() - - var slowCtxDone int32 - // inFlight is closed by index 1 once it is genuinely running, and index 0 - // waits for that before failing. Without the handshake this test is a - // race: index 1's call may not have started when index 0 fails, and Run - // then SKIPS it rather than cancelling it -- correct behaviour, and what - // TestRunSkipsCallsNotYetIssuedAfterAFailureWhenSequential is for, but - // not what this test is about. - inFlight := make(chan struct{}) - _, err := Run(context.Background(), 2, 2, func(ctx context.Context, i int) (struct{}, error) { - if i == 0 { - <-inFlight - return struct{}{}, errors.New("fails once the other call is in flight") - } - close(inFlight) - // In flight when index 0 fails: wait for our own ctx to be cancelled, - // or give up after a bound that would mean it never was. A bound - // rather than a bare receive so a broken Run fails this test instead - // of hanging it. - select { - case <-ctx.Done(): - atomic.AddInt32(&slowCtxDone, 1) - case <-time.After(2 * time.Second): - } - return struct{}{}, nil - }) - if err == nil { - t.Fatal("Run() returned no error, want index 0's failure") - } - if atomic.LoadInt32(&slowCtxDone) != 1 { - t.Error("the in-flight call's context was not cancelled after the other call failed") - } -} - -// --- Map -------------------------------------------------------------------- - -// Map hands each value to its own call and returns the answers in the order -// the values were given, whatever order the calls finished in. -func TestMapCallsOncePerValueInOrder(t *testing.T) { - t.Parallel() - - values := []string{"kvk", "warehouse", "soil_lab"} - results, err := Map(context.Background(), values, 3, func(ctx context.Context, value string) (string, error) { - // The first value sleeps longest, so arrival order is the reverse of - // the order asked for. A caller relying on results[i] meaning "the - // answer for values[i]" has to hold regardless. - if value == "kvk" { - time.Sleep(40 * time.Millisecond) - } - return "answered:" + value, nil - }) - if err != nil { - t.Fatalf("Map() returned an unexpected error: %v", err) - } - want := []string{"answered:kvk", "answered:warehouse", "answered:soil_lab"} - if len(results) != len(want) { - t.Fatalf("results = %v, want %v", results, want) - } - for index := range want { - if results[index] != want[index] { - t.Errorf("results[%d] = %q, want %q -- value order, not arrival order", - index, results[index], want[index]) - } - } -} - -// An empty list makes no calls and is not an error: a caller deciding that an -// empty list is a bad request says so itself, in its own words. -func TestMapOfNoValuesMakesNoCalls(t *testing.T) { - t.Parallel() - - results, err := Map(context.Background(), nil, 2, func(ctx context.Context, value string) (int, error) { - t.Error("a call was made for an empty value list") - return 0, nil - }) - if err != nil { - t.Fatalf("Map() returned an unexpected error: %v", err) - } - if len(results) != 0 { - t.Errorf("results = %v, want empty", results) - } -} - -// One value's failure fails the whole call, and the value's own error is what -// comes back -- the caller named which value it was and why. -func TestMapFailsWhenOneValueFails(t *testing.T) { - t.Parallel() - - sentinel := errors.New("warehouse is unreachable") - _, err := Map(context.Background(), []string{"kvk", "warehouse"}, 1, - func(ctx context.Context, value string) (string, error) { - if value == "warehouse" { - return "", sentinel - } - return "ok", nil - }) - if !errors.Is(err, sentinel) { - t.Errorf("error = %v, want the failing value's own error", err) - } -} - -// --- Bound ------------------------------------------------------------------ - -// Bound is the arithmetic every caller of Run and Map needs: a configured -// limit, a fallback when it is unset, a ceiling when it is over. Zero must -// become the fallback and not reach errgroup, where it means UNBOUNDED. -func TestBoundDefaultsAndClamps(t *testing.T) { - t.Parallel() - - for _, tc := range []struct { - name string - configured, fallback, ceiling, want int - }{ - {"unset takes the fallback", 0, 1, 8, 1}, - {"negative takes the fallback", -4, 2, 8, 2}, - {"within range is honoured", 4, 1, 8, 4}, - {"exactly the ceiling is honoured", 8, 1, 8, 8}, - {"over the ceiling is clamped", 64, 1, 8, 8}, - {"a fallback over the ceiling is clamped too", 0, 16, 8, 8}, - } { - if got := Bound(tc.configured, tc.fallback, tc.ceiling); got != tc.want { - t.Errorf("%s: Bound(%d, %d, %d) = %d, want %d", - tc.name, tc.configured, tc.fallback, tc.ceiling, got, tc.want) - } - } -} From 803d81c6d81e51b42357ad6edbb6282ed3618235 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Thu, 10 Sep 2026 02:07:34 +0530 Subject: [PATCH 127/130] chore: stop ignoring /docs/, which would hide real documentation [#91] Both lines were added by the facility plugin's branch and ignore nothing: no docs/ or dev_docs/ directory exists and neither has a tracked file. /docs/ is worse than unnecessary -- it is the conventional place for documentation, so the first person to add any there would find it silently uncommittable, with a rule in an unrelated feature branch to explain it. testdata/.gitignore stays. It ignores schema-cache/, which does exist, and is how schemacache_test.go keeps the fetched schema pack out of the repo instead of vendoring a copy. --- .gitignore | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 1134973c..6292bb58 100644 --- a/.gitignore +++ b/.gitignore @@ -176,8 +176,4 @@ test_request.json .claude # catalogPublish handler's local output root (config/local-beckn-one-bap.yaml) -/catalog/ - -# Local, non-shipped developer notes (review summaries, scratch docs) -/dev_docs/ -/docs/ \ No newline at end of file +/catalog/ \ No newline at end of file From 99d08740e300aa5024a1b0d9b3b9e05c6db4f401 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Thu, 10 Sep 2026 11:13:32 +0530 Subject: [PATCH 128/130] chore: add __MACOSX/ to .gitignore to prevent macOS system files from being tracked --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6292bb58..0e87cdd2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Logs .DS_Store +__MACOSX/ logs .idea *.log From 233793ff8ea8d5e4aea99f5e0cba9b1bdfc7de25 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Thu, 10 Sep 2026 12:07:46 +0530 Subject: [PATCH 129/130] chore: add /catalog/, /docs/, and /dev_docs/ to .gitignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0e87cdd2..f91664b1 100644 --- a/.gitignore +++ b/.gitignore @@ -177,4 +177,6 @@ test_request.json .claude # catalogPublish handler's local output root (config/local-beckn-one-bap.yaml) -/catalog/ \ No newline at end of file +/catalog/ +/docs/ +/dev_docs/ \ No newline at end of file From 7b056ad44340d1fbef0d1559f548f332db03b2e3 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Thu, 10 Sep 2026 12:32:47 +0530 Subject: [PATCH 130/130] feat(common): add a tokenQuery auth scheme for providers that issue their own [#91] For an upstream that mints short-lived tokens from its own endpoint rather than an OAuth2 one. The live Agmarknet Vistaar API is the case: POST a JSON body of credentials, get {"token":""} back, send it as a QUERY PARAMETER. Neither existing scheme can serve that, and bending one would have been wrong in five separate ways: oauth2 posts form-encoded client_id/client_secret, reads access_token and expires_in, sets Authorization: Bearer tokenQuery posts JSON under CONFIGURED keys, reads a CONFIGURED key, sets a query parameter -- and the response carries NO expiry The field names are configured rather than fixed because access_name and password are one provider's spelling, not a standard. WHY tokenTtl IS REQUIRED. The response says nothing about how long the token lives, so there is nothing to read and guessing would be inventing a lifetime. An operator states what they believe instead. A ttl at or below the refresh skew is refused: every token would already be expired on arrival, making two round trips per request and hitting the token endpoint at the request rate. AND WHY A WRONG tokenTtl IS NOT AN OUTAGE. Because the lifetime is an estimate, a too-generous one leaves a dead token cached -- and without help every call would fail until it lapsed. A 401 or 403 from the provider now drops the held token, so the next call exchanges a fresh one. One request pays; the next recovers. This applies to oauth2 too, where an issuer can revoke a token before its expires_in runs out. REDACTION follows the placement, not the scheme it borrows from. A query string is the exposed spot -- proxies log it and Go's transport errors quote the whole URL -- so the token's URL-ESCAPED form is covered as well, or one containing + or = survives redaction. The identifier is left readable, as a client id is: it identifies, it does not authenticate. The HTTP half of the exchange is now shared by both schemes in postForToken. The interesting part there is the retry decision -- unreachable, unreadable, 5xx and 429 retry; a 4xx is configuration -- and two copies of that would drift. VERIFIED AGAINST THE LIVE PROVIDER, not just in unit tests: exchanged a real 36-character UUID from the actual token endpoint, placed it as ?token=, and confirmed the log line reads token=REDACTED. That check read the endpoint and credentials from the environment and is not committed -- no IP and no credential is written down anywhere. Twenty test cases, and the self-heal one carries a negative control: with the forgetToken call removed it fails with "the rejected token was reused instead of re-exchanged", so it is testing the behaviour rather than passing by accident. Full suite green under -race. --- .../implementation/internal/common/auth.go | 84 +++- .../implementation/internal/common/http.go | 13 + .../implementation/internal/common/redact.go | 19 + .../implementation/internal/common/token.go | 156 ++++++-- .../internal/common/tokenquery_test.go | 366 ++++++++++++++++++ .../internal/common/util/constant.go | 26 +- 6 files changed, 626 insertions(+), 38 deletions(-) create mode 100644 pkg/plugin/implementation/internal/common/tokenquery_test.go diff --git a/pkg/plugin/implementation/internal/common/auth.go b/pkg/plugin/implementation/internal/common/auth.go index e53eb4e6..4c780112 100644 --- a/pkg/plugin/implementation/internal/common/auth.go +++ b/pkg/plugin/implementation/internal/common/auth.go @@ -11,6 +11,7 @@ import ( "strings" "sync" "sync/atomic" + "time" "github.com/beckn-one/beckn-onix/pkg/log" "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/util" @@ -26,7 +27,7 @@ type AuthProfile struct { // operator guessing which block to look at. Provider string - // One of none, basic, header, query or oauth2. + // One of none, basic, header, query, oauth2 or tokenQuery. Scheme string // Variable NAMES, never the values. @@ -47,6 +48,30 @@ type AuthProfile struct { // Variable names. The values never appear in config, the registry or a log. ClientIDEnv string ClientSecretEnv string + + // tokenQuery only. The JSON keys the token endpoint expects, and the + // variables holding what goes under them. + // + // The KEYS are configured because they are one provider's spelling rather + // than a standard: the live one this was written against wants + // access_name and password, and the next will want something else. + TokenUserField string + TokenUserEnv string + TokenSecretName string + TokenSecretEnv string + + // Where the token sits in the response body, as a top-level key. + TokenResponseField string + + // How long a token may be held, as written in config and as parsed. + // + // Configured because the response carries no expiry at all -- nothing in it + // says how long the token lives, so an operator states what they believe + // and this refuses to guess. Set it too long and calls fail with the + // provider's own rejection until it lapses; the cache is dropped on such a + // rejection so the next call re-exchanges rather than waiting that out. + TokenTTLRaw string + TokenTTL time.Duration } // authenticator is one provider's profile plus the token it holds. @@ -91,13 +116,43 @@ func (a *AuthProfile) validate() error { "%s: authScheme oauth2 requires tokenUrl, clientIdEnv and clientSecretEnv", a.Provider) } + case util.AuthSchemeTokenQuery: + // Every one of these is required, and none can be defaulted: the JSON + // keys, the response key and the parameter name are all this + // provider's spelling, and guessing any of them sends a malformed + // request whose rejection says nothing about the cause. + if a.TokenURL == "" || a.TokenUserField == "" || a.TokenUserEnv == "" || + a.TokenSecretName == "" || a.TokenSecretEnv == "" || + a.TokenResponseField == "" || a.QueryName == "" { + return fmt.Errorf("%s: authScheme tokenQuery requires tokenUrl, "+ + "tokenUserField, tokenUserEnv, tokenSecretField, tokenSecretEnv, "+ + "tokenResponseField and queryName", a.Provider) + } + if a.TokenTTLRaw == "" { + return fmt.Errorf("%s: authScheme tokenQuery requires tokenTtl, "+ + "because the token response carries no expiry to read", a.Provider) + } + ttl, err := time.ParseDuration(a.TokenTTLRaw) + if err != nil { + return fmt.Errorf("%s: tokenTtl %q is not a duration: %w", + a.Provider, a.TokenTTLRaw, err) + } + // Below the skew every token is already expired when it arrives, so + // each call would exchange one and then discard it -- two round trips + // per request and a token endpoint hit at the request rate. + if ttl <= util.TokenRefreshSkew { + return fmt.Errorf("%s: tokenTtl %s must exceed the %s refresh skew, "+ + "or every token is expired before it is used", + a.Provider, ttl, util.TokenRefreshSkew) + } + a.TokenTTL = ttl case "": return fmt.Errorf("%s: authScheme is required, "+ "and is none where the upstream needs no credential", a.Provider) default: return fmt.Errorf( - "%s: unknown authScheme %q: must be none, basic, header, query or oauth2", - a.Provider, a.Scheme) + "%s: unknown authScheme %q: must be none, basic, header, query, "+ + "oauth2 or tokenQuery", a.Provider, a.Scheme) } return nil } @@ -181,6 +236,16 @@ var authFields = map[string]func(*AuthProfile, string){ "tokenUrl": func(p *AuthProfile, v string) { p.TokenURL = v }, "clientIdEnv": func(p *AuthProfile, v string) { p.ClientIDEnv = v }, "clientSecretEnv": func(p *AuthProfile, v string) { p.ClientSecretEnv = v }, + + "tokenUserField": func(p *AuthProfile, v string) { p.TokenUserField = v }, + "tokenUserEnv": func(p *AuthProfile, v string) { p.TokenUserEnv = v }, + "tokenSecretField": func(p *AuthProfile, v string) { p.TokenSecretName = v }, + "tokenSecretEnv": func(p *AuthProfile, v string) { p.TokenSecretEnv = v }, + "tokenResponseField": func(p *AuthProfile, v string) { p.TokenResponseField = v }, + // Kept as written and turned into a duration by validate, so a bad value is + // refused at startup naming the provider rather than read as zero -- which + // would mean re-exchanging a token on every single call. + "tokenTtl": func(p *AuthProfile, v string) { p.TokenTTLRaw = v }, } // providerIDFrom returns the provider half of "|". @@ -236,11 +301,22 @@ func (s *Step) authenticate(auth *authenticator, req *http.Request) error { query.Set(cfg.QueryName, value) req.URL.RawQuery = query.Encode() case util.AuthSchemeOAuth2: - token, err := s.bearerToken(req.Context(), auth) + token, err := s.providerToken(req.Context(), auth) if err != nil { return err } req.Header.Set("Authorization", "Bearer "+token) + + case util.AuthSchemeTokenQuery: + // Exchanged like oauth2, placed like query: the same held token, put + // where this provider reads it from. + token, err := s.providerToken(req.Context(), auth) + if err != nil { + return err + } + query := req.URL.Query() + query.Set(cfg.QueryName, token) + req.URL.RawQuery = query.Encode() } return nil } diff --git a/pkg/plugin/implementation/internal/common/http.go b/pkg/plugin/implementation/internal/common/http.go index 56304098..d558ca00 100644 --- a/pkg/plugin/implementation/internal/common/http.go +++ b/pkg/plugin/implementation/internal/common/http.go @@ -116,6 +116,19 @@ func (s *Step) attempt(ctx context.Context, auth *authenticator, call model.Acti // quoted back, credential and all. log.Warnf(ctx, "provider returned %s for %s %s: %s", resp.Status, method, requested, s.redactString(util.Explain(body))) + + // A held token the provider has stopped accepting is dropped, so the + // next call exchanges a fresh one. + // + // This is what keeps a wrong tokenTtl from being an outage. The + // lifetime is an operator's ESTIMATE -- the endpoint states no expiry + // -- so a too-generous one leaves a dead token cached, and without this + // every call would fail until it lapsed. One request pays; the next + // recovers. + if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { + s.forgetToken(auth) + } + err := fmt.Errorf("provider returned %s", resp.Status) // 5xx and 429 ask to be tried again. Every other 4xx is a statement // about the request, which will not improve. diff --git a/pkg/plugin/implementation/internal/common/redact.go b/pkg/plugin/implementation/internal/common/redact.go index 155c2d28..6d59711f 100644 --- a/pkg/plugin/implementation/internal/common/redact.go +++ b/pkg/plugin/implementation/internal/common/redact.go @@ -129,6 +129,25 @@ func (a *authenticator) secretForms() []string { forms = append(forms, encoded) } return forms + case util.AuthSchemeTokenQuery: + // oauth2's two halves, with query's escaping: the secret goes to the + // token endpoint, and the token it returns travels in a QUERY STRING -- + // which is the exposed placement, so its escaped form has to be + // covered too or a token containing "+" or "=" survives redaction. + var forms []string + if secret := os.Getenv(a.cfg.TokenSecretEnv); secret != "" { + forms = append(forms, secret) + } + // Read without tokenMu: reached from inside the exchange, which holds it. + if held := a.token.Load(); held != nil && held.value != "" { + forms = append(forms, held.value) + if encoded := url.QueryEscape(held.value); encoded != held.value { + forms = append(forms, encoded) + } + } + // tokenUserField's value is NOT redacted, for the same reason a client + // id is not: it identifies, it does not authenticate. + return forms } return nil } diff --git a/pkg/plugin/implementation/internal/common/token.go b/pkg/plugin/implementation/internal/common/token.go index 93ded54b..093cafa0 100644 --- a/pkg/plugin/implementation/internal/common/token.go +++ b/pkg/plugin/implementation/internal/common/token.go @@ -1,4 +1,5 @@ -// The oauth2 client_credentials exchange and the token each provider holds. +// The token exchanges -- oauth2 client_credentials and tokenQuery -- and the +// token each provider holds. package common import ( @@ -17,16 +18,21 @@ import ( "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/util" ) -// bearerToken returns a token to send, exchanging one if the held token has +// providerToken returns a token to send, exchanging one if the held token has // expired or there is none. // +// Scheme-neutral on purpose: it holds a value and an expiry and knows nothing +// about where the token came from or where it is going. oauth2 sends the +// result in a header, tokenQuery in a query parameter, and both want exactly +// this caching. +// // The lock spans the fetch on purpose: without it a cold start sends every // concurrent request to the issuer at once. It costs one wait per token // lifetime. // // NOTHING IS CACHED ON FAILURE, so a brief outage at the token endpoint does // not leave this step holding a failure for a lifetime it never obtained. -func (s *Step) bearerToken(ctx context.Context, auth *authenticator) (string, error) { +func (s *Step) providerToken(ctx context.Context, auth *authenticator) (string, error) { if held := auth.token.Load(); held != nil && time.Now().Before(held.expiry) { return held.value, nil } @@ -53,18 +59,91 @@ type cachedToken struct { expiry time.Time } +// forgetToken drops a held token, so the next call exchanges a fresh one. +// +// Called when the provider itself rejects the credential, which is the only +// authority on whether a token is still good: an expiry held here is a claim +// about the future -- read from expires_in, or estimated by an operator in +// tokenTtl -- and the provider can disagree with it at any time. +// +// A no-op for a scheme that holds nothing, so the caller does not have to ask +// which scheme it is. +func (s *Step) forgetToken(auth *authenticator) { + if auth == nil { + return + } + auth.token.Store(nil) +} + // tokenResponse is the part of an OAuth2 token response this step reads. type tokenResponse struct { AccessToken string `json:"access_token"` ExpiresIn int `json:"expires_in"` } -// exchangeToken performs the client_credentials grant. +// exchangeToken obtains a token, by whichever exchange the scheme names. // // Every failure carries 502, because it is the exchange with the provider's // issuer that failed. Unclassified they would surface as 500, telling a peer // this adapter broke when it did not. func (s *Step) exchangeToken(ctx context.Context, auth *authenticator) (string, time.Duration, error) { + if auth.cfg.Scheme == util.AuthSchemeTokenQuery { + return s.exchangeQueryToken(ctx, auth) + } + return s.exchangeOAuth2Token(ctx, auth) +} + +// exchangeQueryToken POSTs a JSON body of configured field names and reads the +// token out of a configured response field. +// +// The lifetime comes from config, not from the response: this endpoint says +// nothing about how long its token lives. That is the whole reason tokenTtl is +// required rather than defaulted -- see AuthProfile. +func (s *Step) exchangeQueryToken(ctx context.Context, auth *authenticator) (string, time.Duration, error) { + cfg := auth.cfg + user, secret := os.Getenv(cfg.TokenUserEnv), os.Getenv(cfg.TokenSecretEnv) + if user == "" || secret == "" { + return "", 0, s.missingCredential(ctx, cfg.Provider, util.AuthSchemeTokenQuery, + cfg.TokenUserEnv+" and "+cfg.TokenSecretEnv) + } + + // Marshalled rather than concatenated, so a credential containing a quote + // or a backslash cannot break out of the JSON it travels in. + payload, err := json.Marshal(map[string]string{ + cfg.TokenUserField: user, + cfg.TokenSecretName: secret, + }) + if err != nil { + return "", 0, s.permanentTokenErr( + fmt.Errorf("token request body could not be built: %w", err)) + } + + body, err := s.postForToken(ctx, cfg, payload, "application/json") + if err != nil { + return "", 0, err + } + + // Decoded into a map because the field name is configured: a struct tag + // cannot be written for a key that is not known until config is read. + var fields map[string]any + if err := json.Unmarshal(body, &fields); err != nil { + return "", 0, s.permanentTokenErr(fmt.Errorf("token response from %s is not JSON: %w", + cfg.TokenURL, err)) + } + token, ok := fields[cfg.TokenResponseField].(string) + if !ok || token == "" { + return "", 0, s.permanentTokenErr(fmt.Errorf( + "token response from %s carries no %s", cfg.TokenURL, cfg.TokenResponseField)) + } + + // The skew is subtracted here for the same reason oauth2 subtracts it from + // expires_in: a request that passed the expiry check must not arrive after + // the token died. validate has already refused a ttl at or below it. + return token, cfg.TokenTTL - util.TokenRefreshSkew, nil +} + +// exchangeOAuth2Token performs the client_credentials grant. +func (s *Step) exchangeOAuth2Token(ctx context.Context, auth *authenticator) (string, time.Duration, error) { cfg := auth.cfg clientID, clientSecret := os.Getenv(cfg.ClientIDEnv), os.Getenv(cfg.ClientSecretEnv) if clientID == "" || clientSecret == "" { @@ -77,18 +156,51 @@ func (s *Step) exchangeToken(ctx context.Context, auth *authenticator) (string, "client_id": {clientID}, "client_secret": {clientSecret}, } + body, err := s.postForToken(ctx, cfg, []byte(form.Encode()), + "application/x-www-form-urlencoded") + if err != nil { + return "", 0, err + } + + var parsed tokenResponse + if err := json.Unmarshal(body, &parsed); err != nil { + return "", 0, s.permanentTokenErr(fmt.Errorf("token response from %s is not JSON: %w", + cfg.TokenURL, err)) + } + if parsed.AccessToken == "" { + return "", 0, s.permanentTokenErr( + fmt.Errorf("token response from %s carries no access_token", cfg.TokenURL)) + } + lifetime, ok := tokenLifetime(parsed.ExpiresIn, util.TokenRefreshSkew) + if !ok { + return "", 0, s.permanentTokenErr(fmt.Errorf( + "token response from %s carries no usable expires_in, so its lifetime is unknown", + cfg.TokenURL)) + } + return parsed.AccessToken, lifetime, nil +} + +// postForToken sends the exchange and returns the body of a 2xx, classifying +// every failure the same way for both schemes. +// +// Shared deliberately: the retry decision is the interesting part -- an +// unreachable endpoint, an unreadable response, a 5xx and a 429 are worth +// another attempt, and a 4xx is configuration that will fail identically next +// time. Two copies of that would drift. +func (s *Step) postForToken(ctx context.Context, cfg AuthProfile, payload []byte, + contentType string) ([]byte, error) { req, err := http.NewRequestWithContext(ctx, http.MethodPost, cfg.TokenURL, - strings.NewReader(form.Encode())) + strings.NewReader(string(payload))) if err != nil { // A tokenUrl that will not parse is configuration. - return "", 0, s.permanentTokenErr( + return nil, s.permanentTokenErr( fmt.Errorf("token request could not be built: %w", err)) } - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("Content-Type", contentType) resp, err := s.httpClient.Do(req) if err != nil { - return "", 0, s.tokenErr(fmt.Errorf("token endpoint %s could not be reached: %w", + return nil, s.tokenErr(fmt.Errorf("token endpoint %s could not be reached: %w", cfg.TokenURL, err)) } defer resp.Body.Close() @@ -96,7 +208,7 @@ func (s *Step) exchangeToken(ctx context.Context, auth *authenticator) (string, // Bounded like any other upstream read. body, err := io.ReadAll(io.LimitReader(resp.Body, s.config.MaxResponseBytes+1)) if err != nil { - return "", 0, s.tokenErr(fmt.Errorf("token response could not be read: %w", err)) + return nil, s.tokenErr(fmt.Errorf("token response could not be read: %w", err)) } if resp.StatusCode < 200 || resp.StatusCode >= 300 { // The status, not the body: a failure body routinely quotes the @@ -106,32 +218,14 @@ func (s *Step) exchangeToken(ctx context.Context, auth *authenticator) (string, err := fmt.Errorf("token endpoint returned %s", resp.Status) // The same rule the provider's own status gets: 5xx and 429 ask to be // tried again, and every other 4xx is a statement about the request -- - // here, usually that the client credentials are wrong. + // here, usually that the credentials are wrong. if resp.StatusCode < http.StatusInternalServerError && resp.StatusCode != http.StatusTooManyRequests { - return "", 0, s.permanentTokenErr(err) + return nil, s.permanentTokenErr(err) } - return "", 0, s.tokenErr(err) + return nil, s.tokenErr(err) } - - var parsed tokenResponse - if err := json.Unmarshal(body, &parsed); err != nil { - // A 2xx that is not JSON is the issuer misbehaving in a way another - // attempt will not change. - return "", 0, s.permanentTokenErr(fmt.Errorf("token response from %s is not JSON: %w", - cfg.TokenURL, err)) - } - if parsed.AccessToken == "" { - return "", 0, s.permanentTokenErr( - fmt.Errorf("token response from %s carries no access_token", cfg.TokenURL)) - } - lifetime, ok := tokenLifetime(parsed.ExpiresIn, util.TokenRefreshSkew) - if !ok { - return "", 0, s.permanentTokenErr(fmt.Errorf( - "token response from %s carries no usable expires_in, so its lifetime is unknown", - cfg.TokenURL)) - } - return parsed.AccessToken, lifetime, nil + return body, nil } // tokenErr classifies a failed exchange. Always 502: the failure is the diff --git a/pkg/plugin/implementation/internal/common/tokenquery_test.go b/pkg/plugin/implementation/internal/common/tokenquery_test.go new file mode 100644 index 00000000..45496fac --- /dev/null +++ b/pkg/plugin/implementation/internal/common/tokenquery_test.go @@ -0,0 +1,366 @@ +// tokenQuery: the exchange, the placement, the held token and its redaction. +// +// A separate file from common_test.go on purpose -- that one is already past +// two thousand lines, and everything here is about one scheme. +package common + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/beckn-one/beckn-onix/pkg/plugin/implementation/internal/common/util" +) + +// queryTokenServer stands in for a provider's own token endpoint: JSON in, a +// token under a name of its choosing out. +type queryTokenServer struct { + calls atomic.Int32 + field string // the response key; "token" when empty + token string // rotates per call when empty + status int + body string // overrides the JSON when set + gotUser string + gotSecret string + gotType string +} + +func (s *queryTokenServer) start(t *testing.T) string { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + count := s.calls.Add(1) + s.gotType = r.Header.Get("Content-Type") + + raw, _ := io.ReadAll(r.Body) + var sent map[string]string + _ = json.Unmarshal(raw, &sent) + s.gotUser, s.gotSecret = sent["access_name"], sent["password"] + + if s.status != 0 { + w.WriteHeader(s.status) + } + if s.body != "" { + fmt.Fprint(w, s.body) + return + } + token := s.token + if token == "" { + token = fmt.Sprintf("tok-%d", count) + } + field := s.field + if field == "" { + field = "token" + } + fmt.Fprintf(w, `{%q:%q}`, field, token) + })) + t.Cleanup(server.Close) + return server.URL +} + +// tokenQueryProfile is the fully-populated profile, for tests that then blank +// one field to prove it is required. +func tokenQueryProfile(tokenURL string) AuthProfile { + return AuthProfile{ + Scheme: util.AuthSchemeTokenQuery, + TokenURL: tokenURL, + TokenUserField: "access_name", + TokenUserEnv: "TEST_TQ_USER", + TokenSecretName: "password", + TokenSecretEnv: "TEST_TQ_SECRET", + TokenResponseField: "token", + QueryName: "token", + TokenTTLRaw: "10m", + } +} + +// tokenQueryStep records the query string each provider call arrived with. +func tokenQueryStep(t *testing.T, tokenURL string, seen *[]string, + status *int, tweak ...func(*AuthProfile)) *Step { + t.Helper() + provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + *seen = append(*seen, r.URL.RawQuery) + if status != nil && *status != 0 { + w.WriteHeader(*status) + fmt.Fprintf(w, `{"error":"Invalid token"}`) + return + } + fmt.Fprint(w, `{"ok":true}`) + })) + t.Cleanup(provider.Close) + + return newStep(t, &stubRegistry{plan: testPlan(provider.URL, http.MethodGet)}, + &stubMapper{requestResult: []byte(`{}`), responseResult: []byte(`{"a":1}`)}, + func(c *Config) { + profile := tokenQueryProfile(tokenURL) + for _, apply := range tweak { + apply(&profile) + } + // validate normally runs at startup; the profile carries a parsed + // TTL by the time a call is made. + if err := profile.validate(); err != nil { + t.Fatalf("profile does not validate: %v", err) + } + c.setProviderAuth(profile) + }) +} + +// The whole point of the scheme: credentials POSTed as JSON under CONFIGURED +// keys, and the token that comes back placed in the QUERY STRING. Neither half +// matches oauth2, which is why bending that scheme could not serve it. +func TestTokenQuerySendsTheExchangedTokenAsAQueryParameter(t *testing.T) { + t.Setenv("TEST_TQ_USER", "Masters-Data-Provider") + t.Setenv("TEST_TQ_SECRET", "the-password") + + ts := &queryTokenServer{token: "the-token"} + var seen []string + step := tokenQueryStep(t, ts.start(t), &seen, nil) + + if _, err := runStep(t, step, selectBody); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + + if len(seen) != 1 || !strings.Contains(seen[0], "token=the-token") { + t.Errorf("provider saw query %q, want it to carry token=the-token", seen) + } + if ts.gotType != "application/json" { + t.Errorf("token endpoint saw Content-Type %q, want application/json", ts.gotType) + } + if ts.gotUser != "Masters-Data-Provider" || ts.gotSecret != "the-password" { + t.Errorf("token endpoint saw %q/%q under access_name/password, want the configured pair", + ts.gotUser, ts.gotSecret) + } +} + +// The token is held for its configured lifetime. Without this the provider's +// token endpoint takes an extra round trip on every single call. +func TestTokenQueryReusesTheTokenWithinItsTTL(t *testing.T) { + t.Setenv("TEST_TQ_USER", "user") + t.Setenv("TEST_TQ_SECRET", "secret") + + ts := &queryTokenServer{} + var seen []string + step := tokenQueryStep(t, ts.start(t), &seen, nil) + + for i := 0; i < 5; i++ { + if _, err := runStep(t, step, selectBody); err != nil { + t.Fatalf("request %d: %v", i, err) + } + } + if got := ts.calls.Load(); got != 1 { + t.Errorf("token endpoint called %d times for 5 requests, want 1", got) + } + for i, query := range seen { + if !strings.Contains(query, "token=tok-1") { + t.Errorf("request %d sent %q, want the cached token", i, query) + } + } +} + +// THE SELF-HEAL. The lifetime is an operator's estimate -- the endpoint states +// no expiry -- so a too-generous tokenTtl leaves a dead token cached. Without +// dropping it on the provider's own rejection, every call fails until the ttl +// lapses, which for a ten-minute guess is a ten-minute outage. +func TestTokenQueryForgetsATokenTheProviderRejects(t *testing.T) { + t.Setenv("TEST_TQ_USER", "user") + t.Setenv("TEST_TQ_SECRET", "secret") + + ts := &queryTokenServer{} + var seen []string + status := http.StatusForbidden + step := tokenQueryStep(t, ts.start(t), &seen, &status) + + // First call: the provider rejects the freshly exchanged token. + if _, err := runStep(t, step, selectBody); err == nil { + t.Fatal("Run() succeeded against a 403, want an error") + } + if got := ts.calls.Load(); got != 1 { + t.Fatalf("token endpoint called %d times, want 1", got) + } + + // Second call: the held token must NOT be reused, even though the ttl has + // not lapsed. It is a fresh exchange or the scheme cannot recover. + status = 0 + if _, err := runStep(t, step, selectBody); err != nil { + t.Fatalf("second Run() = %v, want it to recover with a fresh token", err) + } + if got := ts.calls.Load(); got != 2 { + t.Errorf("token endpoint called %d times over two requests, want 2 -- "+ + "the rejected token was reused instead of re-exchanged", got) + } + if len(seen) != 2 || strings.Contains(seen[1], "token=tok-1") { + t.Errorf("second request sent %q, want a token other than the rejected tok-1", seen) + } +} + +// Every field is required, and the message has to name what is missing: an +// operator reading "requires tokenUrl, tokenUserField, ..." can fix it, while +// a malformed request rejected by the provider says nothing about the cause. +func TestTokenQueryRefusesAnIncompleteProfile(t *testing.T) { + for _, tc := range []struct { + field string + blank func(*AuthProfile) + }{ + {"tokenUrl", func(p *AuthProfile) { p.TokenURL = "" }}, + {"tokenUserField", func(p *AuthProfile) { p.TokenUserField = "" }}, + {"tokenUserEnv", func(p *AuthProfile) { p.TokenUserEnv = "" }}, + {"tokenSecretField", func(p *AuthProfile) { p.TokenSecretName = "" }}, + {"tokenSecretEnv", func(p *AuthProfile) { p.TokenSecretEnv = "" }}, + {"tokenResponseField", func(p *AuthProfile) { p.TokenResponseField = "" }}, + {"queryName", func(p *AuthProfile) { p.QueryName = "" }}, + } { + t.Run(tc.field, func(t *testing.T) { + profile := tokenQueryProfile("https://issuer.example/token") + profile.Provider = "a-provider" + tc.blank(&profile) + + err := profile.validate() + if err == nil { + t.Fatalf("validate() accepted a profile with no %s", tc.field) + } + if !strings.Contains(err.Error(), tc.field) { + t.Errorf("error %q does not name the missing %s", err, tc.field) + } + if !strings.Contains(err.Error(), "a-provider") { + t.Errorf("error %q does not name the provider, which an operator "+ + "needs to know which block to edit", err) + } + }) + } +} + +// tokenTtl is required, and it has a floor. At or below the refresh skew every +// token is already expired when it arrives, so each call would exchange one and +// throw it away -- two round trips per request, forever, with nothing failing +// loudly enough to notice. +func TestTokenQueryRefusesAnUnusableTTL(t *testing.T) { + for _, tc := range []struct{ name, ttl, wantIn string }{ + {"absent", "", "requires tokenTtl"}, + {"not a duration", "ten minutes", "is not a duration"}, + {"at the skew", util.TokenRefreshSkew.String(), "must exceed"}, + {"below the skew", "1s", "must exceed"}, + } { + t.Run(tc.name, func(t *testing.T) { + profile := tokenQueryProfile("https://issuer.example/token") + profile.Provider = "a-provider" + profile.TokenTTLRaw = tc.ttl + + err := profile.validate() + if err == nil { + t.Fatalf("validate() accepted tokenTtl %q", tc.ttl) + } + if !strings.Contains(err.Error(), tc.wantIn) { + t.Errorf("error %q does not contain %q", err, tc.wantIn) + } + }) + } +} + +// A ttl above the skew is accepted, and the held lifetime is shortened by it -- +// so a request that passed the expiry check cannot arrive after the token died. +func TestTokenQuerySubtractsTheSkewFromTheConfiguredTTL(t *testing.T) { + profile := tokenQueryProfile("https://issuer.example/token") + profile.TokenTTLRaw = "10m" + if err := profile.validate(); err != nil { + t.Fatalf("validate() = %v, want a 10m ttl accepted", err) + } + if profile.TokenTTL != 10*time.Minute { + t.Errorf("TokenTTL = %s, want 10m parsed from config", profile.TokenTTL) + } + // The subtraction itself happens in the exchange; this pins the arithmetic + // it depends on, so a change to the skew cannot silently invert it. + if profile.TokenTTL-util.TokenRefreshSkew <= 0 { + t.Errorf("held lifetime %s is not positive", profile.TokenTTL-util.TokenRefreshSkew) + } +} + +// A 2xx that does not carry the configured field is the endpoint misbehaving in +// a way another attempt will not change, and the message names the field so an +// operator can tell a misconfigured tokenResponseField from a broken provider. +func TestTokenQueryRefusesAResponseWithoutTheConfiguredField(t *testing.T) { + t.Setenv("TEST_TQ_USER", "user") + t.Setenv("TEST_TQ_SECRET", "secret") + + // The endpoint answers 200 with a different key than the one configured. + ts := &queryTokenServer{body: `{"access_token":"wrong-shape"}`} + var seen []string + step := tokenQueryStep(t, ts.start(t), &seen, nil) + + _, err := runStep(t, step, selectBody) + if err == nil { + t.Fatal("Run() succeeded on a response with no token field, want an error") + } + if !strings.Contains(err.Error(), "token") { + t.Errorf("error %q does not name the missing field", err) + } + if len(seen) != 0 { + t.Errorf("provider was called %d times despite no token, want 0", len(seen)) + } +} + +// The secret and the token must never reach a log or a returned error. The +// token matters more here than under oauth2: it travels in a query string, +// which transport errors quote whole. +func TestTokenQueryRedactsTheSecretAndTheToken(t *testing.T) { + t.Setenv("TEST_TQ_USER", "an-identifier") + t.Setenv("TEST_TQ_SECRET", "the-secret-value") + + ts := &queryTokenServer{token: "tok+with/escapes="} + var seen []string + step := tokenQueryStep(t, ts.start(t), &seen, nil) + + if _, err := runStep(t, step, selectBody); err != nil { + t.Fatalf("Run() returned an unexpected error: %v", err) + } + + // secretForms is what redaction substitutes; assert on it directly, since a + // passing request produces no error text to inspect. + var forms []string + for _, auth := range step.auth { + forms = append(forms, auth.secretForms()...) + } + joined := strings.Join(forms, "|") + + for _, want := range []string{"the-secret-value", "tok+with/escapes="} { + if !strings.Contains(joined, want) { + t.Errorf("secretForms %q does not cover %q", forms, want) + } + } + // The escaped form too, or a token with "+" survives redaction in a URL. + if !strings.Contains(joined, "tok%2Bwith%2Fescapes%3D") { + t.Errorf("secretForms %q does not cover the query-escaped token", forms) + } + // The identifier is not a credential and is left readable, so a log still + // says who this adapter authenticated as. + if strings.Contains(joined, "an-identifier") { + t.Errorf("secretForms %q redacts the identifier, which is not a secret", forms) + } +} + +// An unset credential is a configuration fault: it fails permanently rather +// than being retried, and the message names the variables without printing +// them. +func TestTokenQueryWithNoCredentialFailsWithoutRetrying(t *testing.T) { + t.Setenv("TEST_TQ_USER", "") + t.Setenv("TEST_TQ_SECRET", "") + + ts := &queryTokenServer{} + var seen []string + step := tokenQueryStep(t, ts.start(t), &seen, nil) + + _, err := runStep(t, step, selectBody) + if err == nil { + t.Fatal("Run() succeeded with no credential set, want an error") + } + if got := ts.calls.Load(); got != 0 { + t.Errorf("token endpoint called %d times with no credential, want 0", got) + } + if !util.IsPermanent(err) { + t.Errorf("error %v is retryable; an unset credential will not fix itself", err) + } +} diff --git a/pkg/plugin/implementation/internal/common/util/constant.go b/pkg/plugin/implementation/internal/common/util/constant.go index 56fa7e8b..df4cb1d1 100644 --- a/pkg/plugin/implementation/internal/common/util/constant.go +++ b/pkg/plugin/implementation/internal/common/util/constant.go @@ -63,13 +63,33 @@ const ( // Only client_credentials. That grant has no user to redirect and no // refresh token, so the other flows would be dead code. AuthSchemeOAuth2 = "oauth2" + + // Credentials are POSTed as JSON and the token that comes back goes in the + // QUERY STRING. For an upstream that issues short-lived tokens from its own + // endpoint rather than an OAuth2 one -- neither the request shape nor the + // placement matches oauth2, so it cannot be served by bending that scheme: + // + // oauth2 form-encoded client_id/client_secret -> access_token, + // sent as Authorization: Bearer + // tokenQuery JSON body of CONFIGURED field names -> a CONFIGURED field, + // sent as a query parameter + // + // The field names are configured rather than fixed because "access_name" + // and "password" are one provider's spelling, not a standard. + // + // Inherits query's exposure -- the token reaches proxy logs and transport + // errors -- so it is redacted the same way. See redact. + AuthSchemeTokenQuery = "tokenQuery" ) // CodeUpstreamUnavailable: the provider could not be reached or failed. Not // this adapter's fault and not the caller's. const CodeUpstreamUnavailable = "NET_DOWNSTREAM_UNAVAILABLE" -// How early an oauth2 token stops being trusted. Must exceed the round trip to -// the provider, so a request that passed the expiry check cannot arrive after -// the token died. Costs one extra exchange per lifetime. +// How early an exchanged token stops being trusted. Must exceed the round trip +// to the provider, so a request that passed the expiry check cannot arrive +// after the token died. Costs one extra exchange per lifetime. +// +// Applied to oauth2's expires_in and to tokenQuery's configured tokenTtl +// alike: both are a claim about a lifetime, and the same race sits under both. const TokenRefreshSkew = 60 * time.Second